letter.go

Documentation: unicode

		 1  // Copyright 2009 The Go Authors. All rights reserved.
		 2  // Use of this source code is governed by a BSD-style
		 3  // license that can be found in the LICENSE file.
		 4  
		 5  // Package unicode provides data and functions to test some properties of
		 6  // Unicode code points.
		 7  package unicode
		 8  
		 9  const (
		10  	MaxRune				 = '\U0010FFFF' // Maximum valid Unicode code point.
		11  	ReplacementChar = '\uFFFD'		 // Represents invalid code points.
		12  	MaxASCII				= '\u007F'		 // maximum ASCII value.
		13  	MaxLatin1			 = '\u00FF'		 // maximum Latin-1 value.
		14  )
		15  
		16  // RangeTable defines a set of Unicode code points by listing the ranges of
		17  // code points within the set. The ranges are listed in two slices
		18  // to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
		19  // The two slices must be in sorted order and non-overlapping.
		20  // Also, R32 should contain only values >= 0x10000 (1<<16).
		21  type RangeTable struct {
		22  	R16				 []Range16
		23  	R32				 []Range32
		24  	LatinOffset int // number of entries in R16 with Hi <= MaxLatin1
		25  }
		26  
		27  // Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi
		28  // inclusive and has the specified stride.
		29  type Range16 struct {
		30  	Lo		 uint16
		31  	Hi		 uint16
		32  	Stride uint16
		33  }
		34  
		35  // Range32 represents of a range of Unicode code points and is used when one or
		36  // more of the values will not fit in 16 bits. The range runs from Lo to Hi
		37  // inclusive and has the specified stride. Lo and Hi must always be >= 1<<16.
		38  type Range32 struct {
		39  	Lo		 uint32
		40  	Hi		 uint32
		41  	Stride uint32
		42  }
		43  
		44  // CaseRange represents a range of Unicode code points for simple (one
		45  // code point to one code point) case conversion.
		46  // The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas
		47  // are the number to add to the code point to reach the code point for a
		48  // different case for that character. They may be negative. If zero, it
		49  // means the character is in the corresponding case. There is a special
		50  // case representing sequences of alternating corresponding Upper and Lower
		51  // pairs. It appears with a fixed Delta of
		52  //	{UpperLower, UpperLower, UpperLower}
		53  // The constant UpperLower has an otherwise impossible delta value.
		54  type CaseRange struct {
		55  	Lo		uint32
		56  	Hi		uint32
		57  	Delta d
		58  }
		59  
		60  // SpecialCase represents language-specific case mappings such as Turkish.
		61  // Methods of SpecialCase customize (by overriding) the standard mappings.
		62  type SpecialCase []CaseRange
		63  
		64  // BUG(r): There is no mechanism for full case folding, that is, for
		65  // characters that involve multiple runes in the input or output.
		66  
		67  // Indices into the Delta arrays inside CaseRanges for case mapping.
		68  const (
		69  	UpperCase = iota
		70  	LowerCase
		71  	TitleCase
		72  	MaxCase
		73  )
		74  
		75  type d [MaxCase]rune // to make the CaseRanges text shorter
		76  
		77  // If the Delta field of a CaseRange is UpperLower, it means
		78  // this CaseRange represents a sequence of the form (say)
		79  // Upper Lower Upper Lower.
		80  const (
		81  	UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
		82  )
		83  
		84  // linearMax is the maximum size table for linear search for non-Latin1 rune.
		85  // Derived by running 'go test -calibrate'.
		86  const linearMax = 18
		87  
		88  // is16 reports whether r is in the sorted slice of 16-bit ranges.
		89  func is16(ranges []Range16, r uint16) bool {
		90  	if len(ranges) <= linearMax || r <= MaxLatin1 {
		91  		for i := range ranges {
		92  			range_ := &ranges[i]
		93  			if r < range_.Lo {
		94  				return false
		95  			}
		96  			if r <= range_.Hi {
		97  				return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
		98  			}
		99  		}
	 100  		return false
	 101  	}
	 102  
	 103  	// binary search over ranges
	 104  	lo := 0
	 105  	hi := len(ranges)
	 106  	for lo < hi {
	 107  		m := lo + (hi-lo)/2
	 108  		range_ := &ranges[m]
	 109  		if range_.Lo <= r && r <= range_.Hi {
	 110  			return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
	 111  		}
	 112  		if r < range_.Lo {
	 113  			hi = m
	 114  		} else {
	 115  			lo = m + 1
	 116  		}
	 117  	}
	 118  	return false
	 119  }
	 120  
	 121  // is32 reports whether r is in the sorted slice of 32-bit ranges.
	 122  func is32(ranges []Range32, r uint32) bool {
	 123  	if len(ranges) <= linearMax {
	 124  		for i := range ranges {
	 125  			range_ := &ranges[i]
	 126  			if r < range_.Lo {
	 127  				return false
	 128  			}
	 129  			if r <= range_.Hi {
	 130  				return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
	 131  			}
	 132  		}
	 133  		return false
	 134  	}
	 135  
	 136  	// binary search over ranges
	 137  	lo := 0
	 138  	hi := len(ranges)
	 139  	for lo < hi {
	 140  		m := lo + (hi-lo)/2
	 141  		range_ := ranges[m]
	 142  		if range_.Lo <= r && r <= range_.Hi {
	 143  			return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
	 144  		}
	 145  		if r < range_.Lo {
	 146  			hi = m
	 147  		} else {
	 148  			lo = m + 1
	 149  		}
	 150  	}
	 151  	return false
	 152  }
	 153  
	 154  // Is reports whether the rune is in the specified table of ranges.
	 155  func Is(rangeTab *RangeTable, r rune) bool {
	 156  	r16 := rangeTab.R16
	 157  	// Compare as uint32 to correctly handle negative runes.
	 158  	if len(r16) > 0 && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
	 159  		return is16(r16, uint16(r))
	 160  	}
	 161  	r32 := rangeTab.R32
	 162  	if len(r32) > 0 && r >= rune(r32[0].Lo) {
	 163  		return is32(r32, uint32(r))
	 164  	}
	 165  	return false
	 166  }
	 167  
	 168  func isExcludingLatin(rangeTab *RangeTable, r rune) bool {
	 169  	r16 := rangeTab.R16
	 170  	// Compare as uint32 to correctly handle negative runes.
	 171  	if off := rangeTab.LatinOffset; len(r16) > off && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
	 172  		return is16(r16[off:], uint16(r))
	 173  	}
	 174  	r32 := rangeTab.R32
	 175  	if len(r32) > 0 && r >= rune(r32[0].Lo) {
	 176  		return is32(r32, uint32(r))
	 177  	}
	 178  	return false
	 179  }
	 180  
	 181  // IsUpper reports whether the rune is an upper case letter.
	 182  func IsUpper(r rune) bool {
	 183  	// See comment in IsGraphic.
	 184  	if uint32(r) <= MaxLatin1 {
	 185  		return properties[uint8(r)]&pLmask == pLu
	 186  	}
	 187  	return isExcludingLatin(Upper, r)
	 188  }
	 189  
	 190  // IsLower reports whether the rune is a lower case letter.
	 191  func IsLower(r rune) bool {
	 192  	// See comment in IsGraphic.
	 193  	if uint32(r) <= MaxLatin1 {
	 194  		return properties[uint8(r)]&pLmask == pLl
	 195  	}
	 196  	return isExcludingLatin(Lower, r)
	 197  }
	 198  
	 199  // IsTitle reports whether the rune is a title case letter.
	 200  func IsTitle(r rune) bool {
	 201  	if r <= MaxLatin1 {
	 202  		return false
	 203  	}
	 204  	return isExcludingLatin(Title, r)
	 205  }
	 206  
	 207  // to maps the rune using the specified case mapping.
	 208  // It additionally reports whether caseRange contained a mapping for r.
	 209  func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) {
	 210  	if _case < 0 || MaxCase <= _case {
	 211  		return ReplacementChar, false // as reasonable an error as any
	 212  	}
	 213  	// binary search over ranges
	 214  	lo := 0
	 215  	hi := len(caseRange)
	 216  	for lo < hi {
	 217  		m := lo + (hi-lo)/2
	 218  		cr := caseRange[m]
	 219  		if rune(cr.Lo) <= r && r <= rune(cr.Hi) {
	 220  			delta := cr.Delta[_case]
	 221  			if delta > MaxRune {
	 222  				// In an Upper-Lower sequence, which always starts with
	 223  				// an UpperCase letter, the real deltas always look like:
	 224  				//	{0, 1, 0}		UpperCase (Lower is next)
	 225  				//	{-1, 0, -1}	LowerCase (Upper, Title are previous)
	 226  				// The characters at even offsets from the beginning of the
	 227  				// sequence are upper case; the ones at odd offsets are lower.
	 228  				// The correct mapping can be done by clearing or setting the low
	 229  				// bit in the sequence offset.
	 230  				// The constants UpperCase and TitleCase are even while LowerCase
	 231  				// is odd so we take the low bit from _case.
	 232  				return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)), true
	 233  			}
	 234  			return r + delta, true
	 235  		}
	 236  		if r < rune(cr.Lo) {
	 237  			hi = m
	 238  		} else {
	 239  			lo = m + 1
	 240  		}
	 241  	}
	 242  	return r, false
	 243  }
	 244  
	 245  // To maps the rune to the specified case: UpperCase, LowerCase, or TitleCase.
	 246  func To(_case int, r rune) rune {
	 247  	r, _ = to(_case, r, CaseRanges)
	 248  	return r
	 249  }
	 250  
	 251  // ToUpper maps the rune to upper case.
	 252  func ToUpper(r rune) rune {
	 253  	if r <= MaxASCII {
	 254  		if 'a' <= r && r <= 'z' {
	 255  			r -= 'a' - 'A'
	 256  		}
	 257  		return r
	 258  	}
	 259  	return To(UpperCase, r)
	 260  }
	 261  
	 262  // ToLower maps the rune to lower case.
	 263  func ToLower(r rune) rune {
	 264  	if r <= MaxASCII {
	 265  		if 'A' <= r && r <= 'Z' {
	 266  			r += 'a' - 'A'
	 267  		}
	 268  		return r
	 269  	}
	 270  	return To(LowerCase, r)
	 271  }
	 272  
	 273  // ToTitle maps the rune to title case.
	 274  func ToTitle(r rune) rune {
	 275  	if r <= MaxASCII {
	 276  		if 'a' <= r && r <= 'z' { // title case is upper case for ASCII
	 277  			r -= 'a' - 'A'
	 278  		}
	 279  		return r
	 280  	}
	 281  	return To(TitleCase, r)
	 282  }
	 283  
	 284  // ToUpper maps the rune to upper case giving priority to the special mapping.
	 285  func (special SpecialCase) ToUpper(r rune) rune {
	 286  	r1, hadMapping := to(UpperCase, r, []CaseRange(special))
	 287  	if r1 == r && !hadMapping {
	 288  		r1 = ToUpper(r)
	 289  	}
	 290  	return r1
	 291  }
	 292  
	 293  // ToTitle maps the rune to title case giving priority to the special mapping.
	 294  func (special SpecialCase) ToTitle(r rune) rune {
	 295  	r1, hadMapping := to(TitleCase, r, []CaseRange(special))
	 296  	if r1 == r && !hadMapping {
	 297  		r1 = ToTitle(r)
	 298  	}
	 299  	return r1
	 300  }
	 301  
	 302  // ToLower maps the rune to lower case giving priority to the special mapping.
	 303  func (special SpecialCase) ToLower(r rune) rune {
	 304  	r1, hadMapping := to(LowerCase, r, []CaseRange(special))
	 305  	if r1 == r && !hadMapping {
	 306  		r1 = ToLower(r)
	 307  	}
	 308  	return r1
	 309  }
	 310  
	 311  // caseOrbit is defined in tables.go as []foldPair. Right now all the
	 312  // entries fit in uint16, so use uint16. If that changes, compilation
	 313  // will fail (the constants in the composite literal will not fit in uint16)
	 314  // and the types here can change to uint32.
	 315  type foldPair struct {
	 316  	From uint16
	 317  	To	 uint16
	 318  }
	 319  
	 320  // SimpleFold iterates over Unicode code points equivalent under
	 321  // the Unicode-defined simple case folding. Among the code points
	 322  // equivalent to rune (including rune itself), SimpleFold returns the
	 323  // smallest rune > r if one exists, or else the smallest rune >= 0.
	 324  // If r is not a valid Unicode code point, SimpleFold(r) returns r.
	 325  //
	 326  // For example:
	 327  //	SimpleFold('A') = 'a'
	 328  //	SimpleFold('a') = 'A'
	 329  //
	 330  //	SimpleFold('K') = 'k'
	 331  //	SimpleFold('k') = '\u212A' (Kelvin symbol, K)
	 332  //	SimpleFold('\u212A') = 'K'
	 333  //
	 334  //	SimpleFold('1') = '1'
	 335  //
	 336  //	SimpleFold(-2) = -2
	 337  //
	 338  func SimpleFold(r rune) rune {
	 339  	if r < 0 || r > MaxRune {
	 340  		return r
	 341  	}
	 342  
	 343  	if int(r) < len(asciiFold) {
	 344  		return rune(asciiFold[r])
	 345  	}
	 346  
	 347  	// Consult caseOrbit table for special cases.
	 348  	lo := 0
	 349  	hi := len(caseOrbit)
	 350  	for lo < hi {
	 351  		m := lo + (hi-lo)/2
	 352  		if rune(caseOrbit[m].From) < r {
	 353  			lo = m + 1
	 354  		} else {
	 355  			hi = m
	 356  		}
	 357  	}
	 358  	if lo < len(caseOrbit) && rune(caseOrbit[lo].From) == r {
	 359  		return rune(caseOrbit[lo].To)
	 360  	}
	 361  
	 362  	// No folding specified. This is a one- or two-element
	 363  	// equivalence class containing rune and ToLower(rune)
	 364  	// and ToUpper(rune) if they are different from rune.
	 365  	if l := ToLower(r); l != r {
	 366  		return l
	 367  	}
	 368  	return ToUpper(r)
	 369  }
	 370
View as plain text