rss-tools/vendor/golang.org/x/text/internal/language/lookup.go (view raw)
| 1 | // Copyright 2013 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | package language |
| 6 | |
| 7 | import ( |
| 8 | "bytes" |
| 9 | "fmt" |
| 10 | "sort" |
| 11 | "strconv" |
| 12 | |
| 13 | "golang.org/x/text/internal/tag" |
| 14 | ) |
| 15 | |
| 16 | // findIndex tries to find the given tag in idx and returns a standardized error |
| 17 | // if it could not be found. |
| 18 | func findIndex(idx tag.Index, key []byte, form string) (index int, err error) { |
| 19 | if !tag.FixCase(form, key) { |
| 20 | return 0, ErrSyntax |
| 21 | } |
| 22 | i := idx.Index(key) |
| 23 | if i == -1 { |
| 24 | return 0, NewValueError(key) |
| 25 | } |
| 26 | return i, nil |
| 27 | } |
| 28 | |
| 29 | func searchUint(imap []uint16, key uint16) int { |
| 30 | return sort.Search(len(imap), func(i int) bool { |
| 31 | return imap[i] >= key |
| 32 | }) |
| 33 | } |
| 34 | |
| 35 | type Language uint16 |
| 36 | |
| 37 | // getLangID returns the langID of s if s is a canonical subtag |
| 38 | // or langUnknown if s is not a canonical subtag. |
| 39 | func getLangID(s []byte) (Language, error) { |
| 40 | if len(s) == 2 { |
| 41 | return getLangISO2(s) |
| 42 | } |
| 43 | return getLangISO3(s) |
| 44 | } |
| 45 | |
| 46 | // TODO language normalization as well as the AliasMaps could be moved to the |
| 47 | // higher level package, but it is a bit tricky to separate the generation. |
| 48 | |
| 49 | func (id Language) Canonicalize() (Language, AliasType) { |
| 50 | return normLang(id) |
| 51 | } |
| 52 | |
| 53 | // normLang returns the mapped langID of id according to mapping m. |
| 54 | func normLang(id Language) (Language, AliasType) { |
| 55 | k := sort.Search(len(AliasMap), func(i int) bool { |
| 56 | return AliasMap[i].From >= uint16(id) |
| 57 | }) |
| 58 | if k < len(AliasMap) && AliasMap[k].From == uint16(id) { |
| 59 | return Language(AliasMap[k].To), AliasTypes[k] |
| 60 | } |
| 61 | return id, AliasTypeUnknown |
| 62 | } |
| 63 | |
| 64 | // getLangISO2 returns the langID for the given 2-letter ISO language code |
| 65 | // or unknownLang if this does not exist. |
| 66 | func getLangISO2(s []byte) (Language, error) { |
| 67 | if !tag.FixCase("zz", s) { |
| 68 | return 0, ErrSyntax |
| 69 | } |
| 70 | if i := lang.Index(s); i != -1 && lang.Elem(i)[3] != 0 { |
| 71 | return Language(i), nil |
| 72 | } |
| 73 | return 0, NewValueError(s) |
| 74 | } |
| 75 | |
| 76 | const base = 'z' - 'a' + 1 |
| 77 | |
| 78 | func strToInt(s []byte) uint { |
| 79 | v := uint(0) |
| 80 | for i := 0; i < len(s); i++ { |
| 81 | v *= base |
| 82 | v += uint(s[i] - 'a') |
| 83 | } |
| 84 | return v |
| 85 | } |
| 86 | |
| 87 | // converts the given integer to the original ASCII string passed to strToInt. |
| 88 | // len(s) must match the number of characters obtained. |
| 89 | func intToStr(v uint, s []byte) { |
| 90 | for i := len(s) - 1; i >= 0; i-- { |
| 91 | s[i] = byte(v%base) + 'a' |
| 92 | v /= base |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | // getLangISO3 returns the langID for the given 3-letter ISO language code |
| 97 | // or unknownLang if this does not exist. |
| 98 | func getLangISO3(s []byte) (Language, error) { |
| 99 | if tag.FixCase("und", s) { |
| 100 | // first try to match canonical 3-letter entries |
| 101 | for i := lang.Index(s[:2]); i != -1; i = lang.Next(s[:2], i) { |
| 102 | if e := lang.Elem(i); e[3] == 0 && e[2] == s[2] { |
| 103 | // We treat "und" as special and always translate it to "unspecified". |
| 104 | // Note that ZZ and Zzzz are private use and are not treated as |
| 105 | // unspecified by default. |
| 106 | id := Language(i) |
| 107 | if id == nonCanonicalUnd { |
| 108 | return 0, nil |
| 109 | } |
| 110 | return id, nil |
| 111 | } |
| 112 | } |
| 113 | if i := altLangISO3.Index(s); i != -1 { |
| 114 | return Language(altLangIndex[altLangISO3.Elem(i)[3]]), nil |
| 115 | } |
| 116 | n := strToInt(s) |
| 117 | if langNoIndex[n/8]&(1<<(n%8)) != 0 { |
| 118 | return Language(n) + langNoIndexOffset, nil |
| 119 | } |
| 120 | // Check for non-canonical uses of ISO3. |
| 121 | for i := lang.Index(s[:1]); i != -1; i = lang.Next(s[:1], i) { |
| 122 | if e := lang.Elem(i); e[2] == s[1] && e[3] == s[2] { |
| 123 | return Language(i), nil |
| 124 | } |
| 125 | } |
| 126 | return 0, NewValueError(s) |
| 127 | } |
| 128 | return 0, ErrSyntax |
| 129 | } |
| 130 | |
| 131 | // StringToBuf writes the string to b and returns the number of bytes |
| 132 | // written. cap(b) must be >= 3. |
| 133 | func (id Language) StringToBuf(b []byte) int { |
| 134 | if id >= langNoIndexOffset { |
| 135 | intToStr(uint(id)-langNoIndexOffset, b[:3]) |
| 136 | return 3 |
| 137 | } else if id == 0 { |
| 138 | return copy(b, "und") |
| 139 | } |
| 140 | l := lang[id<<2:] |
| 141 | if l[3] == 0 { |
| 142 | return copy(b, l[:3]) |
| 143 | } |
| 144 | return copy(b, l[:2]) |
| 145 | } |
| 146 | |
| 147 | // String returns the BCP 47 representation of the langID. |
| 148 | // Use b as variable name, instead of id, to ensure the variable |
| 149 | // used is consistent with that of Base in which this type is embedded. |
| 150 | func (b Language) String() string { |
| 151 | if b == 0 { |
| 152 | return "und" |
| 153 | } else if b >= langNoIndexOffset { |
| 154 | b -= langNoIndexOffset |
| 155 | buf := [3]byte{} |
| 156 | intToStr(uint(b), buf[:]) |
| 157 | return string(buf[:]) |
| 158 | } |
| 159 | l := lang.Elem(int(b)) |
| 160 | if l[3] == 0 { |
| 161 | return l[:3] |
| 162 | } |
| 163 | return l[:2] |
| 164 | } |
| 165 | |
| 166 | // ISO3 returns the ISO 639-3 language code. |
| 167 | func (b Language) ISO3() string { |
| 168 | if b == 0 || b >= langNoIndexOffset { |
| 169 | return b.String() |
| 170 | } |
| 171 | l := lang.Elem(int(b)) |
| 172 | if l[3] == 0 { |
| 173 | return l[:3] |
| 174 | } else if l[2] == 0 { |
| 175 | return altLangISO3.Elem(int(l[3]))[:3] |
| 176 | } |
| 177 | // This allocation will only happen for 3-letter ISO codes |
| 178 | // that are non-canonical BCP 47 language identifiers. |
| 179 | return l[0:1] + l[2:4] |
| 180 | } |
| 181 | |
| 182 | // IsPrivateUse reports whether this language code is reserved for private use. |
| 183 | func (b Language) IsPrivateUse() bool { |
| 184 | return langPrivateStart <= b && b <= langPrivateEnd |
| 185 | } |
| 186 | |
| 187 | // SuppressScript returns the script marked as SuppressScript in the IANA |
| 188 | // language tag repository, or 0 if there is no such script. |
| 189 | func (b Language) SuppressScript() Script { |
| 190 | if b < langNoIndexOffset { |
| 191 | return Script(suppressScript[b]) |
| 192 | } |
| 193 | return 0 |
| 194 | } |
| 195 | |
| 196 | type Region uint16 |
| 197 | |
| 198 | // getRegionID returns the region id for s if s is a valid 2-letter region code |
| 199 | // or unknownRegion. |
| 200 | func getRegionID(s []byte) (Region, error) { |
| 201 | if len(s) == 3 { |
| 202 | if isAlpha(s[0]) { |
| 203 | return getRegionISO3(s) |
| 204 | } |
| 205 | if i, err := strconv.ParseUint(string(s), 10, 10); err == nil { |
| 206 | return getRegionM49(int(i)) |
| 207 | } |
| 208 | } |
| 209 | return getRegionISO2(s) |
| 210 | } |
| 211 | |
| 212 | // getRegionISO2 returns the regionID for the given 2-letter ISO country code |
| 213 | // or unknownRegion if this does not exist. |
| 214 | func getRegionISO2(s []byte) (Region, error) { |
| 215 | i, err := findIndex(regionISO, s, "ZZ") |
| 216 | if err != nil { |
| 217 | return 0, err |
| 218 | } |
| 219 | return Region(i) + isoRegionOffset, nil |
| 220 | } |
| 221 | |
| 222 | // getRegionISO3 returns the regionID for the given 3-letter ISO country code |
| 223 | // or unknownRegion if this does not exist. |
| 224 | func getRegionISO3(s []byte) (Region, error) { |
| 225 | if tag.FixCase("ZZZ", s) { |
| 226 | for i := regionISO.Index(s[:1]); i != -1; i = regionISO.Next(s[:1], i) { |
| 227 | if e := regionISO.Elem(i); e[2] == s[1] && e[3] == s[2] { |
| 228 | return Region(i) + isoRegionOffset, nil |
| 229 | } |
| 230 | } |
| 231 | for i := 0; i < len(altRegionISO3); i += 3 { |
| 232 | if tag.Compare(altRegionISO3[i:i+3], s) == 0 { |
| 233 | return Region(altRegionIDs[i/3]), nil |
| 234 | } |
| 235 | } |
| 236 | return 0, NewValueError(s) |
| 237 | } |
| 238 | return 0, ErrSyntax |
| 239 | } |
| 240 | |
| 241 | func getRegionM49(n int) (Region, error) { |
| 242 | if 0 < n && n <= 999 { |
| 243 | const ( |
| 244 | searchBits = 7 |
| 245 | regionBits = 9 |
| 246 | regionMask = 1<<regionBits - 1 |
| 247 | ) |
| 248 | idx := n >> searchBits |
| 249 | buf := fromM49[m49Index[idx]:m49Index[idx+1]] |
| 250 | val := uint16(n) << regionBits // we rely on bits shifting out |
| 251 | i := sort.Search(len(buf), func(i int) bool { |
| 252 | return buf[i] >= val |
| 253 | }) |
| 254 | if r := fromM49[int(m49Index[idx])+i]; r&^regionMask == val { |
| 255 | return Region(r & regionMask), nil |
| 256 | } |
| 257 | } |
| 258 | var e ValueError |
| 259 | fmt.Fprint(bytes.NewBuffer([]byte(e.v[:])), n) |
| 260 | return 0, e |
| 261 | } |
| 262 | |
| 263 | // normRegion returns a region if r is deprecated or 0 otherwise. |
| 264 | // TODO: consider supporting BYS (-> BLR), CSK (-> 200 or CZ), PHI (-> PHL) and AFI (-> DJ). |
| 265 | // TODO: consider mapping split up regions to new most populous one (like CLDR). |
| 266 | func normRegion(r Region) Region { |
| 267 | m := regionOldMap |
| 268 | k := sort.Search(len(m), func(i int) bool { |
| 269 | return m[i].From >= uint16(r) |
| 270 | }) |
| 271 | if k < len(m) && m[k].From == uint16(r) { |
| 272 | return Region(m[k].To) |
| 273 | } |
| 274 | return 0 |
| 275 | } |
| 276 | |
| 277 | const ( |
| 278 | iso3166UserAssigned = 1 << iota |
| 279 | ccTLD |
| 280 | bcp47Region |
| 281 | ) |
| 282 | |
| 283 | func (r Region) typ() byte { |
| 284 | return regionTypes[r] |
| 285 | } |
| 286 | |
| 287 | // String returns the BCP 47 representation for the region. |
| 288 | // It returns "ZZ" for an unspecified region. |
| 289 | func (r Region) String() string { |
| 290 | if r < isoRegionOffset { |
| 291 | if r == 0 { |
| 292 | return "ZZ" |
| 293 | } |
| 294 | return fmt.Sprintf("%03d", r.M49()) |
| 295 | } |
| 296 | r -= isoRegionOffset |
| 297 | return regionISO.Elem(int(r))[:2] |
| 298 | } |
| 299 | |
| 300 | // ISO3 returns the 3-letter ISO code of r. |
| 301 | // Note that not all regions have a 3-letter ISO code. |
| 302 | // In such cases this method returns "ZZZ". |
| 303 | func (r Region) ISO3() string { |
| 304 | if r < isoRegionOffset { |
| 305 | return "ZZZ" |
| 306 | } |
| 307 | r -= isoRegionOffset |
| 308 | reg := regionISO.Elem(int(r)) |
| 309 | switch reg[2] { |
| 310 | case 0: |
| 311 | return altRegionISO3[reg[3]:][:3] |
| 312 | case ' ': |
| 313 | return "ZZZ" |
| 314 | } |
| 315 | return reg[0:1] + reg[2:4] |
| 316 | } |
| 317 | |
| 318 | // M49 returns the UN M.49 encoding of r, or 0 if this encoding |
| 319 | // is not defined for r. |
| 320 | func (r Region) M49() int { |
| 321 | return int(m49[r]) |
| 322 | } |
| 323 | |
| 324 | // IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This |
| 325 | // may include private-use tags that are assigned by CLDR and used in this |
| 326 | // implementation. So IsPrivateUse and IsCountry can be simultaneously true. |
| 327 | func (r Region) IsPrivateUse() bool { |
| 328 | return r.typ()&iso3166UserAssigned != 0 |
| 329 | } |
| 330 | |
| 331 | type Script uint16 |
| 332 | |
| 333 | // getScriptID returns the script id for string s. It assumes that s |
| 334 | // is of the format [A-Z][a-z]{3}. |
| 335 | func getScriptID(idx tag.Index, s []byte) (Script, error) { |
| 336 | i, err := findIndex(idx, s, "Zzzz") |
| 337 | return Script(i), err |
| 338 | } |
| 339 | |
| 340 | // String returns the script code in title case. |
| 341 | // It returns "Zzzz" for an unspecified script. |
| 342 | func (s Script) String() string { |
| 343 | if s == 0 { |
| 344 | return "Zzzz" |
| 345 | } |
| 346 | return script.Elem(int(s)) |
| 347 | } |
| 348 | |
| 349 | // IsPrivateUse reports whether this script code is reserved for private use. |
| 350 | func (s Script) IsPrivateUse() bool { |
| 351 | return _Qaaa <= s && s <= _Qabx |
| 352 | } |
| 353 | |
| 354 | const ( |
| 355 | maxAltTaglen = len("en-US-POSIX") |
| 356 | maxLen = maxAltTaglen |
| 357 | ) |
| 358 | |
| 359 | var ( |
| 360 | // grandfatheredMap holds a mapping from legacy and grandfathered tags to |
| 361 | // their base language or index to more elaborate tag. |
| 362 | grandfatheredMap = map[[maxLen]byte]int16{ |
| 363 | [maxLen]byte{'a', 'r', 't', '-', 'l', 'o', 'j', 'b', 'a', 'n'}: _jbo, // art-lojban |
| 364 | [maxLen]byte{'i', '-', 'a', 'm', 'i'}: _ami, // i-ami |
| 365 | [maxLen]byte{'i', '-', 'b', 'n', 'n'}: _bnn, // i-bnn |
| 366 | [maxLen]byte{'i', '-', 'h', 'a', 'k'}: _hak, // i-hak |
| 367 | [maxLen]byte{'i', '-', 'k', 'l', 'i', 'n', 'g', 'o', 'n'}: _tlh, // i-klingon |
| 368 | [maxLen]byte{'i', '-', 'l', 'u', 'x'}: _lb, // i-lux |
| 369 | [maxLen]byte{'i', '-', 'n', 'a', 'v', 'a', 'j', 'o'}: _nv, // i-navajo |
| 370 | [maxLen]byte{'i', '-', 'p', 'w', 'n'}: _pwn, // i-pwn |
| 371 | [maxLen]byte{'i', '-', 't', 'a', 'o'}: _tao, // i-tao |
| 372 | [maxLen]byte{'i', '-', 't', 'a', 'y'}: _tay, // i-tay |
| 373 | [maxLen]byte{'i', '-', 't', 's', 'u'}: _tsu, // i-tsu |
| 374 | [maxLen]byte{'n', 'o', '-', 'b', 'o', 'k'}: _nb, // no-bok |
| 375 | [maxLen]byte{'n', 'o', '-', 'n', 'y', 'n'}: _nn, // no-nyn |
| 376 | [maxLen]byte{'s', 'g', 'n', '-', 'b', 'e', '-', 'f', 'r'}: _sfb, // sgn-BE-FR |
| 377 | [maxLen]byte{'s', 'g', 'n', '-', 'b', 'e', '-', 'n', 'l'}: _vgt, // sgn-BE-NL |
| 378 | [maxLen]byte{'s', 'g', 'n', '-', 'c', 'h', '-', 'd', 'e'}: _sgg, // sgn-CH-DE |
| 379 | [maxLen]byte{'z', 'h', '-', 'g', 'u', 'o', 'y', 'u'}: _cmn, // zh-guoyu |
| 380 | [maxLen]byte{'z', 'h', '-', 'h', 'a', 'k', 'k', 'a'}: _hak, // zh-hakka |
| 381 | [maxLen]byte{'z', 'h', '-', 'm', 'i', 'n', '-', 'n', 'a', 'n'}: _nan, // zh-min-nan |
| 382 | [maxLen]byte{'z', 'h', '-', 'x', 'i', 'a', 'n', 'g'}: _hsn, // zh-xiang |
| 383 | |
| 384 | // Grandfathered tags with no modern replacement will be converted as |
| 385 | // follows: |
| 386 | [maxLen]byte{'c', 'e', 'l', '-', 'g', 'a', 'u', 'l', 'i', 's', 'h'}: -1, // cel-gaulish |
| 387 | [maxLen]byte{'e', 'n', '-', 'g', 'b', '-', 'o', 'e', 'd'}: -2, // en-GB-oed |
| 388 | [maxLen]byte{'i', '-', 'd', 'e', 'f', 'a', 'u', 'l', 't'}: -3, // i-default |
| 389 | [maxLen]byte{'i', '-', 'e', 'n', 'o', 'c', 'h', 'i', 'a', 'n'}: -4, // i-enochian |
| 390 | [maxLen]byte{'i', '-', 'm', 'i', 'n', 'g', 'o'}: -5, // i-mingo |
| 391 | [maxLen]byte{'z', 'h', '-', 'm', 'i', 'n'}: -6, // zh-min |
| 392 | |
| 393 | // CLDR-specific tag. |
| 394 | [maxLen]byte{'r', 'o', 'o', 't'}: 0, // root |
| 395 | [maxLen]byte{'e', 'n', '-', 'u', 's', '-', 'p', 'o', 's', 'i', 'x'}: -7, // en_US_POSIX" |
| 396 | } |
| 397 | |
| 398 | altTagIndex = [...]uint8{0, 17, 31, 45, 61, 74, 86, 102} |
| 399 | |
| 400 | altTags = "xtg-x-cel-gaulishen-GB-oxendicten-x-i-defaultund-x-i-enochiansee-x-i-mingonan-x-zh-minen-US-u-va-posix" |
| 401 | ) |
| 402 | |
| 403 | func grandfathered(s [maxAltTaglen]byte) (t Tag, ok bool) { |
| 404 | if v, ok := grandfatheredMap[s]; ok { |
| 405 | if v < 0 { |
| 406 | return Make(altTags[altTagIndex[-v-1]:altTagIndex[-v]]), true |
| 407 | } |
| 408 | t.LangID = Language(v) |
| 409 | return t, true |
| 410 | } |
| 411 | return t, false |
| 412 | } |