rss-tools/vendor/golang.org/x/text/encoding/unicode/override.go (view raw)
| 1 | // Copyright 2015 The Go Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style |
| 3 | // license that can be found in the LICENSE file. |
| 4 | |
| 5 | package unicode |
| 6 | |
| 7 | import ( |
| 8 | "golang.org/x/text/transform" |
| 9 | ) |
| 10 | |
| 11 | // BOMOverride returns a new decoder transformer that is identical to fallback, |
| 12 | // except that the presence of a Byte Order Mark at the start of the input |
| 13 | // causes it to switch to the corresponding Unicode decoding. It will only |
| 14 | // consider BOMs for UTF-8, UTF-16BE, and UTF-16LE. |
| 15 | // |
| 16 | // This differs from using ExpectBOM by allowing a BOM to switch to UTF-8, not |
| 17 | // just UTF-16 variants, and allowing falling back to any encoding scheme. |
| 18 | // |
| 19 | // This technique is recommended by the W3C for use in HTML 5: "For |
| 20 | // compatibility with deployed content, the byte order mark (also known as BOM) |
| 21 | // is considered more authoritative than anything else." |
| 22 | // http://www.w3.org/TR/encoding/#specification-hooks |
| 23 | // |
| 24 | // Using BOMOverride is mostly intended for use cases where the first characters |
| 25 | // of a fallback encoding are known to not be a BOM, for example, for valid HTML |
| 26 | // and most encodings. |
| 27 | func BOMOverride(fallback transform.Transformer) transform.Transformer { |
| 28 | // TODO: possibly allow a variadic argument of unicode encodings to allow |
| 29 | // specifying details of which fallbacks are supported as well as |
| 30 | // specifying the details of the implementations. This would also allow for |
| 31 | // support for UTF-32, which should not be supported by default. |
| 32 | return &bomOverride{fallback: fallback} |
| 33 | } |
| 34 | |
| 35 | type bomOverride struct { |
| 36 | fallback transform.Transformer |
| 37 | current transform.Transformer |
| 38 | } |
| 39 | |
| 40 | func (d *bomOverride) Reset() { |
| 41 | d.current = nil |
| 42 | d.fallback.Reset() |
| 43 | } |
| 44 | |
| 45 | var ( |
| 46 | // TODO: we could use decode functions here, instead of allocating a new |
| 47 | // decoder on every NewDecoder as IgnoreBOM decoders can be stateless. |
| 48 | utf16le = UTF16(LittleEndian, IgnoreBOM) |
| 49 | utf16be = UTF16(BigEndian, IgnoreBOM) |
| 50 | ) |
| 51 | |
| 52 | const utf8BOM = "\ufeff" |
| 53 | |
| 54 | func (d *bomOverride) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { |
| 55 | if d.current != nil { |
| 56 | return d.current.Transform(dst, src, atEOF) |
| 57 | } |
| 58 | if len(src) < 3 && !atEOF { |
| 59 | return 0, 0, transform.ErrShortSrc |
| 60 | } |
| 61 | d.current = d.fallback |
| 62 | bomSize := 0 |
| 63 | if len(src) >= 2 { |
| 64 | if src[0] == 0xFF && src[1] == 0xFE { |
| 65 | d.current = utf16le.NewDecoder() |
| 66 | bomSize = 2 |
| 67 | } else if src[0] == 0xFE && src[1] == 0xFF { |
| 68 | d.current = utf16be.NewDecoder() |
| 69 | bomSize = 2 |
| 70 | } else if len(src) >= 3 && |
| 71 | src[0] == utf8BOM[0] && |
| 72 | src[1] == utf8BOM[1] && |
| 73 | src[2] == utf8BOM[2] { |
| 74 | d.current = transform.Nop |
| 75 | bomSize = 3 |
| 76 | } |
| 77 | } |
| 78 | if bomSize < len(src) { |
| 79 | nDst, nSrc, err = d.current.Transform(dst, src[bomSize:], atEOF) |
| 80 | } |
| 81 | return nDst, nSrc + bomSize, err |
| 82 | } |