rss-tools @ master

// Copyright 2013 The Go Authors. All rights reserved.

// Use of this source code is governed by a BSD-style

// license that can be found in the LICENSE file.

// Package charset provides common text encodings for HTML documents.

//

// The mapping from encoding labels to encodings is defined at

// https://encoding.spec.whatwg.org/.

package charset // import "golang.org/x/net/html/charset"

import (

	"bytes"

	"fmt"

	"io"

	"mime"

	"strings"

	"unicode/utf8"

	"golang.org/x/net/html"

	"golang.org/x/text/encoding"

	"golang.org/x/text/encoding/charmap"

	"golang.org/x/text/encoding/htmlindex"

	"golang.org/x/text/transform"

)

// Lookup returns the encoding with the specified label, and its canonical

// name. It returns nil and the empty string if label is not one of the

// standard encodings for HTML. Matching is case-insensitive and ignores

// leading and trailing whitespace. Encoders will use HTML escape sequences for

// runes that are not supported by the character set.

func Lookup(label string) (e encoding.Encoding, name string) {

	e, err := htmlindex.Get(label)

	if err != nil {

		return nil, ""

	}

	name, _ = htmlindex.Name(e)

	return &htmlEncoding{e}, name

}

type htmlEncoding struct{ encoding.Encoding }

func (h *htmlEncoding) NewEncoder() *encoding.Encoder {

	// HTML requires a non-terminating legacy encoder. We use HTML escapes to

	// substitute unsupported code points.

	return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder())

}

// DetermineEncoding determines the encoding of an HTML document by examining

// up to the first 1024 bytes of content and the declared Content-Type.

//

// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding

func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) {

	if len(content) > 1024 {

		content = content[:1024]

	}

	for _, b := range boms {

		if bytes.HasPrefix(content, b.bom) {

			e, name = Lookup(b.enc)

			return e, name, true

		}

	}

	if _, params, err := mime.ParseMediaType(contentType); err == nil {

		if cs, ok := params["charset"]; ok {

			if e, name = Lookup(cs); e != nil {

				return e, name, true

			}

		}

	}

	if len(content) > 0 {

		e, name = prescan(content)

		if e != nil {

			return e, name, false

		}

	}

	// Try to detect UTF-8.

	// First eliminate any partial rune at the end.

	for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {

		b := content[i]

		if b < 0x80 {

			break

		}

		if utf8.RuneStart(b) {

			content = content[:i]

			break

		}

	}

	hasHighBit := false

	for _, c := range content {

		if c >= 0x80 {

			hasHighBit = true

			break

		}

	}

	if hasHighBit && utf8.Valid(content) {

		return encoding.Nop, "utf-8", false

	}

	// TODO: change default depending on user's locale?

	return charmap.Windows1252, "windows-1252", false

}

// NewReader returns an io.Reader that converts the content of r to UTF-8.

// It calls DetermineEncoding to find out what r's encoding is.

func NewReader(r io.Reader, contentType string) (io.Reader, error) {

	preview := make([]byte, 1024)

	n, err := io.ReadFull(r, preview)

	switch {

	case err == io.ErrUnexpectedEOF:

		preview = preview[:n]

		r = bytes.NewReader(preview)

	case err != nil:

		return nil, err

	default:

		r = io.MultiReader(bytes.NewReader(preview), r)

	}

	if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {

		r = transform.NewReader(r, e.NewDecoder())

	}

	return r, nil

}

// NewReaderLabel returns a reader that converts from the specified charset to

// UTF-8. It uses Lookup to find the encoding that corresponds to label, and

// returns an error if Lookup returns nil. It is suitable for use as

// encoding/xml.Decoder's CharsetReader function.

func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {

	e, _ := Lookup(label)

	if e == nil {

		return nil, fmt.Errorf("unsupported charset: %q", label)

	}

	return transform.NewReader(input, e.NewDecoder()), nil

}

func prescan(content []byte) (e encoding.Encoding, name string) {

	z := html.NewTokenizer(bytes.NewReader(content))

	for {

		switch z.Next() {

		case html.ErrorToken:

			return nil, ""

		case html.StartTagToken, html.SelfClosingTagToken:

			tagName, hasAttr := z.TagName()

			if !bytes.Equal(tagName, []byte("meta")) {

				continue

			}

			attrList := make(map[string]bool)

			gotPragma := false

			const (

				dontKnow = iota

				doNeedPragma

				doNotNeedPragma

			)

			needPragma := dontKnow

			name = ""

			e = nil

			for hasAttr {

				var key, val []byte

				key, val, hasAttr = z.TagAttr()

				ks := string(key)

				if attrList[ks] {

					continue

				}

				attrList[ks] = true

				for i, c := range val {

					if 'A' <= c && c <= 'Z' {

						val[i] = c + 0x20

					}

				}

				switch ks {

				case "http-equiv":

					if bytes.Equal(val, []byte("content-type")) {

						gotPragma = true

					}

				case "content":

					if e == nil {

						name = fromMetaElement(string(val))

						if name != "" {

							e, name = Lookup(name)

							if e != nil {

								needPragma = doNeedPragma

							}

						}

					}

				case "charset":

					e, name = Lookup(string(val))

					needPragma = doNotNeedPragma

				}

			}

			if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {

				continue

			}

			if strings.HasPrefix(name, "utf-16") {

				name = "utf-8"

				e = encoding.Nop

			}

			if e != nil {

				return e, name

			}

		}

	}

}

func fromMetaElement(s string) string {

	for s != "" {

		csLoc := strings.Index(s, "charset")

		if csLoc == -1 {

			return ""

		}

		s = s[csLoc+len("charset"):]

		s = strings.TrimLeft(s, " \t\n\f\r")

		if !strings.HasPrefix(s, "=") {

			continue

		}

		s = s[1:]

		s = strings.TrimLeft(s, " \t\n\f\r")

		if s == "" {

			return ""

		}

		if q := s[0]; q == '"' || q == '\'' {

			s = s[1:]

			closeQuote := strings.IndexRune(s, rune(q))

			if closeQuote == -1 {

				return ""

			}

			return s[:closeQuote]

		}

		end := strings.IndexAny(s, "; \t\n\f\r")

		if end == -1 {

			end = len(s)

		}

		return s[:end]

	}

	return ""

}

var boms = []struct {

	bom []byte

	enc string

}{

	{[]byte{0xfe, 0xff}, "utf-16be"},

	{[]byte{0xff, 0xfe}, "utf-16le"},

	{[]byte{0xef, 0xbb, 0xbf}, "utf-8"},

}