1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

package logic

// htmlToText turns a feed item's title or description, or the content of a
// note sent to the birb, into plain text.
//
// Every tag goes; their text stays, except that of the elements in
// textDropped, which is not text a reader sees. Text comes out unescaped:
// whoever puts it into HTML escapes it there (texts.WithVals does), and
// getUrl must see "&" in a URL, not "&amp;". Blocks and <br> become line
// breaks, so the text of two paragraphs does not run together, nor a URL into
// the word after it. Runs of white space become one space or line break, as
// a browser would show them.

import (
	"strings"
	"unicode"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

// textDropped are left out with everything in them. These are the elements
// whose content bluemonday's StrictPolicy, used before, skipped, plus
// template and svg.
var textDropped = map[string]bool{
	"script": true, "style": true, "title": true, "noscript": true,
	"iframe": true, "frame": true, "frameset": true, "noframes": true,
	"noembed": true, "nostyle": true, "object": true,
	"template": true, "svg": true,
}

// textBlocks start and end a line.
var textBlocks = map[atom.Atom]bool{
	atom.P: true, atom.Div: true, atom.Br: true, atom.Hr: true,
	atom.Section: true, atom.Article: true, atom.Header: true, atom.Footer: true,
	atom.Main: true, atom.Nav: true, atom.Aside: true, atom.Address: true,
	atom.H1: true, atom.H2: true, atom.H3: true, atom.H4: true, atom.H5: true, atom.H6: true,
	atom.Ul: true, atom.Ol: true, atom.Li: true, atom.Dl: true, atom.Dt: true, atom.Dd: true,
	atom.Table: true, atom.Thead: true, atom.Tbody: true, atom.Tfoot: true, atom.Tr: true,
	atom.Caption: true, atom.Blockquote: true, atom.Pre: true,
	atom.Figure: true, atom.Figcaption: true,
}

// textCells are set apart by a space, so that a row's cells do not run
// together.
var textCells = map[atom.Atom]bool{atom.Td: true, atom.Th: true}

func htmlToText(s string) string {
	body := &html.Node{Type: html.ElementNode, Data: "body", DataAtom: atom.Body}
	// Reading from a strings.Reader cannot fail, and the parser repairs
	// any markup, so there is no error to handle.
	nodes, _ := html.ParseFragment(strings.NewReader(s), body)
	var w textWriter
	for _, n := range nodes {
		w.walk(n)
	}
	return w.b.String()
}

// textWriter writes text with its white space collapsed. A break, a space or
// a line break, is only written before the next text, so that there is none
// at the start or the end, and a line break wins over a space.
type textWriter struct {
	b     strings.Builder
	space bool
	line  bool
}

func (w *textWriter) walk(n *html.Node) {
	switch n.Type {
	case html.TextNode:
		w.text(n.Data)
		return
	case html.ElementNode:
		if textDropped[n.Data] {
			return
		}
	case html.DocumentNode:
	default: // comments, doctypes
		return
	}
	block := textBlocks[n.DataAtom]
	if block {
		w.line = true
	}
	if textCells[n.DataAtom] {
		w.space = true
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		w.walk(c)
	}
	if block {
		w.line = true
	}
}

func (w *textWriter) text(s string) {
	for _, r := range s {
		if unicode.IsSpace(r) {
			w.space = true
			continue
		}
		if w.b.Len() > 0 {
			if w.line {
				w.b.WriteByte('\n')
			} else if w.space {
				w.b.WriteByte(' ')
			}
		}
		w.space, w.line = false, false
		w.b.WriteRune(r)
	}
}