1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274

package logic

// readSiteMeta finds what a requested web page says about itself: the feed it
// names, its title and its description.
//
// They belong in the page's head, and the whole head is read: a page may
// name several feeds, and which of them is followed is decided among all of
// them, not by whichever comes first.
//
// They are not always in the head, though. YouTube closes the head after
// some 770 KB of script and style and then writes a second block of head
// tags – the canonical link, the title, the description and the feed –
// inside the body; Lemmy writes the feed link next to the RSS icon in the
// post list; and a theme that closes the head too early leaves WordPress's
// feed links in the body. A browser takes a <link> anywhere, so when the
// head named no feed, or no title, the body is read on for what is missing,
// and stopped at as soon as it is found: at the first feed link that is not
// a comment feed, and at the first title. What the head said is never
// overruled by the body.
//
// Of the feeds the head names, a comment feed is taken only if it names no
// other, and of the rest the first Atom feed is taken, else the first JSON
// Feed, else the first RSS feed: Atom and JSON Feed require every item to
// have an id and Atom a date, which is what kakapo tells items apart by (see
// items.go), and RSS requires neither. A link names a feed if one of the
// words of its rel is "alternate" and its type is one of the three, both in
// any case and the type without parameters such as "; charset=utf-8".
//
// Whatever happens, no more than siteMetaLimit bytes are read.

import (
	"io"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

const siteMetaLimit = 1 << 20

// feedTypes are the media types of the feeds a page may name, most
// preferred first.
var feedTypes = []string{"application/atom+xml", "application/feed+json", "application/rss+xml"}

// commentFeedWords mark a feed of the comments on a site's posts, which a
// WordPress page names right after the posts themselves. Nobody asking for a
// site wants to follow its comments.
var commentFeedWords = []string{"comment", "kommentar", "commentaire", "comentário", "comentarios"}

type siteMeta struct {
	FeedHref    string // as written in the page, not yet resolved
	Title       string
	Description string
}

func readSiteMeta(r io.Reader) siteMeta {
	m := metaReader{z: html.NewTokenizer(io.LimitReader(r, siteMetaLimit)), inHead: true}
	m.read()
	return siteMeta{FeedHref: m.feed, Title: m.title, Description: m.desc}
}

type metaReader struct {
	z *html.Tokenizer

	feed, title, desc string
	// feedRank is the rank of feed while it came from the head; see
	// rankHeadFeed. A feed from the body is not ranked: it is the first that
	// was good enough.
	feedRank int
	// commentFeed is the best comment feed of the body, taken if the body
	// names no other feed.
	commentFeed string

	inHead bool
	// drawing counts the <svg> and <math> elements around: in one, a <title>
	// names a drawing, not the page. Lemmy's RSS icon holds a
	// <title>rss</title>.
	drawing int
}

func (m *metaReader) read() {
	m.feedRank = rankWorst
	for {
		switch m.z.Next() {
		case html.ErrorToken:
			// The body was read to its end (or to the limit) without the
			// feed it was read for; a comment feed will do after all.
			if m.feed == "" {
				m.feed = m.commentFeed
			}
			return
		case html.StartTagToken:
			if m.startTag(true) {
				return
			}
		case html.SelfClosingTagToken:
			if m.startTag(false) {
				return
			}
		case html.EndTagToken:
			if m.endTag() {
				return
			}
		}
	}
}

// startTag handles a start tag, and reports whether reading is done. open is
// false for one that closes itself, <link/>, whose content there is none of.
func (m *metaReader) startTag(open bool) bool {
	name, hasAttr := m.z.TagName()
	a := atom.Lookup(name)
	switch a {
	case atom.Body:
		// A page may leave out </head>; its <body> ends the head too.
		m.inHead = false
		return m.done()
	case atom.Svg, atom.Math:
		if open {
			m.drawing++
		}
		return false
	case atom.Title:
		if open && m.drawing == 0 && m.title == "" {
			m.title = collapseSpace(m.text(atom.Title))
			return m.done()
		}
		return false
	}
	if !hasAttr || (a != atom.Link && a != atom.Meta) {
		return false
	}
	attrs := map[string]string{}
	for more := true; more; {
		var k, v []byte
		k, v, more = m.z.TagAttr()
		if _, seen := attrs[string(k)]; !seen {
			attrs[string(k)] = string(v)
		}
	}
	switch a {
	case atom.Link:
		return m.feedLink(attrs)
	case atom.Meta:
		if m.desc == "" && strings.EqualFold(strings.TrimSpace(attrs["name"]), "description") {
			m.desc = collapseSpace(attrs["content"])
		}
	}
	return false
}

// endTag handles an end tag, and reports whether reading is done.
func (m *metaReader) endTag() bool {
	switch name, _ := m.z.TagName(); atom.Lookup(name) {
	case atom.Head:
		m.inHead = false
		return m.done()
	case atom.Svg, atom.Math:
		m.drawing = max(m.drawing-1, 0)
	}
	return false
}

// feedLink takes in a <link>, and reports whether reading is done.
//
// In the head every feed link is weighed against the ones before it. In the
// body the first that is not a comment feed is taken as it is; a comment
// feed is remembered in case the body names no other.
func (m *metaReader) feedLink(attrs map[string]string) bool {
	href := strings.TrimSpace(attrs["href"])
	typ := feedType(attrs["type"])
	if href == "" || typ == "" || !hasWord(attrs["rel"], "alternate") {
		return false
	}
	comment := isCommentFeed(attrs["title"], href)
	if m.inHead {
		if rank := rankHeadFeed(typ, comment); rank < m.feedRank {
			m.feed, m.feedRank = href, rank
		}
		return false
	}
	if comment {
		if m.commentFeed == "" {
			m.commentFeed = href
		}
		return false
	}
	if m.feed == "" {
		m.feed = href
	}
	return m.done()
}

// done reports whether what the body is still being read for has been found:
// the head is over, and it left neither the feed nor the title unanswered.
func (m *metaReader) done() bool {
	return !m.inHead && m.feed != "" && m.title != ""
}

// text is the text of the element just started, up to its end tag. The
// tokenizer hands <title> and the like over as one text token, but a page
// may put markup in a title, and then it is several.
func (m *metaReader) text(a atom.Atom) string {
	var b strings.Builder
	for {
		switch m.z.Next() {
		case html.ErrorToken:
			return b.String()
		case html.TextToken:
			b.Write(m.z.Text())
		case html.EndTagToken:
			if name, _ := m.z.TagName(); atom.Lookup(name) == a {
				return b.String()
			}
		}
	}
}

// rankWorst is worse than any rank rankHeadFeed gives.
var rankWorst = 2 * len(feedTypes)

// rankHeadFeed ranks a feed link of the head: a comment feed after anything
// else, and among those the order of feedTypes. The lower, the better.
func rankHeadFeed(typ string, comment bool) int {
	rank := 0
	if comment {
		rank += len(feedTypes)
	}
	for i, t := range feedTypes {
		if t == typ {
			return rank + i
		}
	}
	return rankWorst // unreachable: typ comes from feedType
}

// feedType is the feed type of a link's type attribute, or "".
func feedType(s string) string {
	s, _, _ = strings.Cut(s, ";")
	s = strings.ToLower(strings.TrimSpace(s))
	for _, t := range feedTypes {
		if s == t {
			return t
		}
	}
	return ""
}

// isCommentFeed reports whether a feed link is a site's comment feed, by its
// title ("Blog » Comments Feed") or its address.
func isCommentFeed(title, href string) bool {
	t := strings.ToLower(title)
	for _, w := range commentFeedWords {
		if strings.Contains(t, w) {
			return true
		}
	}
	h := strings.ToLower(href)
	return strings.Contains(h, "/comments/") || strings.HasSuffix(h, "/comments")
}

// hasWord reports whether the space-separated list s holds word, in any case.
func hasWord(s, word string) bool {
	for _, w := range strings.Fields(s) {
		if strings.EqualFold(w, word) {
			return true
		}
	}
	return false
}

func collapseSpace(s string) string {
	return strings.Join(strings.Fields(s), " ")
}