1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
|
package logic
import (
"io"
"strings"
"testing"
)
const (
atomLink = `<link rel="alternate" type="application/atom+xml" href="/atom.xml">`
jsonLink = `<link rel="alternate" type="application/feed+json" href="/feed.json">`
rssLink = `<link rel="alternate" type="application/rss+xml" href="/rss.xml">`
commentLink = `<link rel="alternate" type="application/rss+xml" title="Blog » Comments Feed" href="/comments/feed">`
)
// head is a page whose head holds s, with a title, and a body that holds
// nothing of interest.
func head(s string) string {
return "<!DOCTYPE html><html><head>" + s + "<title>T</title></head><body>x</body></html>"
}
// TestReadSiteMetaHead is what a page's head says, which is read whole: of
// several feeds it names, the best is taken, not the first.
func TestReadSiteMetaHead(t *testing.T) {
for _, c := range []struct {
name, page string
want siteMeta
}{
{"atom over rss", head(rssLink + atomLink), siteMeta{FeedHref: "/atom.xml", Title: "T"}},
{"rss after atom", head(atomLink + rssLink), siteMeta{FeedHref: "/atom.xml", Title: "T"}},
{"atom over json", head(jsonLink + atomLink), siteMeta{FeedHref: "/atom.xml", Title: "T"}},
{"json over rss", head(rssLink + jsonLink), siteMeta{FeedHref: "/feed.json", Title: "T"}},
{"rss alone", head(rssLink), siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"the first of a type",
head(`<link rel="alternate" type="application/rss+xml" href="/main.xml">` +
`<link rel="alternate" type="application/rss+xml" href="/other.xml">`),
siteMeta{FeedHref: "/main.xml", Title: "T"}},
{"rel with more words",
head(`<link rel="home Alternate" type="application/rss+xml" href="/rss.xml">`),
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"type with parameters",
head(`<link rel="alternate" type="Application/RSS+XML; charset=utf-8" href="/rss.xml">`),
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"upper case",
`<HTML><HEAD><LINK REL="ALTERNATE" TYPE="application/atom+xml" HREF="/a"><TITLE>T</TITLE></HEAD>`,
siteMeta{FeedHref: "/a", Title: "T"}},
{"self-closing", head(`<link rel="alternate" type="application/rss+xml" href="/rss.xml" />`),
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"not alternate", head(`<link rel="stylesheet" type="application/rss+xml" href="/x">`),
siteMeta{Title: "T"}},
{"another type", head(`<link rel="alternate" type="text/html" hreflang="de" href="/de">`),
siteMeta{Title: "T"}},
{"no href", head(`<link rel="alternate" type="application/rss+xml">` + atomLink),
siteMeta{FeedHref: "/atom.xml", Title: "T"}},
{"a head that does not end", "<title>T</title>" + rssLink + "<p>text</p>",
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"the title late in the head",
"<html><head>" + rssLink + `<meta name="x" content="y"><title>T</title></head><body>x`,
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"the title",
head("<title>\n Tom & Jerry\n <b>Blog</b>\n</title><title>second</title>"),
siteMeta{Title: "Tom & Jerry <b>Blog</b>"}},
{"the description",
head(`<meta name="Description" content=" A blog
about things "><meta name="description" content="second">`),
siteMeta{Title: "T", Description: "A blog about things"}},
{"og:description is not it", head(`<meta property="og:description" content="x">`),
siteMeta{Title: "T"}},
{"empty", "", siteMeta{}},
} {
t.Run(c.name, func(t *testing.T) {
if got := readSiteMeta(strings.NewReader(c.page)); got != c.want {
t.Errorf("readSiteMeta = %+v, want %+v", got, c.want)
}
})
}
}
// TestReadSiteMetaCommentFeeds is the comment feed a WordPress page names
// along with its posts: never followed instead of the posts, but better
// than nothing.
func TestReadSiteMetaCommentFeeds(t *testing.T) {
for _, c := range []struct {
name, page string
want siteMeta
}{
{"in the head, before the feed", head(commentLink + rssLink),
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"by its address alone",
head(`<link rel="alternate" type="application/rss+xml" href="/comments/feed">` + rssLink),
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"in another language",
head(`<link rel="alternate" type="application/rss+xml" title="Blog » Kommentar-Feed" href="/kf">` + rssLink),
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"the only one in the head", head(commentLink),
siteMeta{FeedHref: "/comments/feed", Title: "T"}},
{"the best of the ones in the head",
head(commentLink + `<link rel="alternate" type="application/atom+xml" title="Comments" href="/comments/atom">`),
siteMeta{FeedHref: "/comments/atom", Title: "T"}},
{"in the body, before the feed",
"<html><head><title>T</title></head><body>" + commentLink + rssLink + "</body>",
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"the only one in the body",
"<html><head><title>T</title></head><body>" + commentLink + "<p>text</p></body>",
siteMeta{FeedHref: "/comments/feed", Title: "T"}},
} {
t.Run(c.name, func(t *testing.T) {
if got := readSiteMeta(strings.NewReader(c.page)); got != c.want {
t.Errorf("readSiteMeta = %+v, want %+v", got, c.want)
}
})
}
}
// TestReadSiteMetaBody is the body, read only for what the head left
// unanswered.
func TestReadSiteMetaBody(t *testing.T) {
for _, c := range []struct {
name, page string
want siteMeta
}{
{"the head's feed is not overruled by the body's",
"<html><head>" + rssLink + "<title>T</title></head><body>" + atomLink + "</body>",
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"the first feed of the body, whatever its type",
"<html><head><title>T</title></head><body>" + rssLink + atomLink + "</body>",
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"the body's title if the head named none",
"<html><head>" + rssLink + "</head><body><title>Name</title>x</body>",
siteMeta{FeedHref: "/rss.xml", Title: "Name"}},
{"the head's title is not overruled",
"<html><head><title>Head</title></head><body><title>Body</title>" + rssLink,
siteMeta{FeedHref: "/rss.xml", Title: "Head"}},
{"a title in a drawing is not the page's",
"<html><head>" + rssLink + `</head><body><svg><title>rss</title></svg><title>Name</title>`,
siteMeta{FeedHref: "/rss.xml", Title: "Name"}},
{"the body's description, if it comes before the stop",
"<html><head>" + rssLink + `</head><body><meta name="description" content="About"><title>Name</title>`,
siteMeta{FeedHref: "/rss.xml", Title: "Name", Description: "About"}},
{"a description after the stop is not waited for",
"<html><head>" + rssLink + `</head><body><title>Name</title><meta name="description" content="About">`,
siteMeta{FeedHref: "/rss.xml", Title: "Name"}},
{"a title after the body's feed",
"<html><head></head><body>" + rssLink + "<title>Name</title><p>text</p>",
siteMeta{FeedHref: "/rss.xml", Title: "Name"}},
{"neither a feed nor a title", "<html><head></head><body><p>text</p></body>", siteMeta{}},
} {
t.Run(c.name, func(t *testing.T) {
if got := readSiteMeta(strings.NewReader(c.page)); got != c.want {
t.Errorf("readSiteMeta = %+v, want %+v", got, c.want)
}
})
}
}
// The pages of three sites that name their feed in the body, abridged from
// what they served on 2026-10-29.
func TestReadSiteMetaBodyLinksInTheWild(t *testing.T) {
for _, c := range []struct {
name, page string
want siteMeta
}{
// YouTube closes the head after some 770 KB of script and style,
// then writes the head tags of the channel into the body.
{"youtube", `<html><head><meta http-equiv="X-UA-Compatible" content="IE=edge"/>` +
`<style>#foo{width:86px}</style>` +
`<link rel="search" type="application/opensearchdescription+xml" href="/opensearch" title="YouTube">` +
`<link rel="manifest" href="/manifest.webmanifest"></head><body dir="ltr">` +
`<script>if (window.ytcsi) {window.ytcsi.tick('bs', null, '');}</script>` +
`<link rel="canonical" href="https://www.youtube.com/channel/UCKlZT0bHmZnIakq4KlelUKQ">` +
`<link rel="alternate" media="handheld" href="https://m.youtube.com/channel/UCKlZT0bHmZnIakq4KlelUKQ/feed">` +
`<title>Wonderkey AMV's - YouTube</title>` +
`<meta name="description" content="AVISO : SOLO HAGO AMV YURI">` +
`<link rel="alternate" type="application/rss+xml" title="RSS" href="https://www.youtube.com/feeds/videos.xml?channel_id=UCKlZT0bHmZnIakq4KlelUKQ">` +
`<meta property="og:title" content="Wonderkey AMV's"></body></html>`,
siteMeta{
FeedHref: "https://www.youtube.com/feeds/videos.xml?channel_id=UCKlZT0bHmZnIakq4KlelUKQ",
Title: "Wonderkey AMV's - YouTube",
Description: "AVISO : SOLO HAGO AMV YURI",
}},
// Lemmy writes the link next to the RSS icon of the post list,
// whose <svg> holds a <title> of its own.
{"lemmy", `<html><head><title>test - Lemmy</title>` +
`<link rel="stylesheet" type="text/css" href="/css/code-themes/atom-one-light.css"></head><body>` +
`<a href="https://lemmy.ml/feeds/c/test.xml?sort=Active" title="RSS" rel="noopener nofollow">` +
`<svg class="icon"><use xlink:href="/static/assets/symbols.svg#icon-rss"></use>` +
`<div class="visually-hidden"><title>rss</title></div></svg></a>` +
`<link rel="alternate" type="application/atom+xml" href="https://lemmy.ml/feeds/c/test.xml?sort=Active">` +
`</body></html>`,
siteMeta{FeedHref: "https://lemmy.ml/feeds/c/test.xml?sort=Active", Title: "test - Lemmy"}},
// A theme that closes the head before WordPress has written its
// feed links leaves them in the body, the comment feeds among them.
{"a theme that closes the head too early",
`<html><head><script>var x = 1;</script><title>Immo974</title></head><body class="home page-template">` +
`<link rel="canonical" href="https://www.immo974.com/" />` +
`<link rel='dns-prefetch' href='//s.w.org' />` +
`<link rel="alternate" type="application/rss+xml" title="Immo974 » Flux" href="https://www.immo974.com/feed" />` +
`<link rel="alternate" type="application/rss+xml" title="Immo974 » Flux des commentaires" href="https://www.immo974.com/comments/feed" />` +
`<link rel="alternate" type="application/rss+xml" title="Immo974 » Homepage Flux des commentaires" href="https://www.immo974.com/homepage/feed" />` +
`</body></html>`,
siteMeta{FeedHref: "https://www.immo974.com/feed", Title: "Immo974"}},
} {
t.Run(c.name, func(t *testing.T) {
if got := readSiteMeta(strings.NewReader(c.page)); got != c.want {
t.Errorf("readSiteMeta = %+v, want %+v", got, c.want)
}
})
}
}
// countingReader counts what was read of a page, to tell where reading
// stopped.
type countingReader struct {
r io.Reader
n int
}
func (c *countingReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
c.n += n
return n, err
}
// A page is read no further than it has to be: a body is not read at all
// when the head answered everything, and only up to what it was read for
// when it did not. The tokenizer reads ahead in blocks, so the test asks for
// much less than the filler, not for an exact count.
func TestReadSiteMetaStopsReading(t *testing.T) {
filler := "<p>" + strings.Repeat("x", 4*siteMetaLimit) + "</p>"
for _, c := range []struct {
name, page string
want siteMeta
}{
{"the head answered everything",
"<html><head>" + rssLink + "<title>T</title></head><body>" + filler,
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"stops at the body's feed",
"<html><head><title>T</title></head><body>" + rssLink + filler,
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
{"stops at the body's title",
"<html><head>" + rssLink + "</head><body><title>T</title>" + filler,
siteMeta{FeedHref: "/rss.xml", Title: "T"}},
} {
t.Run(c.name, func(t *testing.T) {
cr := &countingReader{r: strings.NewReader(c.page)}
if got := readSiteMeta(cr); got != c.want {
t.Fatalf("readSiteMeta = %+v, want %+v", got, c.want)
}
if cr.n > siteMetaLimit/2 {
t.Errorf("read %d bytes of a %d-byte page; should have stopped early", cr.n, len(c.page))
}
})
}
}
// Nothing is read beyond siteMetaLimit, however little of the page was
// answered.
func TestReadSiteMetaLimit(t *testing.T) {
filler := "<!-- " + strings.Repeat("x", siteMetaLimit) + " -->"
for _, c := range []struct {
name, page string
want siteMeta
}{
{"a head longer than the limit",
"<html><head><title>T</title>" + filler + rssLink,
siteMeta{Title: "T"}},
{"a feed in the body beyond the limit",
"<html><head><title>T</title></head><body>" + filler + rssLink,
siteMeta{Title: "T"}},
} {
t.Run(c.name, func(t *testing.T) {
cr := &countingReader{r: strings.NewReader(c.page)}
if got := readSiteMeta(cr); got != c.want {
t.Errorf("readSiteMeta = %+v, want %+v", got, c.want)
}
if cr.n > siteMetaLimit {
t.Errorf("read %d bytes, more than the %d-byte limit", cr.n, siteMetaLimit)
}
})
}
}
|