1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
|
package main
import (
"bytes"
"database/sql"
"fmt"
"io"
"net/url"
"regexp"
"rss-kakapo/logic"
"strings"
"golang.org/x/net/html"
"golang.org/x/net/html/atom"
)
// analyze reads every fetched page and records, for SQL afterwards:
//
// - page: what the page is (HTML or not, whether its head ended, whether
// the URL came from a feed), and which feed each rule picks;
// - page_link: every <link> with an href, in the head or not, in order;
// - page_anchor: every <a> whose href or text looks like a feed;
// - meta: the title and description of the page and of its feed, if
// feeds fetched it (see metaSchema).
//
// It replaces what it recorded before, so it can be run again after a
// change to the rules.
func analyze(db *sql.DB, log io.Writer) error {
if _, err := db.Exec(analyzeSchema + metaSchema); err != nil {
return err
}
rows, err := db.Query(`SELECT f.handle, COALESCE(f.final_url, f.url), l.site_url, l.feed_url,
COALESCE(f.content_type, ''), f.body
FROM fetch f JOIN listing l ON l.handle=f.handle
WHERE f.status=200 AND f.body IS NOT NULL`)
if err != nil {
return err
}
type page struct {
handle, finalUrl, siteUrl, feedUrl, contentType string
body []byte
}
var pages []page
for rows.Next() {
var p page
if err = rows.Scan(&p.handle, &p.finalUrl, &p.siteUrl, &p.feedUrl, &p.contentType, &p.body); err != nil {
rows.Close()
return err
}
pages = append(pages, p)
}
rows.Close()
if err = rows.Err(); err != nil {
return err
}
tx, err := db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
// Children first: page_link, page_anchor and meta reference page.
for _, t := range []string{"page_link", "page_anchor", "meta", "page"} {
if _, err = tx.Exec("DELETE FROM " + t); err != nil {
return err
}
}
for _, p := range pages {
base, err := url.Parse(p.finalUrl)
if err != nil {
continue
}
s := scanPage(p.body)
isHtml := strings.Contains(strings.ToLower(p.contentType), "html") || s.sawHtml
resolve := func(href string) string {
if href == "" {
return ""
}
return feedKey(base, href)
}
// The rules, each on the links of the head only, as kakapo and
// upstream read them.
var head []scannedLink
for _, l := range s.links {
if l.inHead {
head = append(head, l)
}
}
kakapo := resolve(logic.ReadSiteMeta(bytes.NewReader(p.body)).FeedHref)
upstream := resolve(upstreamRule(head))
first := resolve(firstFeedRule(head))
sameHost := resolve(sameHostRule(head, base))
// The URL came from a feed for sure if it ends in "/": upstream
// trimmed that off a URL it fetched as a page.
fromFeed := strings.HasSuffix(p.siteUrl, "/")
if _, err = tx.Exec(`INSERT INTO page (handle, final_url, is_html, head_ended, body_reached,
from_feed, listed_feed, kakapo, upstream, first, same_host)
VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
p.handle, p.finalUrl, isHtml, s.headEnded, s.bodyReached, fromFeed,
feedKey(nil, p.feedUrl), kakapo, upstream, first, sameHost); err != nil {
return err
}
for i, l := range s.links {
if _, err = tx.Exec(`INSERT INTO page_link (handle, pos, in_head, rel, type, href, resolved, title)
VALUES (?,?,?,?,?,?,?,?)`,
p.handle, i, l.inHead, l.rel, l.typ, l.href, resolve(l.href), l.title); err != nil {
return err
}
}
for i, a := range s.anchors {
if _, err = tx.Exec(`INSERT INTO page_anchor (handle, pos, href, resolved, text)
VALUES (?,?,?,?,?)`, p.handle, i, a.href, resolve(a.href), a.text); err != nil {
return err
}
}
}
if err = analyzeMeta(tx); err != nil {
return err
}
if err = tx.Commit(); err != nil {
return err
}
fmt.Fprintf(log, "analysed %d pages\n", len(pages))
return nil
}
const analyzeSchema = `
CREATE TABLE IF NOT EXISTS page (
handle TEXT PRIMARY KEY REFERENCES listing(handle),
final_url TEXT NOT NULL,
is_html INTEGER NOT NULL,
head_ended INTEGER NOT NULL, -- a </head> or <body> was seen
body_reached INTEGER NOT NULL, -- a <body> was seen
from_feed INTEGER NOT NULL, -- site_url ends in "/": upstream took it from a feed
listed_feed TEXT NOT NULL, -- the feed RSS Parrot follows, as feedKey
kakapo TEXT NOT NULL, -- each rule's pick, as feedKey, or ""
upstream TEXT NOT NULL,
first TEXT NOT NULL,
same_host TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS page_link (
handle TEXT NOT NULL REFERENCES page(handle),
pos INTEGER NOT NULL,
in_head INTEGER NOT NULL,
rel TEXT NOT NULL,
type TEXT NOT NULL,
href TEXT NOT NULL,
resolved TEXT NOT NULL,
title TEXT NOT NULL,
PRIMARY KEY (handle, pos)
);
CREATE TABLE IF NOT EXISTS page_anchor (
handle TEXT NOT NULL REFERENCES page(handle),
pos INTEGER NOT NULL,
href TEXT NOT NULL,
resolved TEXT NOT NULL,
text TEXT NOT NULL,
PRIMARY KEY (handle, pos)
);
`
type scannedLink struct {
inHead bool
rel, typ, href, title string
}
type scannedAnchor struct{ href, text string }
type scan struct {
links []scannedLink
anchors []scannedAnchor
headEnded, bodyReached, sawHtml bool
}
// feedish is what an <a> to a feed looks like, by its href or its text.
var feedishHref = regexp.MustCompile(`(?i)(rss|atom|feed|\.xml)(\b|$|[/?._-])`)
var feedishText = regexp.MustCompile(`(?i)\b(rss|atom|feeds?)\b`)
// scanPage reads all of a page, unlike logic.ReadSiteMeta, to see what lies
// beyond the head too.
func scanPage(body []byte) scan {
var s scan
z := html.NewTokenizer(bytes.NewReader(body))
inHead := true
var a *scannedAnchor
var text strings.Builder
for {
tt := z.Next()
switch tt {
case html.ErrorToken:
return s
case html.TextToken:
if a != nil {
text.Write(z.Text())
}
case html.EndTagToken:
name, _ := z.TagName()
switch atom.Lookup(name) {
case atom.Head:
inHead, s.headEnded = false, true
case atom.A:
if a != nil {
a.text = strings.Join(strings.Fields(text.String()), " ")
if feedishHref.MatchString(a.href) || feedishText.MatchString(a.text) {
s.anchors = append(s.anchors, *a)
}
a = nil
}
}
case html.StartTagToken, html.SelfClosingTagToken:
name, hasAttr := z.TagName()
at := atom.Lookup(name)
attrs := map[string]string{}
for hasAttr {
var k, v []byte
k, v, hasAttr = z.TagAttr()
if _, seen := attrs[string(k)]; !seen {
attrs[string(k)] = string(v)
}
}
switch at {
case atom.Html:
s.sawHtml = true
case atom.Body:
inHead, s.headEnded, s.bodyReached = false, true, true
case atom.Link:
if href := strings.TrimSpace(attrs["href"]); href != "" {
s.links = append(s.links, scannedLink{inHead, attrs["rel"], attrs["type"], href, attrs["title"]})
}
case atom.A:
if tt == html.StartTagToken {
a = &scannedAnchor{href: strings.TrimSpace(attrs["href"])}
text.Reset()
}
}
}
}
}
// upstreamRule is RSS Parrot's getFeedUrl: rel exactly "alternate", type
// exactly RSS or Atom; the first RSS feed, else the first Atom feed.
func upstreamRule(links []scannedLink) string {
res, isRss := "", false
for _, l := range links {
if l.rel != "alternate" {
continue
}
if l.typ == "application/atom+xml" && !isRss && res == "" {
res = l.href
}
if l.typ == "application/rss+xml" && (res == "" || !isRss) {
res, isRss = l.href, true
}
}
return res
}
// isFeedLink is kakapo's test of a link: alternate among the words of rel,
// a feed type without parameters, both in any case.
func isFeedLink(l scannedLink) bool {
if !hasWord(l.rel, "alternate") {
return false
}
t, _, _ := strings.Cut(l.typ, ";")
switch strings.ToLower(strings.TrimSpace(t)) {
case "application/atom+xml", "application/feed+json", "application/rss+xml":
return true
}
return false
}
// firstFeedRule takes the first feed link of any type.
func firstFeedRule(links []scannedLink) string {
for _, l := range links {
if isFeedLink(l) {
return l.href
}
}
return ""
}
// sameHostRule takes the first feed link on the page's own host, ignoring
// "www.", else the first feed link.
func sameHostRule(links []scannedLink, base *url.URL) string {
host := strings.TrimPrefix(strings.ToLower(base.Hostname()), "www.")
for _, l := range links {
if !isFeedLink(l) {
continue
}
u, err := base.Parse(l.href)
if err == nil && strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.") == host {
return l.href
}
}
return firstFeedRule(links)
}
// feedKey is a feed URL resolved against base, as getFeedUrl makes it
// (absolute, query mostly trimmed, no trailing slash), then without scheme
// and "www." and in lower case, so that two spellings of one feed compare
// equal.
func feedKey(base *url.URL, href string) string {
u, err := url.Parse(strings.TrimSpace(href))
if err != nil {
return ""
}
if base != nil && !u.IsAbs() {
u = base.ResolveReference(u)
}
trimQuery(u)
u.Fragment = ""
return normaliseUrl(u.String())
}
// trimQuery is logic.feedFollower.trimQueryParams, which needs a feed
// follower; it is the same in upstream.
func trimQuery(u *url.URL) {
if strings.Contains(u.Host, "youtube.com") &&
(strings.Contains(u.RawQuery, "channel_id") || strings.Contains(u.RawQuery, "playlist_id")) {
return
}
if strings.Contains(u.Host, "archive.org") || strings.Contains(u.Host, "languagelog.ldc.upenn.edu") {
return
}
if strings.Contains(u.RawQuery, "format=rss") {
u.RawQuery = "format=rss"
return
}
u.RawQuery = ""
}
|