1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

package main

import (
	"bytes"
	"database/sql"
	"io"
	"net/url"
	"rss-kakapo/logic"
	"strings"
)

// feedAccept asks for any of the feed formats kakapo reads, and takes
// anything.
const feedAccept = "application/atom+xml,application/rss+xml,application/feed+json,application/xml;q=0.9,*/*;q=0.8"

// feeds gets the feed that rss-kakapo would follow for every fetched page
// that names one and whose feed was not fetched yet, into feed_fetch: the
// feed of logic.ReadSiteMeta, resolved against the page as served, as
// logic.feedFollower.getFeedUrl does. It asks as politely as fetch, and
// stores a failure the same way.
func feeds(db *sql.DB, hosts int, log io.Writer) error {
	rows, err := db.Query(`SELECT handle, COALESCE(final_url, url), body FROM fetch
		WHERE status=200 AND body IS NOT NULL
		AND handle NOT IN (SELECT handle FROM feed_fetch)`)
	if err != nil {
		return err
	}
	var jobs []job
	for rows.Next() {
		var handle, pageUrl string
		var body []byte
		if err = rows.Scan(&handle, &pageUrl, &body); err != nil {
			rows.Close()
			return err
		}
		if u := kakapoFeedUrl(pageUrl, body); u != "" {
			jobs = append(jobs, job{handle: handle, url: u})
		}
	}
	rows.Close()
	if err = rows.Err(); err != nil {
		return err
	}
	return fetchAll(jobs, hosts, "feeds", feedAccept, log, func(j job, r fetchRow) error {
		return storeFetch(db, "feed_fetch", j, r)
	})
}

// kakapoFeedUrl is the URL of the feed rss-kakapo follows for a page served
// from pageUrl, or "" if it finds none.
func kakapoFeedUrl(pageUrl string, body []byte) string {
	href := logic.ReadSiteMeta(bytes.NewReader(body)).FeedHref
	if href == "" {
		return ""
	}
	base, err := url.Parse(pageUrl)
	if err != nil {
		return ""
	}
	u, err := url.Parse(strings.TrimSpace(href))
	if err != nil {
		return ""
	}
	u = base.ResolveReference(u)
	if u.Scheme != "http" && u.Scheme != "https" {
		return ""
	}
	trimQuery(u)
	return strings.TrimRight(u.String(), "/")
}