1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
// feed-survey collects the web pages of sites that RSS Parrot follows, to
// study how sites name their feeds: which <link>s their heads hold, in what
// order, and whether a head-only reader (logic.ReadSiteMeta) finds the feed
// RSS Parrot follows.
//
// It is a research tool, not part of rss-kakapo, and runs in five steps,
// each resumable, against one SQLite database:
//
// feed-survey list DB read rss-parrot.net's feed listing
// feed-survey sample DB [-n N] pick N sites whose URLs differ the most
// feed-survey fetch DB fetch the page of every sampled site
// feed-survey feeds DB fetch the feed rss-kakapo picks of every page
// feed-survey analyze DB record the links of every page, the feed
// each rule picks, and what the page and its
// feed say they are called, for SQL
//
// Run it from the repository root, where tmp/ is ignored by git:
//
// go run ./cmd/feed-survey list tmp/feed-survey.db
//
// See feed-survey.1 next to this file.
package main
import (
"database/sql"
"flag"
"fmt"
"io"
"os"
_ "modernc.org/sqlite"
)
const usage = `usage: feed-survey list DB
feed-survey sample DB [-n N]
feed-survey fetch DB [-hosts N]
feed-survey feeds DB [-hosts N]
feed-survey analyze DB
`
// userAgent says who is asking, so that a site owner who wonders can find
// out.
const userAgent = "rss-kakapo-feed-survey/2 (study of how sites name their feeds; +https://sources.profpatsch.de/rss-kakapo/)"
func main() {
os.Exit(run(os.Args[1:], os.Stderr))
}
func run(args []string, stderr io.Writer) int {
if len(args) < 2 {
fmt.Fprint(stderr, usage)
return 2
}
cmd, dbPath := args[0], args[1]
fs := flag.NewFlagSet(cmd, flag.ContinueOnError)
fs.SetOutput(stderr)
n := fs.Int("n", 1000, "sample size")
hosts := fs.Int("hosts", 8, "hosts fetched from in parallel")
if err := fs.Parse(args[2:]); err != nil || fs.NArg() != 0 {
fmt.Fprint(stderr, usage)
return 2
}
db, err := openDB(dbPath)
if err != nil {
fmt.Fprintf(stderr, "feed-survey: %v\n", err)
return 1
}
defer db.Close()
switch cmd {
case "list":
err = list(db, stderr)
case "sample":
err = sample(db, *n, stderr)
case "fetch":
err = fetch(db, *hosts, stderr)
case "feeds":
err = feeds(db, *hosts, stderr)
case "analyze":
err = analyze(db, stderr)
default:
fmt.Fprint(stderr, usage)
return 2
}
if err != nil {
fmt.Fprintf(stderr, "feed-survey: %v\n", err)
return 1
}
return 0
}
const schema = `
CREATE TABLE IF NOT EXISTS listing (
handle TEXT PRIMARY KEY,
feed_url TEXT NOT NULL,
site_url TEXT NOT NULL,
title TEXT NOT NULL,
since TEXT NOT NULL,
last_posted TEXT NOT NULL,
page INTEGER NOT NULL
);
-- Pages of the listing that were read completely.
CREATE TABLE IF NOT EXISTS listing_page (
page INTEGER PRIMARY KEY,
entries INTEGER NOT NULL
);
-- rank is the order sites were picked in; min_dist the Hamming distance of a
-- site's fingerprint to the nearest one picked before it (64 for the first).
CREATE TABLE IF NOT EXISTS sample (
handle TEXT PRIMARY KEY REFERENCES listing(handle),
rank INTEGER NOT NULL UNIQUE,
simhash INTEGER NOT NULL,
min_dist INTEGER NOT NULL
);
-- One row per sampled site, once it was fetched, successfully or not.
-- body is at most fetchLimit bytes; truncated says whether there was more.
CREATE TABLE IF NOT EXISTS fetch (
handle TEXT PRIMARY KEY REFERENCES listing(handle),
url TEXT NOT NULL,
fetched_at TEXT NOT NULL,
status INTEGER,
final_url TEXT,
content_type TEXT,
headers TEXT,
body BLOB,
truncated INTEGER,
error TEXT,
ms INTEGER NOT NULL
);
-- The feed rss-kakapo follows for a fetched page, once it was fetched, as
-- fetch has it; url is the feed's.
CREATE TABLE IF NOT EXISTS feed_fetch (
handle TEXT PRIMARY KEY REFERENCES listing(handle),
url TEXT NOT NULL,
fetched_at TEXT NOT NULL,
status INTEGER,
final_url TEXT,
content_type TEXT,
headers TEXT,
body BLOB,
truncated INTEGER,
error TEXT,
ms INTEGER NOT NULL
);
`
func openDB(path string) (*sql.DB, error) {
db, err := sql.Open("sqlite", "file:"+path+"?mode=rwc"+
"&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)")
if err != nil {
return nil, err
}
if _, err = db.Exec(schema); err != nil {
db.Close()
return nil, err
}
return db, nil
}
|