1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
|
// link-check is a recursive link checker for live websites.
//
// Give it one or more seed URLs. It fetches each page, extracts links, and
// recurses into links whose host is in scope (by default the hosts of the
// seed URLs, extendable with -allow). Links that are out of scope (external
// outbound links) are checked for liveness once but not recursed into.
//
// A visited set keyed by normalized URL prevents cycles and refetches.
//
// Exit status is non-zero if any link is dead (and, with -strict, if any link
// is only reachable with a 403/429 "blocked but alive" response).
package main
import (
"flag"
"fmt"
"net/http"
"net/url"
"os"
"sort"
"strings"
"sync"
"time"
"golang.org/x/net/html"
)
const userAgent = "link-check/1 (+https://www.profpatsch.de)"
// config holds the parsed command-line configuration.
type config struct {
allow []string // extra host suffixes to recurse into
strict bool // treat WARN (403/429) as failure
timeout time.Duration // per-request timeout
concurrency int // number of worker goroutines
maxPages int // safety cap on pages crawled
checkExt bool // whether to liveness-check external links
}
// status classifies the result of checking a single URL.
type status int
const (
statusOK status = iota
statusWarn
statusFail
)
func (s status) String() string {
switch s {
case statusOK:
return "OK "
case statusWarn:
return "WARN"
default:
return "FAIL"
}
}
// result records the outcome of checking a URL, including where it was found.
type result struct {
url string
status status
code int // HTTP status code, 0 on transport error
detail string // error text or final-redirect note
referrer string // page on which the link was discovered ("" for seeds)
}
// crawler holds shared crawl state.
type crawler struct {
cfg config
client *http.Client
scope []string // host suffixes considered "in scope" for recursion
mu sync.Mutex
visited map[string]bool
results []result
pages int // number of pages fetched (for maxPages cap)
wg sync.WaitGroup
queue chan task
sem chan struct{} // bounds concurrent in-flight work
}
// task is a unit of work: a URL to process, and whether to recurse into it.
type task struct {
url string
referrer string
recurse bool
}
func main() {
cfg, seeds := parseArgs()
if len(seeds) == 0 {
fmt.Fprintln(os.Stderr, "usage: link-check [flags] <seed-url> [seed-url...]")
flag.PrintDefaults()
os.Exit(2)
}
c := newCrawler(cfg, seeds)
c.run(seeds)
os.Exit(c.report())
}
func parseArgs() (config, []string) {
var allow multiFlag
cfg := config{}
flag.Var(&allow, "allow", "host suffix to recurse into (repeatable), e.g. softwaregardening.org")
flag.BoolVar(&cfg.strict, "strict", false, "treat 403/429 (blocked but alive) as failures")
flag.DurationVar(&cfg.timeout, "timeout", 15*time.Second, "per-request timeout")
flag.IntVar(&cfg.concurrency, "concurrency", 8, "number of concurrent workers")
flag.IntVar(&cfg.maxPages, "max-pages", 500, "maximum number of pages to crawl")
extMode := flag.String("external", "check", "external link handling: check|skip")
flag.Parse()
cfg.allow = allow
cfg.checkExt = *extMode != "skip"
return cfg, flag.Args()
}
// multiFlag collects repeated string flags.
type multiFlag []string
func (m *multiFlag) String() string { return strings.Join(*m, ",") }
func (m *multiFlag) Set(v string) error { *m = append(*m, v); return nil }
func newCrawler(cfg config, seeds []string) *crawler {
scope := append([]string{}, cfg.allow...)
for _, s := range seeds {
if u, err := url.Parse(s); err == nil && u.Host != "" {
scope = append(scope, u.Hostname())
}
}
return &crawler{
cfg: cfg,
client: &http.Client{Timeout: cfg.timeout},
scope: scope,
visited: map[string]bool{},
queue: make(chan task, 1024),
sem: make(chan struct{}, cfg.concurrency),
}
}
// inScope reports whether host should be recursed into.
func (c *crawler) inScope(host string) bool {
host = strings.ToLower(host)
for _, s := range c.scope {
s = strings.ToLower(strings.TrimPrefix(s, "*."))
if host == s || strings.HasSuffix(host, "."+s) {
return true
}
}
return false
}
// normalize returns a canonical key for the visited set. It lowercases the
// host, drops the fragment, and removes the default port. The scheme is kept
// so http:// and https:// of the same path are still distinct fetches.
func normalize(u *url.URL) string {
n := *u
n.Fragment = ""
n.Host = strings.ToLower(n.Host)
if (n.Scheme == "http" && strings.HasSuffix(n.Host, ":80")) ||
(n.Scheme == "https" && strings.HasSuffix(n.Host, ":443")) {
n.Host = n.Host[:strings.LastIndex(n.Host, ":")]
}
if n.Path == "" {
n.Path = "/"
}
return n.String()
}
// run enqueues the seeds and processes the queue until it drains.
func (c *crawler) run(seeds []string) {
for _, s := range seeds {
c.enqueue(task{url: s, referrer: "", recurse: true})
}
// Close the queue once all outstanding work is done.
go func() {
c.wg.Wait()
close(c.queue)
}()
for t := range c.queue {
c.process(t)
}
}
// enqueue schedules a task if its URL has not been seen yet.
func (c *crawler) enqueue(t task) {
u, err := url.Parse(t.url)
if err != nil {
return
}
key := normalize(u)
c.mu.Lock()
if c.visited[key] {
c.mu.Unlock()
return
}
c.visited[key] = true
c.mu.Unlock()
c.wg.Add(1)
go func() { c.queue <- t }()
}
// process checks one URL and, if in scope and recursable, extracts and
// enqueues its child links.
func (c *crawler) process(t task) {
defer c.wg.Done()
c.sem <- struct{}{}
defer func() { <-c.sem }()
u, err := url.Parse(t.url)
if err != nil {
c.record(result{url: t.url, status: statusFail, detail: "invalid url: " + err.Error(), referrer: t.referrer})
return
}
recurse := t.recurse && c.inScope(u.Hostname())
// Decide whether we even fetch external links.
if !recurse && !c.cfg.checkExt {
return
}
c.mu.Lock()
over := c.pages >= c.cfg.maxPages
if recurse {
c.pages++
}
c.mu.Unlock()
if recurse && over {
recurse = false // hit the page cap; still liveness-check, don't recurse
}
res, body := c.fetch(u, recurse, t.referrer)
c.record(res)
if recurse && body != nil {
for _, link := range extractLinks(u, body) {
c.enqueue(task{url: link, referrer: t.url, recurse: true})
}
}
}
// fetch performs the HTTP request. When wantBody is true it issues a GET and
// returns the parsed document root; otherwise it issues a HEAD (falling back to
// GET if HEAD is not allowed) and returns a nil node.
func (c *crawler) fetch(u *url.URL, wantBody bool, referrer string) (result, *html.Node) {
method := http.MethodGet
if !wantBody {
method = http.MethodHead
}
res, node := c.do(method, u, wantBody, referrer)
// Some servers reject HEAD; retry with GET to confirm liveness.
if !wantBody && (res.code == http.StatusMethodNotAllowed || res.code == http.StatusNotImplemented) {
res, node = c.do(http.MethodGet, u, false, referrer)
}
return res, node
}
func (c *crawler) do(method string, u *url.URL, wantBody bool, referrer string) (result, *html.Node) {
req, err := http.NewRequest(method, u.String(), nil)
if err != nil {
return result{url: u.String(), status: statusFail, detail: err.Error(), referrer: referrer}, nil
}
req.Header.Set("User-Agent", userAgent)
resp, err := c.client.Do(req)
if err != nil {
return result{url: u.String(), status: statusFail, detail: err.Error(), referrer: referrer}, nil
}
defer resp.Body.Close()
res := result{url: u.String(), code: resp.StatusCode, referrer: referrer}
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 400:
res.status = statusOK
case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
res.status = statusWarn
res.detail = "blocked but alive"
default:
res.status = statusFail
res.detail = resp.Status
}
if resp.Request != nil && resp.Request.URL.String() != u.String() {
res.detail = strings.TrimSpace(res.detail + " -> " + resp.Request.URL.String())
}
var node *html.Node
if wantBody && res.status != statusFail {
ct := resp.Header.Get("Content-Type")
if ct == "" || strings.Contains(ct, "html") {
if n, perr := html.Parse(resp.Body); perr == nil {
node = n
}
}
}
return res, node
}
// extractLinks pulls absolute, crawlable URLs out of an HTML document.
func extractLinks(base *url.URL, root *html.Node) []string {
var out []string
seen := map[string]bool{}
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode {
var attr, raw string
switch n.Data {
case "a", "link":
attr = "href"
case "img", "script":
attr = "src"
}
if attr != "" {
for _, a := range n.Attr {
if a.Key == attr {
raw = a.Val
}
}
}
if raw != "" {
if abs := resolve(base, raw); abs != "" && !seen[abs] {
seen[abs] = true
out = append(out, abs)
}
}
}
for ch := n.FirstChild; ch != nil; ch = ch.NextSibling {
walk(ch)
}
}
walk(root)
return out
}
// resolve turns a possibly-relative link into an absolute http(s) URL, or
// returns "" for links that should be skipped (mailto:, fragments, etc.).
func resolve(base *url.URL, raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" || strings.HasPrefix(raw, "#") {
return ""
}
switch {
case strings.HasPrefix(raw, "mailto:"),
strings.HasPrefix(raw, "tel:"),
strings.HasPrefix(raw, "javascript:"),
strings.HasPrefix(raw, "data:"):
return ""
}
ref, err := url.Parse(raw)
if err != nil {
return ""
}
abs := base.ResolveReference(ref)
if abs.Scheme != "http" && abs.Scheme != "https" {
return ""
}
abs.Fragment = ""
return abs.String()
}
func (c *crawler) record(r result) {
c.mu.Lock()
c.results = append(c.results, r)
c.mu.Unlock()
}
// report prints all results and returns the process exit code.
func (c *crawler) report() int {
c.mu.Lock()
defer c.mu.Unlock()
sort.Slice(c.results, func(i, j int) bool {
if c.results[i].status != c.results[j].status {
return c.results[i].status > c.results[j].status // FAIL first
}
return c.results[i].url < c.results[j].url
})
var ok, warn, fail int
for _, r := range c.results {
switch r.status {
case statusOK:
ok++
case statusWarn:
warn++
case statusFail:
fail++
}
line := fmt.Sprintf("%s %3d %s", r.status, r.code, r.url)
if r.detail != "" {
line += " (" + r.detail + ")"
}
if r.referrer != "" {
line += " [on " + r.referrer + "]"
}
fmt.Println(line)
}
fmt.Printf("\n%d ok, %d warn, %d fail (%d urls)\n", ok, warn, fail, len(c.results))
if fail > 0 {
return 1
}
if warn > 0 && c.cfg.strict {
return 1
}
return 0
}
|