1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100

package logic

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// Polite polling (rss-kakapo).
//
// Upstream re-downloads every feed in full on every check, and retries a
// failing feed on its normal schedule forever. Here a check sends the
// validators of the last 200 response back (If-None-Match/If-Modified-Since),
// so an unchanged feed costs a 304 with no body; and every consecutive failure
// doubles the wait before the next attempt, while a server that says when to
// come back (Retry-After on a 429 or 503) is listened to.
//
// Neither may silence a feed for long, though: a misconfigured server can send
// "Retry-After: 3024000" (five weeks), and a feed that is down for a while
// would otherwise back off for days. So the wait after a failure is capped at
// max_retry_delay_hours (a day by default) — except for feeds whose normal
// interval is longer than that, which simply stay on their normal schedule,
// so that a failing feed is never polled more often than a healthy one.

const (
	// maxBackoffFailures caps the doubling: 2^6 = 64 times the normal interval.
	maxBackoffFailures = 6
	// defaultMaxRetryDelay is the cap when max_retry_delay_hours is not set.
	defaultMaxRetryDelay = 24 * time.Hour
)

// feedFetchError is a failed feed fetch. RetryAfter is what the server asked
// for (zero if it said nothing).
type feedFetchError struct {
	StatusCode int
	RetryAfter time.Duration
}

func (e *feedFetchError) Error() string {
	if e.RetryAfter > 0 {
		return fmt.Sprintf("request failed with status %d (retry after %s)", e.StatusCode, e.RetryAfter)
	}
	return fmt.Sprintf("request failed with status %d", e.StatusCode)
}

// parseRetryAfter reads a Retry-After header, which is either a number of
// seconds or an HTTP date. Unparseable or past values yield zero.
func parseRetryAfter(val string, now time.Time) time.Duration {
	val = strings.TrimSpace(val)
	if val == "" {
		return 0
	}
	if secs, err := strconv.Atoi(val); err == nil {
		if secs <= 0 {
			return 0
		}
		return time.Duration(secs) * time.Second
	}
	if t, err := http.ParseTime(val); err == nil {
		if d := t.Sub(now); d > 0 {
			return d
		}
	}
	return 0
}

// backoffDelay is how long to wait before checking a feed again after its
// failures-th consecutive failed check: the normal interval doubled once per
// failure, or longer if the server asked for more via Retry-After — but never
// longer than maxDelay, or than the normal interval if that is longer still.
func backoffDelay(interval time.Duration, failures int, retryAfter, maxDelay time.Duration) time.Duration {
	if failures < 1 {
		failures = 1
	}
	if failures > maxBackoffFailures {
		failures = maxBackoffFailures
	}
	delay := interval << failures
	if retryAfter > delay {
		delay = retryAfter
	}
	ceiling := maxDelay
	if interval > ceiling {
		ceiling = interval
	}
	if delay > ceiling {
		delay = ceiling
	}
	return delay
}

// maxRetryDelay is the configured cap on the wait after a failed check.
func maxRetryDelay(hours int) time.Duration {
	if hours <= 0 {
		return defaultMaxRetryDelay
	}
	return time.Duration(hours) * time.Hour
}