1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270

package logic

import (
	"net/url"
	"rss_parrot/dal"
	"sort"
	"strings"
	"time"

	"github.com/mmcdole/gofeed"
)

// Which feed items get tooted (rss-kakapo).
//
// Upstream decides by date alone: an item is new if it is dated after the
// newest item seen so far (feed_last_updated), and a 32-bit hash of guid and
// link only weeds out duplicates among those. Posts are forgotten when they
// are purged, so a feed that re-dates its archive gets it tooted again; and
// an item is only stored once it is to be tooted, so one dated too far ahead
// is dropped and, if the feed does not change again, never looked at again.
//
// Here every item is remembered in a ledger (feed_items) that is never
// purged, by its guid and by its link. A fetch only records items; tooting is
// a separate step (postDueItems), which toots an item once its date has come.
// An item is tooted only if both
//
//   - it is new: neither its guid nor its link was ever seen for the feed;
//   - and it is recent: dated after the newest item seen so far, or at most
//     backdate_grace_hours (default 48) before it. An item without a date
//     counts as dated when it was first seen.
//
// So a feed that re-dates old items toots nothing (they are not new), nor does
// one that changes their guids or links but not their dates (they are not
// recent). Within the grace period, an item with a new guid and link may
// still just be an old one renamed; it is taken for one if it is dated
// exactly like a remembered item that is gone from the feed. (A second post
// on the same day of a feed that only has dates is not mistaken for one: the
// first is still in the feed.) Only a feed that changes the identities and the
// dates of its items at once gets through; for that, max_new_items_per_fetch
// can cap how many items one fetch may toot.
//
// feed_last_updated, the "newest item seen so far", never moves past now, so
// an item dated into the future does not hide the ones published before it.

const defaultBackdateGrace = 48 * time.Hour

func (ff *feedFollower) backdateGrace() time.Duration {
	if ff.cfg.BackdateGraceHours == nil {
		return defaultBackdateGrace
	}
	return time.Duration(*ff.cfg.BackdateGraceHours) * time.Hour
}

// itemDate is the later of an item's published and updated dates.
func itemDate(itm *gofeed.Item) (time.Time, bool) {
	var t time.Time
	if itm.PublishedParsed != nil {
		t = *itm.PublishedParsed
	}
	if itm.UpdatedParsed != nil && itm.UpdatedParsed.After(t) {
		t = *itm.UpdatedParsed
	}
	return t, !t.IsZero()
}

// itemLinkKey is the form of a link that items are matched by: without
// scheme (so moving to https changes nothing), host in lower case, without a
// trailing slash, and without utm_* tracking parameters.
func itemLinkKey(link string) string {
	link = strings.TrimSpace(link)
	if link == "" {
		return ""
	}
	u, err := url.Parse(link)
	if err != nil || u.Host == "" {
		return link
	}
	q := u.Query()
	for k := range q {
		if strings.HasPrefix(strings.ToLower(k), "utm_") {
			q.Del(k)
		}
	}
	key := strings.ToLower(u.Host) + strings.TrimRight(u.EscapedPath(), "/")
	if len(q) != 0 {
		key += "?" + q.Encode()
	}
	if u.Fragment != "" {
		key += "#" + u.Fragment
	}
	return key
}

// classifyItems turns the items of a fetched feed that are not in the ledger
// yet into ledger rows, each either due for tooting or skipped, and works out
// the feed's new feed_last_updated.
//
// tootNew is false when the feed is being followed: then every item is due,
// and updateAccountPosts publishes it without broadcasting. baselined is false
// for an account whose feed was never read into the ledger (it was followed
// before the ledger existed): its first read follows upstream's rules, so it
// toots exactly what upstream would have.
func (ff *feedFollower) classifyItems(
	accountId int,
	accountHandle string,
	feedItems []*gofeed.Item,
	lastUpdated, now time.Time,
	baselined, tootNew bool,
) (items []*dal.FeedItem, newLastUpdated time.Time, err error) {

	// Older versions let an item dated into the future move
	// feed_last_updated past now.
	if lastUpdated.After(now) {
		lastUpdated = now
	}
	newLastUpdated = lastUpdated
	grace := ff.backdateGrace()

	// A link that several items of this fetch share (a feed that links every
	// item to its home page) says nothing about which item it is.
	linkCount := map[string]int{}
	for _, itm := range feedItems {
		fixPodcastLink(itm)
		if k := itemLinkKey(itm.Link); k != "" {
			linkCount[k]++
		}
	}

	// Every guid and link key of this fetch, to tell which remembered items
	// are still in the feed.
	inFeed := map[string]bool{}
	identity := func(itm *gofeed.Item) (guid, linkKey string) {
		guid = strings.TrimSpace(itm.GUID)
		linkKey = itemLinkKey(itm.Link)
		if linkCount[linkKey] > 1 {
			linkKey = ""
		}
		return
	}
	for _, itm := range feedItems {
		guid, linkKey := identity(itm)
		inFeed["g"+guid], inFeed["l"+linkKey] = true, true
	}

	inFetch := map[string]bool{}
	for _, itm := range feedItems {
		guid, linkKey := identity(itm)
		if guid == "" && linkKey == "" {
			ff.logger.Debugf("Ignoring an item of %s without guid or distinct link: %q", accountHandle, itm.Title)
			continue
		}
		if inFetch["g"+guid] && guid != "" || inFetch["l"+linkKey] && linkKey != "" {
			continue
		}
		inFetch["g"+guid], inFetch["l"+linkKey] = true, true

		byGuid, byLink, err := ff.repo.FindSeenItem(accountId, guid, linkKey)
		if err != nil {
			return nil, lastUpdated, err
		}
		if byGuid || byLink {
			continue
		}

		date, dated := itemDate(itm)
		postTime := date
		if !dated {
			postTime = now
		}
		fi := &dal.FeedItem{
			AccountId:    accountId,
			Guid:         guid,
			LinkKey:      linkKey,
			PostGuidHash: int64(getItemHash(itm)),
			PostTime:     postTime,
			FirstSeen:    now,
			Status:       dal.ItemSkipped,
			Link:         itm.Link,
			Title:        stripHtml(itm.Title),
			Description:  stripHtml(itm.Description),
		}
		switch {
		case !tootNew:
			fi.Status = dal.ItemDue
		case !baselined:
			// Upstream: dated after feed_last_updated, and not stored yet.
			if dated && date.After(lastUpdated) {
				stored, err := ff.repo.HasFeedPost(accountId, fi.PostGuidHash)
				if err != nil {
					return nil, lastUpdated, err
				}
				if !stored {
					fi.Status = dal.ItemDue
				}
			}
		case postTime.After(lastUpdated.Add(-grace)):
			renamed, err := ff.isRenamed(accountId, postTime, inFeed)
			if err != nil {
				return nil, lastUpdated, err
			}
			if renamed {
				ff.logger.Infof("Not tooting a new item of %s dated like one that is gone from the feed: %q",
					accountHandle, fi.Title)
			} else {
				fi.Status = dal.ItemDue
			}
		default:
			ff.logger.Infof("Not tooting a new item of %s dated %s, before the last one seen (%s): %q",
				accountHandle, postTime.Format(time.RFC3339), lastUpdated.Format(time.RFC3339), fi.Title)
		}
		if fi.Status != dal.ItemDue {
			fi.Link, fi.Title, fi.Description = "", "", ""
		} else {
			changed := postTime
			if changed.After(now) {
				changed = now
			}
			if changed.After(newLastUpdated) {
				newLastUpdated = changed
			}
		}
		items = append(items, fi)
	}

	if tootNew {
		ff.capNewItems(accountHandle, items)
	}
	return items, newLastUpdated, nil
}

// isRenamed reports whether an item dated postTime looks like a remembered
// item under a new guid and link: one dated exactly the same that is gone
// from the feed (inFeed holds the guids and link keys of the fetch).
func (ff *feedFollower) isRenamed(accountId int, postTime time.Time, inFeed map[string]bool) (bool, error) {
	same, err := ff.repo.GetItemsAt(accountId, postTime)
	if err != nil {
		return false, err
	}
	for _, old := range same {
		if !(old.Guid != "" && inFeed["g"+old.Guid] || old.LinkKey != "" && inFeed["l"+old.LinkKey]) {
			return true, nil
		}
	}
	return false, nil
}

// capNewItems enforces max_new_items_per_fetch: if more items are due than
// that, only the newest that many stay due.
func (ff *feedFollower) capNewItems(accountHandle string, items []*dal.FeedItem) {
	maxNew := ff.cfg.MaxNewItemsPerFetch
	if maxNew <= 0 {
		return
	}
	var due []*dal.FeedItem
	for _, fi := range items {
		if fi.Status == dal.ItemDue {
			due = append(due, fi)
		}
	}
	if len(due) <= maxNew {
		return
	}
	sort.SliceStable(due, func(i, j int) bool { return due[i].PostTime.After(due[j].PostTime) })
	for _, fi := range due[maxNew:] {
		fi.Status = dal.ItemSkipped
		fi.Link, fi.Title, fi.Description = "", "", ""
	}
	ff.logger.Warnf("Feed %s has %d new items at once; tooting only the newest %d (max_new_items_per_fetch)",
		accountHandle, len(due), maxNew)
}