1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

package main

import (
	"bytes"
	"database/sql"
	"rss-kakapo/logic"
	"strings"
	"unicode/utf8"

	"github.com/mmcdole/gofeed"
)

// metaSchema is filled by analyzeMeta: for every analysed page, what it and
// the feed rss-kakapo follows for it say the site is called, and what it is
// about. Which of the two to name a feed account after is the question.
const metaSchema = `
CREATE TABLE IF NOT EXISTS meta (
	handle       TEXT PRIMARY KEY REFERENCES page(handle),
	page_title   TEXT NOT NULL,  -- logic.ReadSiteMeta's, as rss-kakapo names the account now
	page_desc    TEXT NOT NULL,
	feed_url     TEXT,           -- NULL: no feed fetched for the page
	feed_error   TEXT,           -- the fetch failed, or the feed did not parse
	feed_type    TEXT,           -- rss, atom or json
	feed_title   TEXT,           -- white space collapsed, like page_title
	feed_desc    TEXT,
	items        INTEGER,
	-- page_title is the title of one of the feed's items, or holds it: the
	-- page is a post, or names itself after the newest one.
	page_is_item INTEGER,
	-- the title of the item page_is_item matched, for checking by eye
	item_title   TEXT
);
`

// minItemTitle is the shortest item title page_is_item matches on, in
// characters: shorter ones ("Home", "News", "第１話") are as likely part of
// the site's own name. "No Alarm" and "Comet Ice" are long enough.
const minItemTitle = 6

// analyzeMeta fills meta for the pages of page, within analyze's tx, which
// emptied it.
func analyzeMeta(tx *sql.Tx) error {
	rows, err := tx.Query(`SELECT f.handle, f.body, ff.url, ff.status, ff.error, ff.body
		FROM page p JOIN fetch f ON f.handle=p.handle
		LEFT JOIN feed_fetch ff ON ff.handle=p.handle`)
	if err != nil {
		return err
	}
	type row struct {
		handle           string
		page             []byte
		feedUrl, errText *string
		status           *int
		feed             []byte
	}
	var all []row
	for rows.Next() {
		var r row
		if err = rows.Scan(&r.handle, &r.page, &r.feedUrl, &r.status, &r.errText, &r.feed); err != nil {
			rows.Close()
			return err
		}
		all = append(all, r)
	}
	rows.Close()
	if err = rows.Err(); err != nil {
		return err
	}

	for _, r := range all {
		sm := logic.ReadSiteMeta(bytes.NewReader(r.page))
		m := metaRow{handle: r.handle, pageTitle: sm.Title, pageDesc: sm.Description, feedUrl: r.feedUrl}
		if r.feedUrl != nil {
			m.readFeed(r.status, r.errText, r.feed)
		}
		if err = m.store(tx); err != nil {
			return err
		}
	}
	return nil
}

type metaRow struct {
	handle, pageTitle, pageDesc string
	feedUrl                     *string
	feedError                   *string
	feedType, feedTitle         *string
	feedDesc                    *string
	items                       *int
	pageIsItem                  *bool
	itemTitle                   *string
}

func (m *metaRow) readFeed(status *int, errText *string, body []byte) {
	fail := func(s string) { m.feedError = &s }
	switch {
	case errText != nil:
		fail(*errText)
		return
	case status == nil || *status != 200:
		fail("status not 200")
		return
	}
	feed, err := gofeed.NewParser().Parse(bytes.NewReader(body))
	if err != nil {
		fail("parse: " + err.Error())
		return
	}
	typ, title, desc := feed.FeedType, collapse(feed.Title), collapse(feed.Description)
	n := len(feed.Items)
	m.feedType, m.feedTitle, m.feedDesc, m.items = &typ, &title, &desc, &n

	isItem := false
	pt := strings.ToLower(m.pageTitle)
	for _, itm := range feed.Items {
		it := collapse(itm.Title)
		if utf8.RuneCountInString(it) < minItemTitle || !strings.Contains(pt, strings.ToLower(it)) {
			continue
		}
		// A site whose posts are named after it is not a page named after a
		// post.
		if title != "" && strings.EqualFold(it, title) {
			continue
		}
		isItem = true
		m.itemTitle = &it
		break
	}
	m.pageIsItem = &isItem
}

func (m *metaRow) store(tx *sql.Tx) error {
	_, err := tx.Exec(`INSERT INTO meta (handle, page_title, page_desc, feed_url, feed_error,
		feed_type, feed_title, feed_desc, items, page_is_item, item_title)
		VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
		m.handle, m.pageTitle, m.pageDesc, m.feedUrl, m.feedError,
		m.feedType, m.feedTitle, m.feedDesc, m.items, m.pageIsItem, m.itemTitle)
	return err
}

// collapse trims s and collapses its runs of white space, as
// logic.ReadSiteMeta does with a page's title.
func collapse(s string) string {
	return strings.Join(strings.Fields(s), " ")
}