1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
package main
import (
"database/sql"
"fmt"
"log"
"strings"
"github.com/emersion/go-imap"
)
// printMIMETree prints a BodyStructure tree with indentation.
func printMIMETree(bs *imap.BodyStructure, depth int) {
indent := strings.Repeat(" ", depth)
mt := strings.ToLower(bs.MIMEType) + "/" + strings.ToLower(bs.MIMESubType)
if bs.MIMEType == "multipart" || strings.ToLower(bs.MIMEType) == "multipart" {
fmt.Printf("%s[%s]\n", indent, mt)
for i, child := range bs.Parts {
fmt.Printf("%s %d:\n", indent, i+1)
printMIMETree(child, depth+2)
}
} else {
fmt.Printf("%s%s (size=%d)\n", indent, mt, bs.Size)
}
}
// analyzeBodyStructures fetches BODYSTRUCTURE for every message whose MIME
// structure has not been examined, records the attachments it finds, and prints
// a frequency table of MIME structures.
//
// This is both a survey and the backfill for the attachments table. The two are
// the same pass: naming which part would be displayed requires the whole tree,
// and the tree is exactly what says what else is in the message. Running it
// separately would mean fetching every BODYSTRUCTURE in the archive twice.
//
// The set of messages is those with no bodystructure_scanned_at, not those with
// no display_part. Attachments are unknown for every message synced before the
// table existed, including — in fact especially — the ones that have been read,
// since those are the messages someone cared about.
func analyzeBodyStructures(db *sql.DB, creds imapCreds) error {
// Pool for pre-fetching display_part of multipart/related messages.
prefetchPool := newGlobalPool(creds, 1, 10)
defer prefetchPool.close()
// Load all (id, mailbox, uid, uidvalidity) for messages never examined.
rows, err := db.Query(
`SELECT id, mailbox, uid, uidvalidity FROM messages
WHERE bodystructure_scanned_at IS NULL
ORDER BY mailbox, uidvalidity, uid`,
)
if err != nil {
return fmt.Errorf("query: %w", err)
}
defer rows.Close()
type msgInfo struct {
id int64
mailbox string
uid uint32
uidvalidity uint32
}
// uidToMsg lets us look up the msgInfo by UID when processing BODYSTRUCTURE results.
uidToMsg := make(map[uint32]msgInfo)
var msgs []msgInfo
for rows.Next() {
var m msgInfo
if err := rows.Scan(&m.id, &m.mailbox, &m.uid, &m.uidvalidity); err != nil {
return fmt.Errorf("scan: %w", err)
}
msgs = append(msgs, m)
uidToMsg[m.uid] = m
}
if err := rows.Err(); err != nil {
return err
}
log.Printf("analyze: %d messages with unexamined MIME structure", len(msgs))
// Connect once, process in batches of 100.
c, err := connectIMAP(creds)
if err != nil {
return err
}
defer c.Logout()
// Track current mailbox to avoid redundant SELECTs.
currentMailbox := ""
// Frequency tables.
mimeFreq := make(map[string]int) // top-level MIME type
displayPartFreq := make(map[string]int) // what findDisplayPart picks
attachFreq := make(map[string]int) // MIME types found as attachments
withAttachments := 0 // messages carrying at least one
const batchSize = 100
for i := 0; i < len(msgs); i += batchSize {
end := min(i+batchSize, len(msgs))
batch := msgs[i:end]
// Group by mailbox within this batch (usually all same mailbox).
// For simplicity just re-select if needed.
if batch[0].mailbox != currentMailbox {
if _, err := c.Select(batch[0].mailbox, true); err != nil {
return fmt.Errorf("select %s: %w", batch[0].mailbox, err)
}
currentMailbox = batch[0].mailbox
}
seqset := new(imap.SeqSet)
for _, m := range batch {
seqset.AddNum(m.uid)
}
bsMsgs := make(chan *imap.Message, batchSize)
done := make(chan error, 1)
go func() {
done <- c.UidFetch(seqset, []imap.FetchItem{imap.FetchUid, imap.FetchBodyStructure}, bsMsgs)
}()
for msg := range bsMsgs {
if msg.BodyStructure == nil {
// A UID that returns no BODYSTRUCTURE was expunged; it is not
// marked as scanned, since there is nothing to scan and the row
// will be pruned by the next sync.
mimeFreq["(expunged)"]++
displayPartFreq["(expunged)"]++
continue
}
bs := msg.BodyStructure
topLevel := strings.ToLower(bs.MIMEType) + "/" + strings.ToLower(bs.MIMESubType)
mimeFreq[topLevel]++
dp := findDisplayPart(bs)
if dp.rfc822Text {
displayPartFreq["RFC822.TEXT fallback"]++
} else {
displayPartFreq[dp.mimeType]++
}
// Record what else is in the message. This is the backfill: after
// this pass every message either has its attachments listed or is
// gone from the server.
if m, ok := uidToMsg[msg.Uid]; ok {
atts := collectAttachments(bs, dp)
if err := recordAttachments(db, m.id, atts); err != nil {
log.Printf("analyze: record attachments uid=%d: %v", msg.Uid, err)
}
if len(atts) > 0 {
withAttachments++
}
for _, a := range atts {
attachFreq[a.MimeType]++
}
}
// Pre-fetch display_part for multipart/related messages so the
// ?test=cid filter works without needing to open each mail first.
if topLevel == "multipart/related" {
if m, ok := uidToMsg[msg.Uid]; ok {
ref := msgRef{
id: m.id,
mbox: Mailbox{Name: m.mailbox, UIDValidity: m.uidvalidity},
uid: m.uid,
}
if _, _, err := fetchSingleBody(db, prefetchPool, ref); err != nil {
log.Printf("analyze: prefetch uid=%d: %v", msg.Uid, err)
} else {
log.Printf("analyze: prefetched display_part for uid=%d", msg.Uid)
}
}
}
}
if err := <-done; err != nil {
log.Printf("warn: batch %d: %v", i/batchSize, err)
}
log.Printf("analyze: processed %d/%d", end, len(msgs))
}
fmt.Println("\n=== Top-level MIME type frequency ===")
for mt, n := range mimeFreq {
fmt.Printf(" %5d %s\n", n, mt)
}
fmt.Println("\n=== findDisplayPart result frequency ===")
for mt, n := range displayPartFreq {
fmt.Printf(" %5d %s\n", n, mt)
}
fmt.Printf("\n=== attachments (%d messages carry at least one) ===\n", withAttachments)
for mt, n := range attachFreq {
fmt.Printf(" %5d %s\n", n, mt)
}
return nil
}
|