1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
|
package main
import (
"bytes"
"errors"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"time"
"unicode"
)
// Accepting submissions
// ============================================================================
//
// Everything a stranger can influence is bounded before it is stored: the size
// of the upload, what the bytes are allowed to be, how often one address may
// submit, and how much may accumulate in total. The last of these is the one
// that protects the machine rather than the service — see checkQuota.
// maxUploadBytes bounds a single submission.
//
// 5 MB is roughly five minutes from a phone recorder and a quarter of an hour
// at the bitrate observations itself records at, which suits replies rather
// than essays. It is also small enough that the whole thing can be held in
// memory to sniff and store without thinking about it.
const maxUploadBytes = 5 << 20
// quotaBytes is the total the inbox may hold before it stops accepting.
//
// This is not about the inbox, it is about the machine: the database lives on
// the same filesystem as nginx, source-forge, gotosocial and hosty, with very
// little headroom. A submission endpoint is the one part of this system a
// stranger can make write to disk, so it gets a ceiling well below anything
// that could fill the volume, and refuses rather than degrade everything else.
const quotaBytes = 50 << 20
// submitRate is how often one address may submit.
//
// Deliberately loose. This is a brake on a script, not a quota on a person:
// the real ceilings are the passphrase (which a scanner does not have) and the
// 50 MB total (which bounds the damage anyway). Set too tight it punishes the
// honest case — a second thought after sending one, a retake, a household
// behind one address — and the failure it produces, a flat refusal, is
// indistinguishable from the site being broken.
const (
submitRate = 12
submitRateWind = time.Hour
)
var errQuotaFull = errors.New("inbox full")
// audioType is a recognised audio format.
type audioType struct {
MIME string
Ext string
}
// sniffAudio identifies an upload from its bytes.
//
// Content decides, never the filename: an extension is a claim by whoever
// named the file, and this endpoint is open to people we have no reason to
// trust. Anything not recognised as audio is refused outright rather than
// stored as "probably fine", because the only thing this inbox is for is
// recordings.
func sniffAudio(data []byte) (*audioType, error) {
switch {
// ISO base media (m4a/mp4): "ftyp" at offset 4, then a brand.
case len(data) >= 12 && bytes.Equal(data[4:8], []byte("ftyp")):
brand := string(data[8:12])
switch brand {
case "M4A ", "M4B ", "mp42", "isom", "iso2", "mp41", "dash":
return &audioType{"audio/mp4", ".m4a"}, nil
}
// Some recorders write other brands; the container is still MP4 and
// every player treats it as such.
return &audioType{"audio/mp4", ".m4a"}, nil
case bytes.HasPrefix(data, []byte("OggS")):
// Ogg carries Opus or Vorbis; both are audio and both play natively.
return &audioType{"audio/ogg", ".ogg"}, nil
case bytes.HasPrefix(data, []byte("fLaC")):
return &audioType{"audio/flac", ".flac"}, nil
case bytes.HasPrefix(data, []byte("ID3")):
return &audioType{"audio/mpeg", ".mp3"}, nil
// A bare MP3 frame: 11 sync bits. Checked after ID3 because a tagged file
// starts with the tag instead.
case len(data) >= 2 && data[0] == 0xff && data[1]&0xe0 == 0xe0:
return &audioType{"audio/mpeg", ".mp3"}, nil
case len(data) >= 12 && bytes.Equal(data[0:4], []byte("RIFF")) &&
bytes.Equal(data[8:12], []byte("WAVE")):
return &audioType{"audio/wav", ".wav"}, nil
// WebM/Matroska (what a browser's MediaRecorder produces).
case bytes.HasPrefix(data, []byte{0x1a, 0x45, 0xdf, 0xa3}):
return &audioType{"audio/webm", ".webm"}, nil
}
return nil, errors.New("not a recognised audio file")
}
// Matching the passphrase
// ----------------------------------------------------------------------------
//
// The passphrase is spoken aloud at the start of an episode and typed in from
// memory, possibly on a phone, possibly minutes later. Everything that can go
// wrong between hearing a phrase and typing it is a person doing their best
// and being told "no" by a form, so matching is generous:
//
// - case is ignored, and so is every kind of whitespace, including none at
// all: "OpenSesame" and "open sesame" are the same phrase;
// - punctuation and accents are dropped, so an apostrophe or a hyphen where
// the speaker did not intend one costs nothing;
// - and a small number of typos are tolerated, scaled to length, because
// hearing a word and typing it is exactly where a letter goes missing.
//
// This is not a secret and is not treated as one. It is a filter that keeps
// the endpoint from being found and used by scanners, and every listener has
// it. Widening it trades a security property this thing never had for a usable
// one it needs.
//
// The consequence is that the phrase is held in memory rather than as a hash:
// an edit distance cannot be computed against a digest, since hashing destroys
// exactly the locality it needs. Given the phrase is broadcast in an audio
// recording, the hash was never protecting anything.
// passphraseTypoBudget is how many single-character edits are forgiven, by
// length of the normalised phrase. Short phrases get no slack, because at four
// characters a single edit reaches too many other words.
func passphraseTypoBudget(n int) int {
switch {
case n < 6:
return 0
case n < 12:
return 1
default:
return 2
}
}
// normalisePassphrase reduces a phrase to the letters and digits in it,
// lowercased. Whitespace, punctuation and accents all disappear, so the
// comparison is about the word someone heard rather than how they wrote it
// down.
func normalisePassphrase(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(s) {
switch {
case unicode.IsLetter(r) || unicode.IsDigit(r):
b.WriteRune(foldAccent(r))
default:
// Whitespace, punctuation, emoji: all dropped.
}
}
return b.String()
}
// accentFolds maps the Latin-1 accented letters onto their plain forms, so
// that a phrase heard as "cafe" matches one typed as "café".
//
// A map rather than two parallel strings: the obvious version of this indexes
// a "plain" string with strings.IndexRune of an "accented" one, which silently
// returns a *byte* offset into a multibyte string and folds é to o. Only the
// languages this site is written in are covered; this is a convenience, not a
// general Unicode normalisation.
var accentFolds = map[rune]rune{
'à': 'a', 'á': 'a', 'â': 'a', 'ã': 'a', 'ä': 'a', 'å': 'a',
'ç': 'c',
'è': 'e', 'é': 'e', 'ê': 'e', 'ë': 'e',
'ì': 'i', 'í': 'i', 'î': 'i', 'ï': 'i',
'ñ': 'n',
'ò': 'o', 'ó': 'o', 'ô': 'o', 'õ': 'o', 'ö': 'o', 'ø': 'o', 'œ': 'o',
'ù': 'u', 'ú': 'u', 'û': 'u', 'ü': 'u',
'ý': 'y', 'ÿ': 'y',
'æ': 'a', 'ß': 's',
}
func foldAccent(r rune) rune {
if plain, ok := accentFolds[r]; ok {
return plain
}
return r
}
// passphraseOK reports whether a submitted passphrase matches the configured
// one, after normalisation and within the typo budget.
func passphraseOK(want, got string) bool {
w := normalisePassphrase(want)
g := normalisePassphrase(got)
if w == "" || g == "" {
return false
}
if w == g {
return true
}
budget := passphraseTypoBudget(len(w))
if budget == 0 {
return false
}
// A length difference beyond the budget cannot be closed by edits, and
// checking it first keeps the matrix small.
if abs(len(w)-len(g)) > budget {
return false
}
return editDistanceWithin(w, g, budget)
}
// editDistanceWithin reports whether a and b are within max single-character
// edits of each other, counting a transposition of two adjacent characters as
// one edit (Damerau-Levenshtein).
//
// Transpositions are counted as one rather than two because "seasme" for
// "sesame" is one of the most ordinary ways to mistype a word, and charging it
// double would put it out of reach of any sensible budget.
//
// The full matrix is not needed, only whether the distance stays within a
// small bound, so this keeps three rows and gives up as soon as an entire row
// exceeds the budget.
func editDistanceWithin(a, b string, max int) bool {
ra, rb := []rune(a), []rune(b)
if len(ra) < len(rb) {
ra, rb = rb, ra
}
if len(ra)-len(rb) > max {
return false
}
// prev2 is the row before prev, needed only for the transposition case.
prev2 := make([]int, len(rb)+1)
prev := make([]int, len(rb)+1)
cur := make([]int, len(rb)+1)
for j := range prev {
prev[j] = j
}
for i := 1; i <= len(ra); i++ {
cur[0] = i
best := cur[0]
for j := 1; j <= len(rb); j++ {
cost := 1
if ra[i-1] == rb[j-1] {
cost = 0
}
cur[j] = min3(cur[j-1]+1, prev[j]+1, prev[j-1]+cost)
if i > 1 && j > 1 && ra[i-1] == rb[j-2] && ra[i-2] == rb[j-1] {
if t := prev2[j-2] + 1; t < cur[j] {
cur[j] = t
}
}
if cur[j] < best {
best = cur[j]
}
}
if best > max {
return false
}
prev2, prev, cur = prev, cur, prev2
}
return prev[len(rb)] <= max
}
func min3(a, b, c int) int {
if b < a {
a = b
}
if c < a {
a = c
}
return a
}
func abs(n int) int {
if n < 0 {
return -n
}
return n
}
// rateLimiter counts submissions per address in a sliding window.
//
// Deliberately in memory: a restart forgetting the counts is acceptable, and
// it keeps the abuse defence from becoming another thing that writes to the
// disk the quota exists to protect.
type rateLimiter struct {
mu sync.Mutex
seen map[string][]time.Time
limit int
window time.Duration
nowFunc func() time.Time
}
func newRateLimiter(limit int, window time.Duration) *rateLimiter {
return &rateLimiter{
seen: map[string][]time.Time{},
limit: limit,
window: window,
nowFunc: time.Now,
}
}
// allow records an attempt and reports whether it is within the limit.
func (rl *rateLimiter) allow(key string) bool {
rl.mu.Lock()
defer rl.mu.Unlock()
now := rl.nowFunc()
cutoff := now.Add(-rl.window)
kept := rl.seen[key][:0]
for _, t := range rl.seen[key] {
if t.After(cutoff) {
kept = append(kept, t)
}
}
if len(kept) >= rl.limit {
rl.seen[key] = kept
return false
}
rl.seen[key] = append(kept, now)
// Keep the map from growing without bound when many addresses submit
// once and never return.
if len(rl.seen) > 10000 {
for k, ts := range rl.seen {
if len(ts) == 0 || ts[len(ts)-1].Before(cutoff) {
delete(rl.seen, k)
}
}
}
return true
}
// clientIP is the address a request came from, trusting the reverse proxy's
// X-Forwarded-For only for its last entry (the one the proxy itself observed;
// earlier entries are supplied by the client and can say anything).
func clientIP(r *http.Request) string {
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
parts := strings.Split(xff, ",")
if ip := strings.TrimSpace(parts[len(parts)-1]); ip != "" {
return ip
}
}
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
return host
}
return r.RemoteAddr
}
// readUpload reads the single uploaded file, refusing anything oversized.
//
// The body is wrapped in a MaxBytesReader before the multipart reader sees it,
// so a request that lies about its Content-Length cannot make the server
// allocate more than the limit.
func readUpload(w http.ResponseWriter, r *http.Request) (filename string, data []byte, err error) {
r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes+1<<20)
mr, err := r.MultipartReader()
if err != nil {
return "", nil, fmt.Errorf("not a multipart upload: %w", err)
}
var (
fileData []byte
fileName string
)
fields := map[string]string{}
for {
part, err := mr.NextPart()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return "", nil, fmt.Errorf("read upload: %w", err)
}
if part.FileName() == "" {
// An ordinary form field: passphrase, consent, note.
v, err := io.ReadAll(io.LimitReader(part, 4<<10))
part.Close()
if err != nil {
return "", nil, fmt.Errorf("read field %q: %w", part.FormName(), err)
}
fields[part.FormName()] = string(v)
continue
}
if fileData != nil {
part.Close()
return "", nil, errors.New("only one file per submission")
}
buf, err := io.ReadAll(io.LimitReader(part, maxUploadBytes+1))
part.Close()
if err != nil {
return "", nil, fmt.Errorf("read upload body: %w", err)
}
if len(buf) > maxUploadBytes {
return "", nil, fmt.Errorf("file is larger than %s", humanBytes(maxUploadBytes))
}
fileData = buf
fileName = part.FileName()
}
if fileData == nil {
return "", nil, errors.New("no file in submission")
}
// Fields are returned through the request so the handler can read them
// with the usual accessors.
r.Form = nil
r.PostForm = nil
for k, v := range fields {
if r.Form == nil {
r.Form = map[string][]string{}
}
r.Form[k] = []string{v}
}
return fileName, fileData, nil
}
// checkQuota reports whether there is room for another submission.
func checkQuota(used int64, incoming int64) error {
if used+incoming > quotaBytes {
return errQuotaFull
}
return nil
}
func humanBytes(n int64) string {
switch {
case n >= 1<<20:
return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
case n >= 1<<10:
return fmt.Sprintf("%.1f kB", float64(n)/(1<<10))
default:
return fmt.Sprintf("%d B", n)
}
}
|