1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207

package main

import (
	"encoding/base64"
	"strings"
	"testing"
)

// goldenStream is the first three segments of the transcription track of
// episode 001, byte for byte as ffmpeg extracts it:
//
//	ffmpeg -i rec.m4a -map 0:1 -c copy -f data -
//
// The full track is 102384 bytes; this is the first 738. It is checked in
// because the recording itself deliberately never enters the repository (see
// observations(7)), and a decoder for an undocumented third-party format
// needs a real sample to test against rather than one we made up to match
// our own understanding of it.
const goldenStream = `
AAAAjQoYCgNIZXkSBQpIZXksGJYBIKIHOgQIABAAChEKA2FuZBiiByDWCDoECAAQAAoVCgd3ZWxj
b21lGNYIIIoKOgQIABAAChAKAnRvGIoKILYMOgQIABAACikKDG9ic2VydmF0aW9ucxINb2JzZXJ2
YXRpb25zLhi2DCDuFDoECAAQABUAAAAAGgVlbi1VUwAAAd0KGwoFVG9kYXkSBlRvZGF5LBiGISDm
JDoECAEQAQoPCgFJGOYkIN4lOgQIABABChMKBXdvdWxkGN4lIJomOgQIABABChIKBGxpa2UYmiYg
kic6BAgAEAEKEAoCdG8Ykicgiig6BAgAEAEKGQoEdGFsaxIFdGFsay4Yiiggvik6BAgAEAEKDwoB
SRi+KSCyMjoECAAQAQoTCgV3b3VsZBiyMiCqMzoECAAQAQoSCgRsaWtlGKozIKI0OgQIABABChAK
AnRvGKI0IJo1OgQIABABChIKBHJlYWQYmjUggjg6BAgAEAEKEQoDYW5kGII4IJY9OgQIABABChIK
BHRoZW4Ylj0ghj86BAgAEAEKEwoFbWF5YmUYhj8gukA6BAgAEAEKEgoEdGFsaxi6QCCyQToECAAQ
AQoPCgFhGLJBIKpCOgQIABABChQKBmxpdHRsZRiqQiCrQjoECAAQAQoRCgNiaXQYq0IgokM6BAgA
EAEKEwoFYWJvdXQYokMg1kQ6BAgAEAEKEAoCYW4Y1kQglkw6BAgAEAEKFQoHYXJ0aWNsZRiWTCD+
TjoECAAQAQojCglwdWJsaXNoZWQSCnB1Ymxpc2hlZC4Y/k4gglY6BAgAEAEVAAAAABoFZW4tVVMA
AABsChAKAkluGO5fIKpgOgQIABABChkKBDIwMjMSBTIwMjMsGKpgIPJmOgQIABABChYKCE5vdmVt
YmVyGPJmIMprOgQIABABChkKBDMwdGgSBTMwdGguGMprILZ1OgQIABABFQAAAAAaBWVuLVVT
`

func golden(t *testing.T) []byte {
	t.Helper()
	b, err := base64.StdEncoding.DecodeString(strings.ReplaceAll(goldenStream, "\n", ""))
	if err != nil {
		t.Fatalf("decoding golden fixture: %v", err)
	}
	return b
}

func TestDecodeTranscriptGolden(t *testing.T) {
	tr, err := DecodeTranscript(golden(t))
	if err != nil {
		t.Fatalf("DecodeTranscript: %v", err)
	}

	if got, want := len(tr.Segments), 3; got != want {
		t.Fatalf("segments = %d, want %d", got, want)
	}
	if got, want := tr.WordCount(), 5+22+4; got != want {
		t.Errorf("WordCount = %d, want %d", got, want)
	}

	// First segment: the opening line, and the timings that make the whole
	// track worth keeping.
	seg := tr.Segments[0]
	if got, want := seg.Language, "en-US"; got != want {
		t.Errorf("language = %q, want %q", got, want)
	}
	want := []Word{
		{Text: "Hey", Formatted: "Hey,", StartMs: 150, EndMs: 930, Paragraph: true},
		{Text: "and", StartMs: 930, EndMs: 1110},
		{Text: "welcome", StartMs: 1110, EndMs: 1290},
		{Text: "to", StartMs: 1290, EndMs: 1590},
		{Text: "observations", Formatted: "observations.", StartMs: 1590, EndMs: 2670},
	}
	for i, w := range want {
		if got := seg.Words[i]; got != w {
			t.Errorf("segment 0 word %d = %+v, want %+v", i, got, w)
		}
	}
}

// The formatted form carries punctuation, and a leading newline that means
// "paragraph break". That newline is structure rather than text, so it must
// not survive into the word itself — otherwise every renderer has to know to
// strip it.
func TestParagraphBreakIsLiftedOutOfText(t *testing.T) {
	tr, err := DecodeTranscript(golden(t))
	if err != nil {
		t.Fatalf("DecodeTranscript: %v", err)
	}
	for si, seg := range tr.Segments {
		for wi, w := range seg.Words {
			if strings.ContainsAny(w.Text+w.Formatted, "\n") {
				t.Errorf("segment %d word %d retains a newline: %q / %q", si, wi, w.Text, w.Formatted)
			}
		}
	}
	if !tr.Segments[0].Words[0].Paragraph {
		t.Error("first word should start a paragraph")
	}
	if tr.Segments[0].Words[1].Paragraph {
		t.Error("second word should not start a paragraph")
	}
}

// Formatted is only stored when it differs from Text, so that the common case
// costs nothing in the stored JSON.
func TestFormattedOmittedWhenRedundant(t *testing.T) {
	tr, err := DecodeTranscript(golden(t))
	if err != nil {
		t.Fatalf("DecodeTranscript: %v", err)
	}
	for _, seg := range tr.Segments {
		for _, w := range seg.Words {
			if w.Formatted != "" && w.Formatted == w.Text {
				t.Errorf("word %q stores a redundant formatted form", w.Text)
			}
		}
	}
}

func TestText(t *testing.T) {
	tr, err := DecodeTranscript(golden(t))
	if err != nil {
		t.Fatalf("DecodeTranscript: %v", err)
	}
	got := tr.Text()
	want := "Hey, and welcome to observations. Today, I would like to talk."
	if !strings.HasPrefix(got, want) {
		t.Errorf("Text() = %q..., want prefix %q", got[:min(len(got), 80)], want)
	}
	// Segment boundaries are not paragraph boundaries: the recogniser emits
	// far more segments (308 in episode 001) than paragraph marks (83), so
	// joining segments with blank lines would invent structure that is not
	// there. Only an explicit mark breaks a paragraph.
	if strings.Contains(got, "\n\n") {
		t.Errorf("Text() invented a paragraph break: %q", got)
	}
}

// The recogniser marks paragraphs sparsely, and the renderer must reproduce
// exactly those breaks and no others.
func TestTextParagraphBreak(t *testing.T) {
	tr := &Transcript{Segments: []Segment{{Words: []Word{
		{Text: "one", Paragraph: true},
		{Text: "two"},
		{Text: "three", Formatted: "three.", Paragraph: true},
	}}}}
	if got, want := tr.Text(), "one two\n\nthree."; got != want {
		t.Errorf("Text() = %q, want %q", got, want)
	}
}

// A file that has been through a transcoder has no transcription track at
// all, which is handled by the caller. An empty track, though, must decode to
// an empty transcript rather than an error.
func TestDecodeEmpty(t *testing.T) {
	tr, err := DecodeTranscript(nil)
	if err != nil {
		t.Fatalf("DecodeTranscript(nil): %v", err)
	}
	if len(tr.Segments) != 0 {
		t.Errorf("segments = %d, want 0", len(tr.Segments))
	}
}

// Corrupt input must be reported, not silently truncated into a short
// transcript that looks plausible.
func TestDecodeTruncated(t *testing.T) {
	full := golden(t)
	for _, n := range []int{2, 6, 100, len(full) - 1} {
		if _, err := DecodeTranscript(full[:n]); err == nil {
			t.Errorf("DecodeTranscript(truncated to %d bytes) succeeded, want error", n)
		}
	}
}

// The format is not ours, so an added field must not break decoding: a future
// recorder version should cost us nothing.
func TestUnknownFieldsIgnored(t *testing.T) {
	// A word with an unknown field 9 (varint) and unknown field 10 (bytes),
	// wrapped in a segment, wrapped in the length prefix.
	word := []byte{
		0x0a, 0x03, 'f', 'o', 'o', // 1: text "foo"
		0x18, 0x64, // 3: start 100
		0x20, 0xc8, 0x01, // 4: end 200
		0x48, 0x2a, // 9: varint 42       (unknown)
		0x52, 0x02, 'x', 'y', // 10: bytes "xy"     (unknown)
	}
	seg := append([]byte{0x0a, byte(len(word))}, word...)
	seg = append(seg, 0x1a, 0x05, 'e', 'n', '-', 'U', 'S') // 3: language
	stream := append([]byte{0, 0, 0, byte(len(seg))}, seg...)

	tr, err := DecodeTranscript(stream)
	if err != nil {
		t.Fatalf("DecodeTranscript: %v", err)
	}
	if len(tr.Segments) != 1 || len(tr.Segments[0].Words) != 1 {
		t.Fatalf("got %d segments", len(tr.Segments))
	}
	w := tr.Segments[0].Words[0]
	if w.Text != "foo" || w.StartMs != 100 || w.EndMs != 200 {
		t.Errorf("word = %+v, want foo/100/200", w)
	}
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}