1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
|
package main
// tools.go declares the inventory tools exposed to the Gemini Live model and
// implements their runtime against SQLite. The set is intentionally small and
// generic (record/update/relate/search/request_photo) so the model organises
// the open entity graph itself. Add/remove tools here as real sessions reveal
// what's needed.
import (
"fmt"
"strconv"
"strings"
"time"
)
// inventoryTools is the tool set advertised in the Live setup message.
var inventoryTools = []geminiTool{
{FunctionDeclarations: []functionDeclaration{
{
Name: "record_entity",
Description: "Create a new inventory entity (an item, box, location, container, …) and " +
"return its id. Use search first to avoid duplicating existing containers/locations.",
Parameters: map[string]any{
"type": "object",
"properties": map[string]any{
"kind": map[string]any{"type": "string", "description": "Free-text kind: item, box, location, tote, …"},
"name": map[string]any{"type": "string", "description": "Human name of the entity"},
"notes": map[string]any{"type": "string", "description": "Optional free-text notes"},
"attrs": map[string]any{
"type": "object",
"description": "Optional key/value attributes, e.g. {\"quantity\":\"3\",\"colour\":\"red\"}",
},
"image_id": map[string]any{
"type": "string",
"description": "Optional id of a photo the user took (given to you as text " +
"with the photo, e.g. 'photo id A3F9'). Pass it to attach that full-resolution " +
"photo to this item. You don't need the camera still pointing at the item.",
},
},
"required": []string{"kind", "name"},
},
},
{
Name: "update_entity",
Description: "Update an existing entity's name, notes and/or attributes (attributes are merged).",
Parameters: map[string]any{
"type": "object",
"properties": map[string]any{
"id": map[string]any{"type": "integer", "description": "Entity id"},
"name": map[string]any{"type": "string"},
"notes": map[string]any{"type": "string"},
"attrs": map[string]any{"type": "object", "description": "Key/value attributes to set/merge"},
"image_id": map[string]any{
"type": "string",
"description": "Optional id of a photo the user took (given to you as text with the photo, e.g. 'photo id A3F9') to attach to this item.",
},
},
"required": []string{"id"},
},
},
{
Name: "relate",
Description: "Create a relation between two entities, e.g. an item contained_in a box, " +
"or a box located_at a location.",
Parameters: map[string]any{
"type": "object",
"properties": map[string]any{
"from_id": map[string]any{"type": "integer"},
"to_id": map[string]any{"type": "integer"},
"rel": map[string]any{"type": "string", "description": "Free-text relation: contained_in, located_at, part_of, …"},
},
"required": []string{"from_id", "to_id", "rel"},
},
},
{
Name: "search",
Description: "Find existing entities BEFORE creating new ones — always search first so " +
"you reuse a place/item that already exists instead of duplicating it. Each field " +
"is a list of substrings that are OR-combined within the field; different fields " +
"are AND-combined (all must match). Matching is case-insensitive substring. " +
"E.g. to find a cellar room: kind=[\"room\",\"location\"], name=[\"cellar\",\"keller\"]. " +
"To find a place to put things in, search by its kind (room, box, tote, shelf, " +
"location, …). Each result includes its attributes and its neighbours as " +
"relations_incoming and relations_outgoing (each a list of {id, name, kind, rel}, " +
"where rel is the relation verb such as located_at or contained_in). " +
"relations_incoming are entities pointing AT this one (e.g. items located in a " +
"room, things inside a box); relations_outgoing are entities this one points at " +
"(e.g. an item's room or box). To answer \"what's in/at X\", search for X and read " +
"its relations_incoming. Provide either at least one search field or `id`.",
Parameters: map[string]any{
"type": "object",
"properties": map[string]any{
"kind": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Match entity kind against ANY of these substrings (room, box, item, …)"},
"name": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Match entity name against ANY of these substrings"},
"notes": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Match entity notes against ANY of these substrings"},
"attr_key": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Match any attribute key against ANY of these substrings (e.g. color, brand)"},
"attr_value": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Match any attribute value against ANY of these substrings"},
"id": map[string]any{"type": "integer", "description": "Look up one entity by id"},
},
},
},
{
Name: "request_photo",
Description: "Ask the user's browser to capture and store an archival photo of an entity (e.g. to document an item).",
Parameters: map[string]any{
"type": "object",
"properties": map[string]any{
"entity_id": map[string]any{"type": "integer"},
"reason": map[string]any{"type": "string", "description": "Why a photo is useful"},
},
"required": []string{"entity_id"},
},
},
{
Name: "attach_photo",
Description: "Attach a photo the user took (identified by the id you were given as text " +
"with the photo) to an existing entity. Use this to add a photo to an item you " +
"recorded earlier.",
Parameters: map[string]any{
"type": "object",
"properties": map[string]any{
"entity_id": map[string]any{"type": "integer"},
"image_id": map[string]any{"type": "string", "description": "The photo id you were given as text (e.g. A3F9)"},
},
"required": []string{"entity_id", "image_id"},
},
},
{
Name: "delete_entity",
Description: "Permanently delete an entity and everything attached to it (its " +
"attributes, relations and photos). Use this when an item was recorded by " +
"mistake or no longer exists. Prefer update_entity to correct details rather " +
"than deleting. If the entity contains other entities (is a box/container/" +
"location with things related to it), do NOT delete it without first warning " +
"the user out loud and getting their confirmation; set confirmed=true only " +
"after they agree. Deleting a container removes the containment links but " +
"keeps the contained items.",
Parameters: map[string]any{
"type": "object",
"properties": map[string]any{
"id": map[string]any{"type": "integer"},
"reason": map[string]any{"type": "string", "description": "Why it is being deleted"},
"confirmed": map[string]any{"type": "boolean", "description": "Set true once the user has confirmed deleting a container that still holds items"},
},
"required": []string{"id"},
},
},
{
Name: "note_ignored_speaker",
Description: "Call this when you hear a voice that is clearly different from the " +
"primary user (fainter, further away, or a different person) and you are " +
"therefore ignoring it — not recording or acting on what it said. This surfaces " +
"the decision to the user. Call it once per distinct interruption, not for " +
"every audio buffer.",
Parameters: map[string]any{
"type": "object",
"properties": map[string]any{
"voice_characteristic": map[string]any{
"type": "string",
"description": "Short description of the ignored voice so the user knows " +
"who/what was ignored, e.g. \"deeper male voice\", \"distant background " +
"voice\", \"child's voice\", \"second person / TV\".",
},
"heard": map[string]any{
"type": "string",
"description": "Optional brief paraphrase of what that voice said, for context.",
},
},
"required": []string{"voice_characteristic"},
},
},
}},
}
// argString / argInt extract typed args from the model's JSON (numbers arrive
// as float64).
func argString(args map[string]any, key string) string {
if v, ok := args[key].(string); ok {
return strings.TrimSpace(v)
}
return ""
}
func argInt(args map[string]any, key string) int64 {
switch v := args[key].(type) {
case float64:
return int64(v)
case int64:
return v
case int:
return int64(v)
case string:
return atoi64(v)
}
return 0
}
// argStrings extracts a list of trimmed, non-empty strings for key. It accepts a
// JSON array (arriving as []any) and, leniently, a single string.
func argStrings(args map[string]any, key string) []string {
var out []string
switch v := args[key].(type) {
case []any:
for _, e := range v {
if s, ok := e.(string); ok {
if s = strings.TrimSpace(s); s != "" {
out = append(out, s)
}
}
}
case []string:
for _, s := range v {
if s = strings.TrimSpace(s); s != "" {
out = append(out, s)
}
}
case string:
if s := strings.TrimSpace(v); s != "" {
out = append(out, s)
}
}
return out
}
func argBool(args map[string]any, key string) bool {
switch v := args[key].(type) {
case bool:
return v
case string:
return v == "true" || v == "1"
}
return false
}
func argAttrs(args map[string]any) map[string]string {
out := map[string]string{}
m, ok := args["attrs"].(map[string]any)
if !ok {
return out
}
for k, v := range m {
out[strings.TrimSpace(k)] = strings.TrimSpace(fmt.Sprintf("%v", v))
}
return out
}
// dispatchInventoryTool runs one tool call. It returns:
// - result: the map sent back to Gemini as the functionResponse,
// - event: a short human-readable line for the browser (may be ""),
// - browserAction: an optional extra envelope to send the browser (e.g. a
// request_photo prompt), or nil.
func (s *server) dispatchInventoryTool(call functionCall, pending *pendingPhotos) (result map[string]any, event string, browserAction map[string]any) {
switch call.Name {
case "record_entity":
return s.toolRecordEntity(call.Args, pending)
case "update_entity":
return s.toolUpdateEntity(call.Args, pending)
case "relate":
return s.toolRelate(call.Args)
case "search":
return s.toolSearch(call.Args)
case "request_photo":
return s.toolRequestPhoto(call.Args)
case "attach_photo":
return s.toolAttachPhoto(call.Args, pending)
case "delete_entity":
return s.toolDeleteEntity(call.Args)
case "note_ignored_speaker":
return s.toolNoteIgnoredSpeaker(call.Args)
default:
return map[string]any{"error": "unknown tool: " + call.Name}, "", nil
}
}
// attachPendingPhoto pops the pending full-res photo for id (upper-cased) and
// stores it as an archival blob on entityID. Returns a short status string for
// inclusion in a tool result, and whether it attached.
func (s *server) attachPendingPhoto(pending *pendingPhotos, entityID int64, id string) (string, bool) {
if pending == nil || id == "" {
dbg("attachPendingPhoto: empty id or nil pending (id=%q)", id)
return "", false
}
id = strings.ToUpper(strings.TrimSpace(id))
// peek (not pop): keep the photo in the buffer so it can be attached to more
// than one entity and survives model retries within the session.
blob, ok := pending.peek(id)
if !ok {
dbg("attachPendingPhoto: id %q not in pending (have: %v)", id, pending.ids())
return fmt.Sprintf("photo %q not found (it may have expired or the id was misread)", id), false
}
mustExec(s.db, `INSERT INTO photos (entity_id, blob, mime, caption, created_at) VALUES (?,?,?,?,?)`,
entityID, blob, "image/jpeg", "", time.Now().Unix())
dbg("attachPendingPhoto: attached photo %s (%d bytes) to entity %d", id, len(blob), entityID)
return fmt.Sprintf("photo %s attached", id), true
}
func (s *server) toolRecordEntity(args map[string]any, pending *pendingPhotos) (map[string]any, string, map[string]any) {
kind := argString(args, "kind")
if kind == "" {
kind = "item"
}
name := argString(args, "name")
if name == "" {
return map[string]any{"error": "name is required"}, "", nil
}
notes := argString(args, "notes")
now := time.Now().Unix()
res := mustExec(s.db,
`INSERT INTO entities (kind, name, notes, created_at, updated_at) VALUES (?,?,?,?,?)`,
kind, name, notes, now, now)
id, _ := res.LastInsertId()
for k, v := range argAttrs(args) {
if k == "" {
continue
}
mustExec(s.db, `INSERT OR REPLACE INTO entity_attrs (entity_id, key, value) VALUES (?,?,?)`, id, k, v)
}
result := map[string]any{"id": id, "kind": kind, "name": name}
event := fmt.Sprintf("➕ %s: %s (#%d)", kind, name, id)
if photoID := argString(args, "image_id"); photoID != "" {
if status, ok := s.attachPendingPhoto(pending, id, photoID); ok {
result["photo"] = status
event += " 📷"
} else if status != "" {
result["photo"] = status
}
}
return result, event, nil
}
func (s *server) toolUpdateEntity(args map[string]any, pending *pendingPhotos) (map[string]any, string, map[string]any) {
id := argInt(args, "id")
if id == 0 {
return map[string]any{"error": "id is required"}, "", nil
}
e, err := loadEntity(s.db, id)
if err != nil {
return map[string]any{"error": "unknown entity"}, "", nil
}
name := e.Name
if n := argString(args, "name"); n != "" {
name = n
}
notes := e.Notes
if _, ok := args["notes"]; ok {
notes = argString(args, "notes")
}
mustExec(s.db, `UPDATE entities SET name=?, notes=?, updated_at=? WHERE id=?`,
name, notes, time.Now().Unix(), id)
for k, v := range argAttrs(args) {
if k == "" {
continue
}
mustExec(s.db, `INSERT OR REPLACE INTO entity_attrs (entity_id, key, value) VALUES (?,?,?)`, id, k, v)
}
result := map[string]any{"id": id, "name": name}
event := fmt.Sprintf("✏️ updated %s (#%d)", name, id)
if photoID := argString(args, "image_id"); photoID != "" {
if status, ok := s.attachPendingPhoto(pending, id, photoID); ok {
result["photo"] = status
event += " 📷"
} else if status != "" {
result["photo"] = status
}
}
return result, event, nil
}
// toolAttachPhoto attaches a pending photo (by its id) to an existing
// entity, for after-the-fact attachment.
func (s *server) toolAttachPhoto(args map[string]any, pending *pendingPhotos) (map[string]any, string, map[string]any) {
entityID := argInt(args, "entity_id")
photoID := argString(args, "image_id")
if entityID == 0 || photoID == "" {
return map[string]any{"error": "entity_id and image_id are required"}, "", nil
}
var name string
if err := s.db.QueryRow(`SELECT name FROM entities WHERE id=?`, entityID).Scan(&name); err != nil {
return map[string]any{"error": "unknown entity"}, "", nil
}
status, ok := s.attachPendingPhoto(pending, entityID, photoID)
if !ok {
return map[string]any{"error": status}, "", nil
}
return map[string]any{"ok": true, "note": status},
fmt.Sprintf("📷 %s → %s (#%d)", strings.ToUpper(photoID), name, entityID), nil
}
func (s *server) toolRelate(args map[string]any) (map[string]any, string, map[string]any) {
from := argInt(args, "from_id")
to := argInt(args, "to_id")
rel := argString(args, "rel")
if from == 0 || to == 0 || rel == "" {
return map[string]any{"error": "from_id, to_id and rel are required"}, "", nil
}
if from == to {
return map[string]any{"error": "cannot relate an entity to itself"}, "", nil
}
// verify both exist
for _, id := range []int64{from, to} {
var n int
s.db.QueryRow(`SELECT COUNT(*) FROM entities WHERE id=?`, id).Scan(&n)
if n == 0 {
return map[string]any{"error": fmt.Sprintf("entity %d does not exist", id)}, "", nil
}
}
mustExec(s.db, `INSERT INTO relations (from_id, to_id, rel) VALUES (?,?,?)`, from, to, rel)
return map[string]any{"ok": true},
fmt.Sprintf("🔗 #%d %s #%d", from, rel, to), nil
}
func (s *server) toolSearch(args map[string]any) (map[string]any, string, map[string]any) {
id := argInt(args, "id")
crit := searchCriteria{
Kind: argStrings(args, "kind"),
Name: argStrings(args, "name"),
Notes: argStrings(args, "notes"),
AttrKey: argStrings(args, "attr_key"),
AttrValue: argStrings(args, "attr_value"),
}
if id == 0 && crit.empty() {
return map[string]any{"error": "provide at least one search field (kind, name, notes, attr_key, attr_value) or id"}, "", nil
}
ents, total, capped, err := searchEntitiesGraph(s.db, crit, id)
if err != nil {
return map[string]any{"error": "search failed"}, "", nil
}
// `total` is the exact match count (or the window size when capped, meaning
// more exist). `ents` already holds only the detailed top results.
result := map[string]any{"results": ents, "count": total}
if capped {
result["capped"] = true // "count" is a floor; more matches exist
}
if len(ents) < total {
result["shown"] = len(ents) // results were truncated to the top matches
}
return result, formatSearchEvent(crit, id, total, capped), nil
}
// formatSearchEvent renders a capture-log line summarising a search: every
// non-empty criterion (or the id) plus the result count, e.g.
// "🔍 search kind=[room,location] name=[cellar] → 2 results" or, when the count
// hit the window, "→ 50+ results".
func formatSearchEvent(crit searchCriteria, id int64, total int, capped bool) string {
var parts []string
field := func(name string, terms []string) {
if len(terms) > 0 {
parts = append(parts, fmt.Sprintf("%s=[%s]", name, strings.Join(terms, ",")))
}
}
if id > 0 {
parts = append(parts, fmt.Sprintf("id=%d", id))
}
field("kind", crit.Kind)
field("name", crit.Name)
field("notes", crit.Notes)
field("attr_key", crit.AttrKey)
field("attr_value", crit.AttrValue)
count := strconv.Itoa(total)
if capped {
count += "+"
}
return fmt.Sprintf("🔍 search %s → %s result%s", strings.Join(parts, " "), count, plural(total))
}
func (s *server) toolRequestPhoto(args map[string]any) (map[string]any, string, map[string]any) {
id := argInt(args, "entity_id")
if id == 0 {
return map[string]any{"error": "entity_id is required"}, "", nil
}
var name string
if err := s.db.QueryRow(`SELECT name FROM entities WHERE id=?`, id).Scan(&name); err != nil {
return map[string]any{"error": "unknown entity"}, "", nil
}
reason := argString(args, "reason")
action := map[string]any{"type": "request_photo", "entity_id": id, "name": name, "reason": reason}
return map[string]any{"ok": true, "note": "asked the browser to capture a photo"},
fmt.Sprintf("📸 requested photo of %s (#%d)", name, id), action
}
// toolDeleteEntity hard-deletes an entity. Its attributes, relations and photos
// are removed by ON DELETE CASCADE. If the entity still contains other entities
// (something is related to it as its container/location) we require an explicit
// confirmed=true, guarding against accidentally dropping a populated box.
func (s *server) toolDeleteEntity(args map[string]any) (map[string]any, string, map[string]any) {
id := argInt(args, "id")
if id == 0 {
return map[string]any{"error": "id is required"}, "", nil
}
var name, kind string
if err := s.db.QueryRow(`SELECT name, kind FROM entities WHERE id=?`, id).Scan(&name, &kind); err != nil {
return map[string]any{"error": "unknown entity"}, "", nil
}
// Count entities that point at this one (i.e. it is their container/location).
var contains int
s.db.QueryRow(`SELECT COUNT(*) FROM relations WHERE to_id=?`, id).Scan(&contains)
if contains > 0 && !argBool(args, "confirmed") {
// Collect a few names to help the model warn the user precisely.
rows, _ := s.db.Query(`
SELECT e.name FROM relations r JOIN entities e ON e.id = r.from_id
WHERE r.to_id=? LIMIT 10`, id)
var names []string
if rows != nil {
for rows.Next() {
var n string
if rows.Scan(&n) == nil {
names = append(names, n)
}
}
rows.Close()
}
return map[string]any{
"error": "needs_confirmation",
"contains_count": contains,
"contains_sample": names,
"note": fmt.Sprintf("%q (#%d) still has %d related entities. Warn the user "+
"and call again with confirmed=true if they agree. The contained items "+
"themselves are kept; only the links are removed.", name, id, contains),
},
fmt.Sprintf("❓ delete %s (#%d)? still holds %d item%s — needs confirmation", name, id, contains, plural(contains)),
nil
}
mustExec(s.db, `DELETE FROM entities WHERE id=?`, id)
return map[string]any{"ok": true, "deleted_id": id},
fmt.Sprintf("🗑️ deleted %s: %s (#%d)", kind, name, id), nil
}
// toolNoteIgnoredSpeaker records that the model heard and ignored a non-primary
// voice. It writes nothing to the DB — its only effect is a human-readable event
// in the capture log (and the journal) so the user can see the model chose to
// ignore someone. The API has no speaker identity, so this is the model's own
// acoustic judgement made visible; voice_characteristic (and optional heard)
// give the note enough detail to be useful.
func (s *server) toolNoteIgnoredSpeaker(args map[string]any) (map[string]any, string, map[string]any) {
characteristic := argString(args, "voice_characteristic")
if characteristic == "" {
characteristic = "another voice"
}
event := "🙉 ignored " + characteristic
if heard := argString(args, "heard"); heard != "" {
event += fmt.Sprintf(": %q", heard)
}
return map[string]any{"ok": true}, event, nil
}
|