1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
# NixOS VM test for hosty system mode (root portable services via portablectl).
#
# Run with:
#   nix-build users/Profpatsch/hosty/system-test.nix
#
# Two nodes, covering the two shapes an app can have:
#
#   machine — hosty-hello, a statically linked buildGo binary that packs no
#             closure. Covers attach/detach, the sandbox, FUSE, persistence.
#   cgo     — rss-parrot, a CGO binary dynamically linked against glibc at an
#             absolute store path. Covers the packed-closure path at *runtime*,
#             which a build alone cannot: under RootDirectory= with no host
#             /nix/store bound in, the image's own closure is the only thing
#             that can satisfy the ELF interpreter.
{ depot ? import ../../.. {}
, pkgs ? depot.third_party.nixpkgs or (import <nixpkgs> {})
}:

let
  hostyPkgs = import ./default.nix { inherit depot pkgs; };
  hosty = hostyPkgs.hosty;
  helloV1 = hostyPkgs.hosty-hello-v1;
  rssParrot = hostyPkgs.rss-parrot-hosty;
in
pkgs.testers.runNixOSTest {
  name = "hosty-system-attach";

  nodes.machine = { ... }: {
    # portablectl + systemd-portabled ship with pkgs.systemd; portabled is
    # D-Bus/socket-activated on demand, no explicit unit wiring needed.
    # patchelf and file inspect the image binary's linkage (self-containment).
    environment.systemPackages = [
      hosty pkgs.systemd pkgs.sqlite pkgs.curl pkgs.fuse pkgs.patchelf pkgs.file
    ];
    # go-fuse needs the setuid fusermount wrapper at /run/wrappers/bin.
    programs.fuse.userAllowOther = true;
    boot.kernelModules = [ "fuse" ];
    # Give the VM enough room to extract the image tree.
    virtualisation.diskSize = 2048;
    virtualisation.memorySize = 1024;
  };

  # Second node: a CGO app, configured through the NixOS module rather than by
  # repeating its settings. That makes the module itself part of what this test
  # covers — if it stops providing hosty, allow_other or the state dir, this
  # node fails, which is the same way legosi would fail.
  nodes.cgo = { ... }: {
    imports = [ ./nixos-module.nix ];
    _module.args.depot = depot;
    profpatsch.hosty.enable = true;
    # sqlite is for inspecting the environment database directly: asserting on
    # `hosty info` output alone would only prove hosty is self-consistent, not
    # that the rows are actually there.
    environment.systemPackages = [ pkgs.curl pkgs.patchelf pkgs.file pkgs.sqlite ];
    # rss-parrot's image is ~80MB extracted on top of its packed closure.
    virtualisation.diskSize = 4096;
    virtualisation.memorySize = 2048;
  };

  testScript = ''
    machine.wait_for_unit("multi-user.target")

    # Read attachment state from `portablectl list`, never from
    # `portablectl is-attached`. The latter resolves its argument loosely and
    # reports some other image's state when the one asked about is absent —
    # it answers "running-runtime" even for a path that does not exist. A test
    # built on it silently passes for the wrong reason as soon as a second app
    # is attached. See isAttached() in main.go.
    def portable_state(node, name: str) -> str:
        for line in node.succeed("portablectl list --no-legend").splitlines():
            fields = line.split()
            if fields and fields[0] == name:
                return fields[-1]
        return "detached"

    def assert_detached() -> None:
        state = portable_state(machine, "hosty-hello")
        assert state == "detached", f"expected detached, got {state!r}"
        # No stale unit symlinks may be left behind in the attach directory.
        leftovers = machine.succeed(
            "ls -1 /run/systemd/system.attached/ 2>/dev/null | grep hosty-hello || true"
        ).strip()
        assert not leftovers, f"stale attached units: {leftovers!r}"

    # Sanity: portablectl is available.
    machine.succeed("portablectl --version")

    # Confirm the state root's mount is visible in the host namespace
    # (default shared propagation on NixOS); FUSE bind relies on this later.
    machine.succeed("mkdir -p /var/lib/hosty")
    print(machine.succeed("findmnt -no PROPAGATION -T /var/lib/hosty || true"))

    # Regression: .hosty files normally live on read-only media (the Nix store).
    # Opening one must not try to switch it to WAL, which would need to create a
    # -shm sidecar next to it and fail with SQLITE_CANTOPEN. `hosty pack` reads the
    # source file the same way `start` does, so a plain read is enough to cover it.
    machine.succeed("test ! -w ${helloV1}")
    machine.succeed("test ! -w $(dirname ${helloV1})")

    # Attach hosty-hello as a root portable service.
    machine.succeed("hosty start -system -file ${helloV1} -name hosty-hello 2>&1 | tee /tmp/start.log")

    # The portable image should be attached and the unit running.
    assert portable_state(machine, "hosty-hello") != "detached"
    machine.wait_for_unit("hosty-hello.service")

    # Discover the assigned port and curl the service.
    port = machine.succeed(
        "hosty info -state-dir /var/lib/hosty -name hosty-hello | "
        "awk '/port:/ {print $2}'"
    ).strip()
    assert port, "no port assigned"
    machine.wait_until_succeeds(f"curl -sf http://127.0.0.1:{port}/ >/tmp/resp")
    machine.succeed("grep -q 'Visit #' /tmp/resp")

    # The app must actually be sandboxed: DynamicUser means it does not run as
    # root. (The whole point of system mode is containerising arbitrary apps.)
    uid = machine.succeed(
        "systemctl show hosty-hello.service -p MainPID --value | "
        "xargs -I{} ps -o user= -p {} | tr -d ' '"
    ).strip()
    print(f"app runs as user: {uid!r}")
    assert uid and uid != "root", f"app should not run as root, got {uid!r}"

    # HOSTY_FS must be writable from inside the sandbox (FUSE allow_other +
    # ReadWritePaths). hosty-hello appends to visits.log on each request.
    machine.succeed(f"curl -sf http://127.0.0.1:{port}/log >/tmp/log")
    machine.succeed("grep -q 'visit #' /tmp/log")

    # Data must persist across a stop/start cycle (guards against the app and the
    # host disagreeing about where state lives).
    before = machine.succeed(f"curl -sf http://127.0.0.1:{port}/").strip()
    print(f"before restart: {before}")
    machine.succeed("hosty stop -system -name hosty-hello")
    assert_detached()

    machine.succeed("hosty start -system -file ${helloV1} -name hosty-hello")
    machine.wait_for_unit("hosty-hello.service")
    port2 = machine.succeed(
        "hosty info -state-dir /var/lib/hosty -name hosty-hello | "
        "awk '/port:/ {print $2}'"
    ).strip()
    machine.wait_until_succeeds(f"curl -sf http://127.0.0.1:{port2}/ >/tmp/resp2")
    after = machine.succeed("cat /tmp/resp2").strip()
    print(f"after restart: {after}")

    # Visit counter lives in data.hosty; it must have advanced, not reset to 1.
    import re

    def visit_count(s: str) -> int:
        m = re.search(r"Visit #(\d+)", s)
        assert m is not None, f"no visit count in response: {s!r}"
        return int(m.group(1))

    n_before = visit_count(before)
    n_after = visit_count(after)
    assert n_after > n_before, f"visit count did not persist: {n_before} -> {n_after}"

    # HOSTY_ROOT is how an app reaches files inside its own image. Under
    # RootDirectory= the image *is* the root, so it must be empty here: an app
    # joining it with an image-absolute path must get that path back unchanged.
    dropin = machine.succeed(
        "cat /run/systemd/system.attached/hosty-hello.service.d/50-hosty.conf"
    )
    assert "Environment=HOSTY_ROOT=\n" in dropin, dropin

    # Config values are quoted. systemd splits an unquoted Environment= value on
    # whitespace and drops everything after the first word, so hosty-hello's
    # default greeting used to reach the app as just "Hello".
    assert 'Environment=GREETING="Hello from hosty!"' in dropin, dropin
    machine.succeed(f"curl -sf http://127.0.0.1:{port2}/ | grep -q 'Hello from hosty!'")

    # ---------------------------------------------------------------------
    # Self-containment: the image must carry everything the app needs.
    #
    # hosty used to put BindReadOnlyPaths=-/nix/store in the drop-in, because
    # Nix-built binaries are dynamically linked against absolute store paths.
    # Images now pack their own closure instead, so the host store must not be
    # bound in — and the app must still run.
    #
    # The load-bearing assertion is the negative one: without it this whole
    # section would pass just as happily on a host store that happened to have
    # the right paths, i.e. for entirely the wrong reason.
    # ---------------------------------------------------------------------
    dropin_sc = machine.succeed(
        "cat /run/systemd/system.attached/hosty-hello.service.d/50-hosty.conf"
    )
    assert "/nix/store" not in dropin_sc, (
        f"drop-in must not bind the host store:\n{dropin_sc}"
    )

    # The app is running right now, so prove the store really is absent from
    # *its* namespace rather than merely unmentioned in the unit.
    #
    # This is read from outside via /proc/<pid>/root rather than by entering
    # the namespace: there is no shell inside a self-contained image to run,
    # which is rather the point. (An earlier version used `nsenter -- sh` and
    # failed with "failed to execute sh: No such file or directory" — a real
    # signal, but not the one being tested here.)
    pid = machine.succeed(
        "systemctl show hosty-hello.service -p MainPID --value"
    ).strip()

    # The host store must not be visible in the app's namespace. An app that
    # packs a closure has its own /nix/store there (fewer entries than the
    # host's); hosty-hello is statically linked and packs nothing, so it has no
    # /nix/store at all. Both are correct — what must never happen is seeing
    # everything the host has.
    host_count = int(machine.succeed("ls -1 /nix/store | wc -l").strip())
    # `|| true`, not just 2>/dev/null: with no /nix/store in the namespace ls
    # exits non-zero and machine.succeed would fail on the pipeline's status
    # rather than report the count of zero we are after.
    ns_count = int(machine.succeed(
        f"ls -1 /proc/{pid}/root/nix/store 2>/dev/null | wc -l || true"
    ).strip())
    print(f"store entries: host={host_count} app namespace={ns_count}")
    assert ns_count < host_count, (
        f"app sees {ns_count} store entries but host has {host_count}; "
        "the host store appears to be bound into the sandbox"
    )

    # hosty-hello specifically must need nothing outside the image: buildGo
    # links statically, so the binary has no interpreter to find. (patchelf
    # exits non-zero on a static binary, which is the expected outcome here.)
    machine.fail(
        "patchelf --print-interpreter "
        "/var/lib/hosty/images/hosty-hello/usr/bin/hosty"
    )
    filetype = machine.succeed(
        "file /var/lib/hosty/images/hosty-hello/usr/bin/hosty"
    ).strip()
    print(f"image binary: {filetype}")
    assert "statically linked" in filetype, filetype

    # Final teardown must be clean.
    machine.succeed("hosty stop -system -name hosty-hello")
    assert_detached()
    machine.fail("mountpoint -q /run/hosty/hosty-hello/fs")

    # A relative -state-dir used to be accepted and then produce a unit systemd
    # refuses to load ("WorkingDirectory= path is not absolute"), leaving a
    # half-installed app behind. It must now be rejected up front.
    err = machine.fail("hosty start -system -file ${helloV1} -name relcheck -state-dir rel/state 2>&1")
    assert "must be an absolute path" in err, err

    # App names must be valid DNS labels, because the same string is the state
    # directory, the unit name, PORTABLE_PREFIXES and (Phase 2) a subdomain
    # label. Rejection must happen before any state exists — the same
    # half-installed-app failure mode the relative -state-dir bug had, which is
    # why each case asserts the state dir was never created.
    for bad, reason in [
        ("MyApp", "lowercase"),        # would be one subdomain but two images
        ("bad_name", "only lowercase"),
        ("-lead", "start or end with a dash"),
        ("foo-fs", "-fs"),             # collides with the companion FUSE unit
        ("www", "reserved"),
    ]:
        err = machine.fail(
            f"hosty start -system -file ${helloV1} -name {bad} 2>&1"
        )
        assert reason in err, f"{bad}: {err}"
        machine.succeed(f"test ! -e /var/lib/hosty/{bad}")
        machine.succeed(f"test ! -e /var/lib/hosty/images/{bad}")
        # And nothing may have been registered with systemd either. `--` and
        # `grep -F --` are load-bearing: a name like "-lead" is otherwise
        # parsed as a flag, and the command then fails for the wrong reason,
        # making the assertion vacuous.
        machine.fail(
            f"systemctl list-unit-files --no-legend -- {bad}.service "
            f"| grep -qF -- {bad}"
        )

    # `hosty pack` must reject the same names, so a bad name cannot be baked
    # into an image in the first place.
    err = machine.fail("hosty pack -name Bad_Name -version 0.1 -out /tmp/bad.hosty 2>&1")
    assert "lowercase" in err, err
    machine.succeed("test ! -e /tmp/bad.hosty")

    # Regression guard for user mode. A full --user manager needs a login session,
    # which is awkward here, so assert the part this change could actually have
    # broken: `hosty pack` no longer ships a unit file, and user mode must still
    # synthesize a complete one (with a host-absolute ExecStart into the rootfs).
    machine.execute(
        "hosty start -file ${helloV1} -name hosty-hello-user "
        "-state-dir /root/.local/state/hosty 2>&1"
    )
    unit = machine.succeed("cat /root/.config/systemd/user/hosty-hello-user.service")
    print("=== synthesized user unit ===")
    print(unit)
    assert "ExecStart=/root/.local/state/hosty/hosty-hello-user/rootfs/usr/bin/hosty" in unit, unit
    assert "HOSTY_DB=" in unit and "HOSTY_PORT=" in unit and "HOSTY_FS=" in unit, unit
    # In user mode the image is an ordinary directory, so HOSTY_ROOT points at
    # it — the counterpart of the empty value asserted for system mode above.
    assert (
        "HOSTY_ROOT=/root/.local/state/hosty/hosty-hello-user/rootfs" in unit
    ), unit
    # And the packed image must indeed no longer carry a unit file.
    machine.succeed(
        "test -z \"$(find /root/.local/state/hosty/hosty-hello-user/rootfs "
        "-name '*.service' -print -quit)\""
    )

    # =====================================================================
    # CGO node: rss-parrot.
    #
    # Everything above runs a statically linked binary, which needs nothing
    # outside its image whether or not the closure machinery works. rss_parrot
    # is the opposite: buildGoModule with CGO_ENABLED=1, so it is dynamically
    # linked against an ELF interpreter at an absolute /nix/store path. Under
    # RootDirectory= the host store is not there, so the process can only reach
    # execve() if the image carries that path itself. Until now the packed
    # closure was only ever exercised by a build.
    # =====================================================================
    cgo.wait_for_unit("multi-user.target")

    # The NixOS module must have delivered its side of the bargain. These are
    # the exact prerequisites that were missing on legosi, so assert them here
    # rather than discovering them on the host.
    cgo.succeed("command -v hosty")
    cgo.succeed("grep -q '^user_allow_other' /etc/fuse.conf")
    cgo.succeed("test -d /var/lib/hosty")
    cgo.succeed("portablectl --version")

    # HOST is declared required, and rss-parrot cannot be started without it:
    # it is the domain baked into every ActivityPub URL, so there is no sane
    # default. The first start must therefore fail — but still install the app,
    # since `hosty config` needs a data.hosty to write into. That ordering is
    # load-bearing for every required-config app, so assert it rather than
    # working around it.
    err_rp = cgo.fail("hosty start -system -file ${rssParrot} -name rss-parrot 2>&1")
    assert "missing required config" in err_rp, err_rp
    assert "HOST" in err_rp, err_rp
    cgo.succeed("test -e /var/lib/hosty/rss-parrot/data.hosty")

    cgo.succeed("hosty config -system -name rss-parrot -set HOST=parrot.example.com")

    # The image binary is dynamically linked — the property that makes this
    # node worth having. If this ever reports "statically linked", the test has
    # silently stopped covering what it claims to.
    cgo.succeed("hosty start -system -name rss-parrot 2>&1 | tee /tmp/rp-start.log")
    interp = cgo.succeed(
        "patchelf --print-interpreter "
        "/var/lib/hosty/images/rss-parrot/usr/bin/rss_parrot"
    ).strip()
    print(f"rss_parrot interpreter: {interp}")
    assert interp.startswith("/nix/store/"), interp

    # ...and the image must carry that interpreter itself.
    cgo.succeed(f"test -e /var/lib/hosty/images/rss-parrot{interp}")

    cgo.wait_for_unit("rss-parrot.service")

    rp_port = cgo.succeed(
        "hosty info -state-dir /var/lib/hosty -name rss-parrot | "
        "awk '/port:/ {print $2}'"
    ).strip()
    assert rp_port, "no port assigned to rss-parrot"

    # Serving at all proves the dynamic loader found everything in the image.
    cgo.wait_until_succeeds(f"curl -sf http://127.0.0.1:{rp_port}/ >/tmp/rp-resp")

    # Sandboxed the same way as everything else.
    rp_user = cgo.succeed(
        "systemctl show rss-parrot.service -p MainPID --value | "
        "xargs -I{} ps -o user= -p {} | tr -d ' '"
    ).strip()
    print(f"rss-parrot runs as user: {rp_user!r}")
    assert rp_user and rp_user != "root", f"should not run as root, got {rp_user!r}"

    # The host store must not be visible. Unlike hosty-hello (which has no
    # /nix/store in its namespace at all) this app has its own, so the
    # assertion is that it is strictly smaller than the host's — it should
    # contain rss-parrot's closure and nothing else.
    rp_pid = cgo.succeed("systemctl show rss-parrot.service -p MainPID --value").strip()
    cgo_host_count = int(cgo.succeed("ls -1 /nix/store | wc -l").strip())
    cgo_ns_count = int(cgo.succeed(
        f"ls -1 /proc/{rp_pid}/root/nix/store 2>/dev/null | wc -l || true"
    ).strip())
    print(f"rss-parrot store entries: host={cgo_host_count} namespace={cgo_ns_count}")
    assert cgo_ns_count > 0, "a CGO app must have its closure in its namespace"
    assert cgo_ns_count < cgo_host_count, (
        f"app sees {cgo_ns_count} store entries but host has {cgo_host_count}; "
        "the host store appears to be bound into the sandbox"
    )

    # ---------------------------------------------------------------------
    # The environment database must be outside every app's reach.
    #
    # This is the bug that motivated moving port + secrets out of the per-app
    # runtime.hosty: systemd's StateDirectory= chowns the app's directory to
    # the app's own DynamicUser, so a file placed inside it belongs to the app.
    # runtime.hosty held the cleartext source for every sensitive value, at
    # mode 0600 — but owned by the app, which could therefore read it and,
    # being the owner, rewrite it. hosty would then re-materialise the altered
    # values as credentials on the next start.
    #
    # Asserting the mode alone would not catch that (0600 was already the
    # mode). What matters is *whose* 0600 it is, and whether the path is
    # reachable from inside the sandbox at all — so check both.
    # ---------------------------------------------------------------------
    env_owner = cgo.succeed("stat -c '%U %a' /var/lib/hosty/hosty.db").strip()
    assert env_owner == "root 600", (
        f"environment database is {env_owner!r}, want 'root 600' — "
        "it holds every app's secrets in cleartext"
    )

    # The app's own state directory is app-owned; that is expected and is
    # precisely why the environment database must not live inside it.
    app_dir_owner = cgo.succeed(
        "stat -c '%U' /var/lib/private/hosty/rss-parrot"
    ).strip()
    assert app_dir_owner != "root", (
        f"app state dir owned by {app_dir_owner!r}; the test's premise "
        "(StateDirectory= hands the directory to the app) no longer holds"
    )

    # And the file must not be visible inside the app's mount namespace at all.
    # Read from outside via /proc/<pid>/root: a self-contained image has no
    # shell to run inside it.
    rp_pid_env = cgo.succeed("systemctl show rss-parrot.service -p MainPID --value").strip()
    cgo.fail(f"test -e /proc/{rp_pid_env}/root/var/lib/hosty/hosty.db")

    # The app sees only its own subdirectory of the state dir, which is what
    # makes the root-owned database unreachable rather than merely unreadable.
    visible = cgo.succeed(
        f"ls -1 /proc/{rp_pid_env}/root/var/lib/hosty/ 2>/dev/null || true"
    ).split()
    assert visible == ["rss-parrot"], (
        f"app sees {visible!r} under the state dir, want only its own directory"
    )

    # The per-app runtime.hosty must be gone: leaving one behind would mean
    # secrets still sitting in app-owned territory.
    cgo.fail("test -e /var/lib/private/hosty/rss-parrot/runtime.hosty")

    # Heavy HOSTY_FS use: the wrapper generates an RSA keypair and writes
    # identity.json, config.json and secrets.json through FUSE on first start.
    # Read them back through the same mount to prove writes actually landed.
    fs_files = cgo.succeed("ls -1 /run/hosty/rss-parrot/fs").strip()
    print(f"HOSTY_FS contents:\n{fs_files}")
    for f in ["identity.json", "config.json", "secrets.json"]:
        assert f in fs_files, f"{f} missing from HOSTY_FS:\n{fs_files}"

    # The generated config must carry the configured HOST through to the
    # ActivityPub URLs; that is what makes the instance's identity.
    cfg = cgo.succeed("cat /run/hosty/rss-parrot/fs/config.json")
    assert '"host": "parrot.example.com"' in cfg, cfg

    # The identity is the one piece that must NEVER be regenerated: rss-parrot's
    # federated identity is its keypair, and remote servers cache the public
    # half. A restart that silently minted a new one would break federation in a
    # way no local check would notice.
    ident_before = cgo.succeed("cat /run/hosty/rss-parrot/fs/identity.json")
    cgo.succeed("systemctl restart rss-parrot.service")
    cgo.wait_for_unit("rss-parrot.service")
    cgo.wait_until_succeeds(f"curl -sf http://127.0.0.1:{rp_port}/ >/dev/null")
    ident_after = cgo.succeed("cat /run/hosty/rss-parrot/fs/identity.json")
    assert ident_before == ident_after, "birb identity was regenerated across restart"

    # A SECOND app must be running for the rest of this section to mean
    # anything. portablectl's name-based commands match loosely across all
    # attached images, so with only one app installed the bugs below are all
    # invisible — which is exactly why they survived until legosi, where
    # hosty-hello and rss-parrot ran side by side:
    #
    #   - `portablectl is-attached <name>` reported another image's state,
    #     so hosty believed a detached image was still attached;
    #   - detach then failed citing a completely unrelated app
    #     ("Unit file 'hosty-hello.service' is active, can't detach").
    #
    # Running two apps at once is the normal case for a host that does its job,
    # so the test has to model it.
    cgo.succeed("hosty start -system -file ${helloV1} -name hosty-hello 2>&1")
    cgo.wait_for_unit("hosty-hello.service")

    # Each app's attachment state must be reported independently.
    for app in ["rss-parrot", "hosty-hello"]:
        state = portable_state(cgo, app)
        assert state != "detached", f"{app} should be attached, got {state!r}"

    # Re-running `hosty start` on an app that is installed AND running must
    # work: that is the upgrade path, and the only way to apply changed config.
    # Note this now runs with hosty-hello also attached and active, which is
    # what made the detach fail before.
    #
    # It used to fail. setupSystemUnits detached with a bare `portablectl
    # detach`, which portabled refuses while the unit is still active; the
    # error was ignored, and the following attach then failed hard with
    # "Unit file 'rss-parrot.service' exists on the host already, refusing",
    # leaving the app running on its old configuration. cleanupSystem had the
    # correct stop-then-detach ordering all along — the two paths now share it.
    #
    # Asserting the app still serves afterwards is the point: a detach that
    # silently half-worked could leave it attached but broken.
    cgo.succeed("hosty start -system -name rss-parrot 2>&1")
    cgo.wait_for_unit("rss-parrot.service")
    cgo.wait_until_succeeds(f"curl -sf http://127.0.0.1:{rp_port}/ >/dev/null")

    # Installing under a name other than the packed one must work. `hosty pack`
    # bakes PORTABLE_PREFIXES=<packed name> into the image's os-release, while
    # the unit is synthesized at deploy time from the *install* name; when the
    # two disagreed, portablectl refused the attach outright:
    #
    #   Selected matches 'rss-parrot-throwaway' are not compatible with portable
    #   service image '...', refusing. (Acceptable prefix matches are: rss-parrot)
    #
    # Found on legosi rather than here, because every previous test installed an
    # app under its own packed name. -name is documented to override, so it has
    # to hold for a renamed install too.
    cgo.fail("hosty start -system -file ${rssParrot} -name renamed-parrot 2>&1")
    cgo.succeed("hosty config -system -name renamed-parrot -set HOST=renamed.example.com")
    cgo.succeed("hosty start -system -name renamed-parrot 2>&1")
    prefixes = cgo.succeed(
        "grep PORTABLE_PREFIXES /var/lib/hosty/images/renamed-parrot/usr/lib/os-release"
    ).strip()
    assert prefixes == "PORTABLE_PREFIXES=renamed-parrot", prefixes
    assert portable_state(cgo, "renamed-parrot") != "detached"
    renamed_port = cgo.succeed(
        "hosty info -state-dir /var/lib/hosty -name renamed-parrot | "
        "awk '/port:/ {print $2}'"
    ).strip()
    cgo.wait_until_succeeds(f"curl -sf http://127.0.0.1:{renamed_port}/ >/dev/null")

    # With three apps installed at once, the environment database is the only
    # place that can answer "is this port already claimed". It used to be
    # recorded solely in each app's own file, so nothing could see across apps:
    # freePort() asks the kernel, which answers "unused right now" rather than
    # "unclaimed by a hosty app that happens to be stopped".
    ports = cgo.succeed(
        "sqlite3 /var/lib/hosty/hosty.db "
        "\"select name || ' ' || coalesce(port, 'none') from _hosty_apps\""
    ).strip().splitlines()
    print("port registry:\n" + "\n".join(ports))
    assigned = [p.split()[1] for p in ports if p.split()[1] != "none"]
    assert len(assigned) == len(set(assigned)), f"duplicate port assignment: {ports}"
    # The renamed install is a distinct app and must hold its own row.
    assert any(p.startswith("renamed-parrot ") for p in ports), ports

    cgo.succeed("hosty stop -system -name renamed-parrot")

    # `hosty stop` without -remove-data keeps the app installed, so its entry
    # (port claim and secrets) must survive; -remove-data is what forgets it.
    # Previously secrets lived in the app directory and were removed only as a
    # side effect of deleting it, so this ordering is what preserves the
    # existing meaning of the flag.
    still_there = cgo.succeed(
        "sqlite3 /var/lib/hosty/hosty.db "
        "\"select count(*) from _hosty_apps where name = 'renamed-parrot'\""
    ).strip()
    assert still_there == "1", "stop without -remove-data forgot the app"

    cgo.succeed("hosty stop -system -name renamed-parrot -remove-data")
    gone = cgo.succeed(
        "sqlite3 /var/lib/hosty/hosty.db "
        "\"select count(*) from _hosty_apps where name = 'renamed-parrot'\""
    ).strip()
    assert gone == "0", "-remove-data left the app in the environment database"
    # ON DELETE CASCADE must take the secrets with it: a secret outliving its
    # app is exactly the leak the old per-app file had.
    orphans = cgo.succeed(
        "sqlite3 /var/lib/hosty/hosty.db "
        "\"select count(*) from _hosty_secrets where app = 'renamed-parrot'\""
    ).strip()
    assert orphans == "0", f"{orphans} secret(s) survived -remove-data"

    # Stopping one app must not disturb the other. With the loose name matching
    # above, tearing one down could detach or block the other.
    cgo.succeed("hosty stop -system -name rss-parrot")
    rp_state = portable_state(cgo, "rss-parrot")
    assert rp_state == "detached", f"expected detached, got {rp_state!r}"
    cgo.fail("mountpoint -q /run/hosty/rss-parrot/fs")

    # ...and the bystander is untouched and still serving.
    cgo.succeed("systemctl is-active hosty-hello.service")
    hello_state = portable_state(cgo, "hosty-hello")
    assert hello_state != "detached", hello_state

    # Final teardown of the second app must also come away clean.
    cgo.succeed("hosty stop -system -name hosty-hello")
    hello_state = portable_state(cgo, "hosty-hello")
    assert hello_state == "detached", f"expected detached, got {hello_state!r}"
    # No unit symlinks may survive anywhere under the attach directory —
    # neither the units themselves nor the default.target.wants enablement
    # links. These are what make a later attach fail with "Unit file
    # '<name>.service' exists on the host already, refusing", and portabled
    # leaves them behind even after a detach it reports as successful.
    #
    # The empty default.target.wants *directory* is portabled's own and does
    # persist, so match on service files rather than on directory entries.
    leftover = cgo.succeed(
        "find /run/systemd/system.attached -name '*.service' 2>/dev/null || true"
    ).strip()
    assert not leftover, f"stale attached units left behind: {leftover!r}"
  '';
}