1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

{
  lib,
  config,
  ...
}:

let
  garage-ephemeral-key = pkgs.callPackage ../../releng/garage-ephemeral-key {
    inherit (pkgs.writers) writePython3Bin;
  };

  pkgs = config.nodes.client.nixpkgs.pkgs;

  pkgA = pkgs.cowsay;
  pkgB = pkgs.hello;
in {
  name = "s3-cache";

  nodes.client = { pkgs, lib, ... }: {
    virtualisation.writableStore = true;
    virtualisation.additionalPaths = [ pkgA pkgB ];

    nix.settings.substituters = lib.mkForce [ ];
    nix.settings.experimental-features = [ "nix-command" ];

    environment.systemPackages = with pkgs; [
      awscli2
      brotli
    ];
  };

  nodes.s3 = { config, pkgs, ... }: {
    services.garage.enable = true;
    services.garage.package = pkgs.garage_2;
    services.garage.settings = {
      replication_factor = 1;

      rpc_bind_addr = "[::]:3901";
      rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6";

      s3_api.api_bind_addr = "[::]:3900";
      s3_api.s3_region = "garage";
      s3_api.root_domain = ".s3";

      admin = {
        api_bind_addr = "[::]:3903";
        admin_token = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4=";
      };
    };

    networking.firewall.allowedTCPPorts = [ 3900 ];

    environment.sessionVariables = {
      GARAGE_ADMIN_TOKEN = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4=";
    };

    environment.systemPackages = [
      config.services.garage.package

      pkgs.git
      pkgs.build-release-notes
      pkgs.jq
      pkgs.nix-eval-jobs
      garage-ephemeral-key
    ];
  };

  testScript = ''
    import json
    import textwrap

    start_all()

    client.wait_for_unit("multi-user.target")
    s3.wait_for_unit("garage")
    s3.wait_for_open_port(3900)

    def run_test_packages(fail=False):
        fun = client.succeed if not fail else client.fail
        fun("${lib.getExe pkgA} <<<awoo >&2")
        fun("${lib.getExe pkgB} >&2")

    def setup_s3():
        nodeId = s3.succeed("garage node id")
        s3.succeed(f"garage layout assign -z dc1 -c 10G {nodeId}")
        s3.succeed("garage layout apply --version 1")
        s3.succeed("garage bucket create cache")

        out = json.loads(
            s3.succeed("garage-ephemeral-key new --name cache --read --write --age-sec 7200 cache")
        )

        aws_config = textwrap.dedent(f"""
        [default]
        endpoint_url = http://s3:3900
        aws_access_key_id = {out['id']}
        aws_secret_access_key = {out['secret_key']}
        region = garage
        """)

        with open("aws-credentials", "w") as f:
            f.write(aws_config)

        client.copy_from_host("aws-credentials", "/root/.aws/credentials")

    setup_s3()

    with subtest("Ensure that communication with S3 works"):
        t.assertEqual("", client.succeed("aws s3 ls s3://cache/").rstrip())

    STORE_URI = "s3://cache?write-nar-listing=1&ls-compression=br&compression=zstd&parallel-compression=true&region=garage&endpoint=s3:3900&scheme=http"

    with subtest("Copy packages to S3"):
        client.succeed(f"nix copy --to '{STORE_URI}' ${pkgA} ${pkgB}")

        # Cache isn't empty anymore.
        t.assertNotEqual("", client.succeed("aws s3 ls s3://cache/"))

    with subtest("Ensure narinfo, listing and nar exist"):
        for store_path in ["${baseNameOf pkgA}", "${baseNameOf pkgB}"]:
            hash_part = store_path.split("-")[0]

            client.succeed(f"aws s3 cp s3://cache/{hash_part}.narinfo .")

            narinfo = {}
            for line in client.succeed(f"cat {hash_part}.narinfo").strip().splitlines():
                key, value = line.strip().split(": ", 1)
                narinfo[key] = value

            t.assertEqual("zstd", narinfo["Compression"])
            t.assertEqual(f"/nix/store/{store_path}", narinfo["StorePath"])

            url = narinfo["URL"]
            client.succeed(f"aws s3 ls s3://cache/{url} >&2")

            client.succeed(f"aws s3 cp s3://cache/{hash_part}.ls .")
            listing = json.loads(client.succeed(f"cat {hash_part}.ls | brotli -d"))

            t.assertIn("root", listing)
            root = listing["root"]["entries"]
            t.assertIn("bin", root)
            t.assertEqual("directory", root["bin"]["type"])

    with subtest("Ensure that the path is substitutable"):
        run_test_packages()

        client.succeed("nix-store --delete ${pkgA}")
        client.succeed("nix-store --delete ${pkgB}")

        run_test_packages(fail=True)

        client.succeed(f"nix copy --from '{STORE_URI}' ${pkgA} ${pkgB} --no-check-sigs")

        run_test_packages()
  '';
}