1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
|
//go:build linux
package builder
// Parent-side spawning of the sandbox launcher.
//
// This re-execs the gonix binary as the "__builder" subcommand inside a fresh
// set of namespaces created via os/exec's SysProcAttr, hands the launcher a
// BuildRequest over the setup socket (fd 0), and waits for the build.
//
//
// ── How lix does it, and why we differ ───────────────────────────────────────
//
// lix runs the build as an unprivileged in-namespace uid (1000 for rootless
// single-uid builds) *while still holding full capabilities inside the user
// namespace*. Achieving that combination requires a three-stage dance
// (LinuxLocalDerivationGoal::startChild, lix/libstore/platform/linux.cc:1343):
//
// 1. NAMESPACE HOLDER. Clone a child with CLONE_NEWUSER (+ CLONE_NEWNET) whose
// only job is to park forever in a `kill(getpid(), SIGSTOP)` loop, keeping
// the freshly-created namespaces alive (linux.cc:1346).
//
// 2. PARENT WRITES THE ID MAPS. From the (privileged or unprivileged) parent,
// write the holder's /proc/<pid>/uid_map, /proc/<pid>/setgroups ("deny" for
// single-uid) and /proc/<pid>/gid_map (linux.cc:1372-1382):
//
// uid_map: "<sandboxUid> <hostUid> <nrIds>" e.g. "1000 <hostuid> 1"
// gid_map: "<sandboxGid> <hostGid> <nrIds>" e.g. "100 <hostgid> 1"
//
// Mapping the sandbox uid to 1000 (rather than 0) requires either being root
// (the daemon case) or, for rootless range maps, the setuid newuidmap/
// newgidmap helpers backed by /etc/subuid.
//
// 3. SETNS BRIDGE + EXEC. The parent vforks; the vfork child drops
// supplementary groups, then setns()'s into the holder's user (and net)
// namespace (linux.cc:1450-1467). It then clones again with
// CLONE_NEWPID|CLONE_NEWNS|CLONE_NEWIPC|CLONE_NEWUTS, raises *all*
// capabilities as ambient via raiseAmbientCaps() (linux.cc:205,1478) so they
// survive the upcoming execve despite the non-zero uid, and finally
// execl()'s launch-builder (linux.cc:1481). raiseAmbientCaps does a raw
// capset() setting permitted+inheritable for every cap, then
// prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_RAISE, cap) for each.
//
// The crucial trick is step 3's setns(CLONE_NEWUSER): joining an *existing*
// user namespace. The kernel only grants a user namespace's creator a full
// capability set when its uid maps to 0; lix sidesteps that by mapping to 1000
// and instead carrying caps in as ambient capabilities raised right before exec.
//
//
// ── Why we cannot replicate this in pure Go (CGO_ENABLED=0) ───────────────────
//
// setns(fd, CLONE_NEWUSER) requires the calling *process* to be single-threaded
// (see man 2 setns, EINVAL "the caller is multithreaded"). The Go runtime is
// multi-threaded from process start (GC, scheduler, sysmon), so any Go process
// that tries setns(CLONE_NEWUSER) gets EINVAL. The only way around it is to do
// the setns + clone + raiseAmbientCaps in a C constructor that runs *before* the
// Go runtime spins up threads — i.e. cgo — which we are deliberately avoiding.
//
// Empirically (verified while writing this):
// - SysProcAttr UidMappings {ContainerID:0, HostID:euid, Size:1}
// → child runs as uid 0, CapEff = all → sethostname/mount/pivot_root work.
// - SysProcAttr UidMappings {ContainerID:1000, HostID:euid, Size:1}
// → child runs as uid 1000, CapEff = 0 → sandbox setup syscalls EPERM.
// - A separate __ns-bridge process doing setns into a holder's userns
// → setns(user) returns EINVAL because the bridge is multithreaded Go.
// - setuid(1000) after mapping only {0→euid,1} → EINVAL (1000 is unmapped);
// multi-entry/subuid range maps via SysProcAttr are rejected for
// unprivileged callers (they need the newuidmap setuid helper).
//
//
// ── What we do instead ───────────────────────────────────────────────────────
//
// We collapse the whole thing into a single os/exec spawn: one re-exec of the
// __builder launcher with SysProcAttr.Cloneflags creating CLONE_NEWUSER plus the
// other namespaces in one shot, and UidMappings/GidMappings mapping the calling
// user to uid/gid 0 *inside* the namespace. The launcher therefore runs as uid 0
// within its own user namespace — which still maps to the unprivileged host euid,
// so it has zero real privilege on the host — and consequently holds the full
// in-namespace capability set needed for mount/pivot_root/sethostname.
//
// The only observable divergence from lix is that the build sees uid 0 instead
// of 1000. This is correctness-neutral for build *outputs*: nix canonicalises
// output ownership during registration regardless of the build uid (and nix's
// historical single-user mode ran builds as root anyway).
//
// Faithful uid-1000 behaviour can be added later via either:
// (a) a tiny cgo launcher whose constructor does lix's setns dance pre-runtime, or
// (b) the privileged/build-users model (run gonix as root, map a real build uid).
import (
"encoding/gob"
"errors"
"fmt"
"io"
"os"
"os/exec"
"syscall"
)
// SpawnConfig controls how the launcher child is created.
type SpawnConfig struct {
// SelfExe is the path to the gonix binary to re-exec (defaults to
// /proc/self/exe).
SelfExe string
// Unprivileged enables rootless mode: a new user namespace (CLONE_NEWUSER)
// is created and the calling user mapped to uid/gid 0 inside it (see the
// package-level rationale above). When false, no user namespace is created
// and the caller is assumed to be privileged (e.g. a daemon running as root).
Unprivileged bool
// UidMap/GidMap override the default single-entry {0→euid, size 1} mapping
// (only used when Unprivileged). Callers with newuidmap-backed subuid
// ranges can supply richer maps here.
UidMap []syscall.SysProcIDMap
GidMap []syscall.SysProcIDMap
// PrivateNetwork adds CLONE_NEWNET; the launcher then brings up loopback.
PrivateNetwork bool
// Stdout/Stderr receive the builder's output. If nil, os.Stdout/os.Stderr
// are used.
Stdout io.Writer
Stderr io.Writer
}
// cloneFlags returns the namespace clone flags for the launcher.
func (c *SpawnConfig) cloneFlags() uintptr {
flags := uintptr(syscall.CLONE_NEWNS |
syscall.CLONE_NEWPID |
syscall.CLONE_NEWUTS |
syscall.CLONE_NEWIPC)
if c.Unprivileged {
flags |= syscall.CLONE_NEWUSER
}
if c.PrivateNetwork {
flags |= syscall.CLONE_NEWNET
}
return flags
}
// Spawn runs the build described by req inside a sandbox. It blocks until the
// builder exits and returns its error (nil on exit 0), or a setup error if the
// sandbox could not be established.
func Spawn(req *BuildRequest, cfg *SpawnConfig) error {
if cfg == nil {
cfg = &SpawnConfig{}
}
selfExe := cfg.SelfExe
if selfExe == "" {
selfExe = "/proc/self/exe"
}
// Record the parent pid so the launcher can detect reparenting (it SIGKILLs
// itself if getppid() changes — i.e. we died).
if req.Linux != nil {
req.Linux.ParentPid = int32(os.Getpid())
}
// Setup socketpair: childFile becomes the launcher's fd 0 (stdin), over which
// it reads the BuildRequest and, on failure, writes a launcherError back.
fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
if err != nil {
return fmt.Errorf("creating setup socketpair: %w", err)
}
parentFile := os.NewFile(uintptr(fds[0]), "setup-parent")
childFile := os.NewFile(uintptr(fds[1]), "setup-child")
defer parentFile.Close()
stdout := cfg.Stdout
if stdout == nil {
stdout = os.Stdout
}
stderr := cfg.Stderr
if stderr == nil {
stderr = os.Stderr
}
cmd := exec.Command(selfExe, BuilderSubcommand)
cmd.Stdin = childFile
cmd.Stdout = stdout
cmd.Stderr = stderr
sysProc := &syscall.SysProcAttr{
Cloneflags: cfg.cloneFlags(),
Pdeathsig: syscall.SIGKILL,
}
if cfg.Unprivileged {
uidMap := cfg.UidMap
if uidMap == nil {
// Map the calling user to uid 0 inside the namespace so the launcher
// holds the full in-namespace capability set (mount/pivot_root/
// sethostname). See the package-level comment for why we use 0
// rather than lix's 1000.
uidMap = []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Geteuid(), Size: 1}}
}
gidMap := cfg.GidMap
if gidMap == nil {
gidMap = []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getegid(), Size: 1}}
}
sysProc.UidMappings = uidMap
sysProc.GidMappings = gidMap
sysProc.GidMappingsEnableSetgroups = false
}
cmd.SysProcAttr = sysProc
if err := cmd.Start(); err != nil {
childFile.Close()
return fmt.Errorf("starting launcher: %w", err)
}
// The child inherited childFile as fd 0; close our copy.
childFile.Close()
// Send the request.
if err := gob.NewEncoder(parentFile).Encode(req); err != nil {
cmd.Process.Kill()
cmd.Wait()
return fmt.Errorf("sending build request: %w", err)
}
// Wait for the setup handshake (EOF = success, launcherError = failure),
// then for the builder itself.
setupErr := readSetupResponse(parentFile)
waitErr := cmd.Wait()
if setupErr != nil {
return setupErr
}
return waitErr
}
// readSetupResponse reads the launcher's setup handshake from the setup socket.
// A successful setup closes the fd (EOF, because the launcher execve'd or
// rerouted fd 0); a failure sends a gob-encoded launcherError.
func readSetupResponse(r io.Reader) error {
dec := gob.NewDecoder(r)
var le launcherError
if err := dec.Decode(&le); err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return nil
}
// Any other decode error (e.g. reset by execve) → treat as success; the
// builder's exit status from Wait() is authoritative.
return nil
}
return fmt.Errorf("sandbox setup failed: %s", le.Message)
}
|