1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
|
# Copyright 2019 Google LLC.
# SPDX-License-Identifier: Apache-2.0
#
# buildGo provides Nix functions to build Go packages in the style of Bazel's
# rules_go.
{ pkgs ? import <nixpkgs> { }
, ...
}:
let
inherit (builtins)
attrNames
baseNameOf
dirOf
elemAt
filter
listToAttrs
map
match
readDir
replaceStrings
toString;
inherit (pkgs) lib runCommand runCommandCC fetchFromGitHub protobuf symlinkJoin go;
goStdlib = buildStdlib go;
# Helper to recursively find all paths to gopkg attributes in a dependency
# Returns a list of dot-separated paths (empty string for root gopkg)
findGoPkgs = prefix: attrSet:
let
# Check if this level has gopkg
hasGoPkg = attrSet ? gopkg;
currentPkg = if hasGoPkg then [ prefix ] else [ ];
# Get all attribute names, filtering out reserved/special attributes
attrNames = builtins.filter
(name: name != "gopkg" && name != "goDeps" && name != "goImportPath")
(builtins.attrNames attrSet);
# Recursively check children (but not derivations)
childPkgs = lib.concatMap
(name:
let
val = attrSet.${name};
newPrefix = if prefix == "" then name else "${prefix}.${name}";
in
if (lib.isAttrs val && !lib.isDerivation val)
then findGoPkgs newPrefix val
else [ ])
attrNames;
in
currentPkg ++ childPkgs;
# Validate that a dependency has a gopkg attribute, providing helpful errors if not
validateDep = depIndex: dep:
if dep ? gopkg
then dep
else
let
# Find all available gopkg paths
availablePkgs = findGoPkgs "" dep;
# Get top-level attributes for additional context
topLevelAttrs = builtins.attrNames dep;
# Format the available packages list
pkgList = lib.concatMapStringsSep "\n "
(path: if path == "" then "(root package)" else path)
availablePkgs;
errorMsg = ''
Dependency at position ${toString depIndex} is missing 'gopkg' attribute.
${if availablePkgs == [ ] then ''
This dependency has no gopkg attributes at all.
Available attributes: ${lib.concatStringsSep ", " topLevelAttrs}
This might not be a valid buildGo.external result.
'' else if (builtins.length availablePkgs == 1 && builtins.head availablePkgs != "") then ''
This is a repository with only subpackages (no root package).
Use this instead:
${builtins.head availablePkgs}
'' else ''
This dependency contains multiple packages. Use one of these instead:
${pkgList}
Top-level attributes: ${lib.concatStringsSep ", " topLevelAttrs}
''}
'';
in
throw errorMsg;
# Helpers for low-level Go compiler invocations
spaceOut = lib.concatStringsSep " ";
includeDepSrc = dep: "-I ${dep}";
includeSources = deps: spaceOut (map includeDepSrc deps);
includeDepLib = dep: "-L ${dep}";
includeLibs = deps: spaceOut (map includeDepLib deps);
srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1;
srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}";
srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs);
# Collect all transitive dependencies (assumes deps already have gopkg attribute)
allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps)));
xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs));
# Add an `overrideGo` attribute to a function result that works
# similar to `overrideAttrs`, but is used specifically for the
# arguments passed to Go builders.
makeOverridable = f: orig: (f orig) // {
overrideGo = new: makeOverridable f (orig // (new orig));
};
buildStdlib = go: runCommandCC "go-stdlib-${go.version}"
{
nativeBuildInputs = [ go ];
} ''
HOME=$NIX_BUILD_TOP/home
mkdir $HOME
goroot="$(go env GOROOT)"
cp -R "$goroot/src" "$goroot/pkg" .
chmod -R +w .
GODEBUG=installgoroot=all GOROOT=$NIX_BUILD_TOP go install -v --trimpath std
mkdir $out
cp -r pkg/*_*/* $out
find $out -name '*.a' | while read -r ARCHIVE_FULL; do
ARCHIVE="''${ARCHIVE_FULL#"$out/"}"
PACKAGE="''${ARCHIVE%.a}"
echo "packagefile $PACKAGE=$ARCHIVE_FULL"
done > $out/importcfg
'';
importcfgCmd = { name, deps, out ? "importcfg" }: ''
echo "# nix buildGo ${name}" > "${out}"
cat "${goStdlib}/importcfg" >> "${out}"
${lib.concatStringsSep "\n" (map (dep: ''
find "${dep}" -name '*.a' | while read -r pkgp; do
relpath="''${pkgp#"${dep}/"}"
pkgname="''${relpath%.a}"
echo "packagefile $pkgname=$pkgp"
done >> "${out}"
'') deps)}
'';
# High-level build functions
# Simple program builder without embed support (used for internal tools)
simpleProgram = { name, srcs, deps ? [ ], x_defs ? { } }:
let
# Validate dependencies first, then extract gopkg
validated = lib.imap1 validateDep deps;
uniqueDeps = allDeps (map (d: d.gopkg) validated);
in runCommand name { } ''
${importcfgCmd { inherit name; deps = uniqueDeps; }}
${go}/bin/go tool compile -o ${name}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p main ${includeSources uniqueDeps} ${spaceOut srcs}
mkdir -p $out/bin
export GOROOT_FINAL=go
${go}/bin/go tool link -o $out/bin/${name} -importcfg=importcfg -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a
'';
# Tool to generate embedcfg JSON from go list output
# Built with simpleProgram to avoid circular dependency
mkembedcfg = simpleProgram {
name = "mkembedcfg";
srcs = [ ./mkembedcfg/main.go ];
};
# Build a Go program out of the specified files and dependencies.
# Supports go:embed directives.
program = { name, srcs, deps ? [ ], x_defs ? { } }:
let
# Validate dependencies first, then extract gopkg
validated = lib.imap1 validateDep deps;
uniqueDeps = allDeps (map (d: d.gopkg) validated);
# Determine source directory for embedded files
srcDir = if srcs != [ ] then dirOf (builtins.head srcs) else ".";
# Generate embedcfg for go:embed support
# Note: go list needs access to embedded files, so we work in source directory
embedcfg = runCommand "${name}-embedcfg" {
nativeBuildInputs = [ go mkembedcfg ];
} ''
# Check if any source files contain go:embed directives
if grep -q "//go:embed" ${spaceOut srcs}; then
# Set HOME for go to use
export HOME=$NIX_BUILD_TOP/home
mkdir -p $HOME
# Create temporary module with symlink to source directory
mkdir -p tmpmod
cd tmpmod
echo "module tempmodule" > go.mod
echo "go 1.16" >> go.mod
# Symlink entire source directory so embedded files are accessible
ln -s ${srcDir} main
# Run go list to detect embed patterns and files
if ${go}/bin/go list -json ./main > golist.json 2>golist.err; then
${mkembedcfg}/bin/mkembedcfg -srcdir ${srcDir} < golist.json > $out
else
echo "go list failed:" >&2
cat golist.err >&2
echo '{"Patterns":{},"Files":{}}' > $out
fi
else
# No go:embed directives found, output empty config
echo '{"Patterns":{},"Files":{}}' > $out
fi
'';
in
runCommand name {
nativeBuildInputs = [ pkgs.jq ];
} ''
${importcfgCmd { inherit name; deps = uniqueDeps; }}
# Check if embedcfg has actual embeds and set flag accordingly
if [ "$(jq '.Patterns | length' < ${embedcfg})" -gt 0 ]; then
EMBED_FLAG="-embedcfg ${embedcfg}"
else
EMBED_FLAG=""
fi
${go}/bin/go tool compile $EMBED_FLAG -o ${name}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p main ${includeSources uniqueDeps} ${spaceOut srcs}
mkdir -p $out/bin
export GOROOT_FINAL=go
${go}/bin/go tool link -o $out/bin/${name} -importcfg=importcfg -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a
'';
# Build a Go library assembled out of the specified files.
#
# This outputs both the sources and compiled binary, as both are
# needed when downstream packages depend on it.
package = { name, srcs, deps ? [ ], path ? name, sfiles ? [ ] }:
let
# Validate dependencies first, then extract gopkg
validated = lib.imap1 validateDep deps;
uniqueDeps = allDeps (map (d: d.gopkg) validated);
# Determine source directory from first source file
# This is needed for resolving embedded file paths
srcDir = if srcs != [ ] then dirOf (builtins.head srcs) else ".";
# Generate embedcfg for go:embed support
# Uses go list to detect embed directives and mkembedcfg to transform to compiler format
embedcfg = runCommand "${name}-embedcfg" {
nativeBuildInputs = [ go mkembedcfg ];
} ''
# Check if any source files contain go:embed directives
if grep -q "//go:embed" ${spaceOut srcs}; then
# Set HOME for go to use
export HOME=$NIX_BUILD_TOP/home
mkdir -p $HOME
# Create temporary module with symlink to source directory
mkdir -p tmpmod
cd tmpmod
echo "module tempmodule" > go.mod
echo "go 1.16" >> go.mod
# Symlink entire source directory so embedded files are accessible
ln -s ${srcDir} pkg
# Run go list to detect embed patterns and files
if ${go}/bin/go list -json ./pkg > golist.json 2>golist.err; then
${mkembedcfg}/bin/mkembedcfg -srcdir ${srcDir} < golist.json > $out
else
echo "go list failed:" >&2
cat golist.err >&2
echo '{"Patterns":{},"Files":{}}' > $out
fi
else
# No go:embed directives found, output empty config
echo '{"Patterns":{},"Files":{}}' > $out
fi
'';
# The build steps below need to be executed conditionally for Go
# assembly if the analyser detected any *.s files.
#
# This is required for several popular packages (e.g. x/sys).
ifAsm = do: lib.optionalString (sfiles != [ ]) do;
asmBuild = ifAsm ''
${go}/bin/go tool asm -p ${path} -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles}
${go}/bin/go tool asm -p ${path} -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles}
'';
asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h";
asmPack = ifAsm ''
${go}/bin/go tool pack r $out/${path}.a ./asm.o
'';
gopkg = (runCommand "golib-${name}" {
nativeBuildInputs = [ pkgs.jq ];
} ''
export HOME=$NIX_BUILD_TOP/home
mkdir -p $out/${path}
${srcList path (map (s: "${s}") srcs)}
${asmBuild}
${importcfgCmd { inherit name; deps = uniqueDeps; }}
# Check if embedcfg has actual embeds and set flag accordingly
if [ "$(jq '.Patterns | length' < ${embedcfg})" -gt 0 ]; then
EMBED_FLAG="-embedcfg ${embedcfg}"
else
EMBED_FLAG=""
fi
${go}/bin/go tool compile -pack ${asmLink} $EMBED_FLAG -o $out/${path}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
${asmPack}
'').overrideAttrs (_: {
passthru = {
inherit gopkg;
goDeps = uniqueDeps;
goImportPath = path;
};
});
in
gopkg;
# Build a tree of Go libraries out of an external Go source
# directory that follows the standard Go layout and was not built
# with buildGo.nix.
#
# The derivation for each actual package will reside in an attribute
# named "gopkg", and an attribute named "gobin" for binaries.
external = import ./external { inherit pkgs program package; };
in
{
# Only the high-level builder functions are exposed, but made
# overrideable.
program = makeOverridable program;
package = makeOverridable package;
external = makeOverridable external;
# Internal tools
inherit mkembedcfg;
# re-expose the Go version used
inherit go;
}
|