1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
# Go dependencies for fedisearch.
#
# fedisearch runs ONNX sentence-transformer models through GoMLX's pure-Go
# backend. That is a big closure — 111 packages across 13 modules — but it is
# entirely CGO-free, so the resulting binary is static and needs no ONNX
# Runtime shared object, no rust tokenizer archive, and no XLA plugin at
# runtime.
#
# Note on what is NOT here: github.com/knights-analytics/hugot. Hugot wraps
# exactly this stack, but pulls in viant/afs (18 packages of cloud-storage
# abstraction) and x/crypto (9, via its model downloader) for ~400 lines of
# glue that fedisearch owns directly. Dropping it removed 37 packages and 4
# modules.
#
# Two things about buildGo.external are worth knowing when editing this file:
#
# 1. It compiles *whole repositories*, but the resulting attribute tree is
# lazy (lib.fix over foldl' recursiveUpdate). Packages that nothing
# references are never realised, which is why gomlx's optional XLA
# backend — and with it go-xla, gonb and plotly — costs nothing here.
# 2. Go resolves imports per *package*, not per symbol. So gomlx's training
# stack (ml/train, optimizer, loss, metric) is in the closure purely
# because ml/layers/norm imports ml/train for BatchNorm. It is dead
# weight in the build graph, but the linker strips it from the binary.
{ depot, pkgs, ... }:
let
sharedDeps = import ../go-deps.nix { inherit depot pkgs; };
in
rec {
# Reused from the shared dependency set rather than pinned again here.
#
# x/text is at v0.36.0 there while go.mod resolves v0.40.0. Only transform
# and unicode/norm are used (by the tokenizer, for NFC normalization), and
# those APIs have been stable for years, so the older pin is taken rather
# than bumping a version that a dozen other packages in this tree depend on.
inherit (sharedDeps) golang-x-sys golang-x-sys-unix golang-x-exp golang-x-text google-uuid;
# ══════════════════════════════════════════════════════════
# Small leaf dependencies
# ══════════════════════════════════════════════════════════
# github.com/pkg/errors - error wrapping, used throughout gomlx
pkg-errors = depot.nix.buildGo.external {
path = "github.com/pkg/errors";
src = pkgs.fetchFromGitHub {
owner = "pkg";
repo = "errors";
rev = "v0.9.1";
sha256 = "1761pybhc2kqr6v5fm8faj08x9bql8427yqg6vnfv6nhrasx1mwq";
};
};
# github.com/gomlx/exceptions - panic/recover helpers used by onnx-gomlx
gomlx-exceptions = depot.nix.buildGo.external {
path = "github.com/gomlx/exceptions";
src = pkgs.fetchFromGitHub {
owner = "gomlx";
repo = "exceptions";
rev = "v0.0.3";
sha256 = "08ryhrdkrpc54g9ahp6npj9i951f26zjq4n5swyjblbrm9k3r5xp";
};
deps = [ pkg-errors ];
};
# github.com/go-logr/logr - logging interface required by klog
go-logr = depot.nix.buildGo.external {
path = "github.com/go-logr/logr";
src = pkgs.fetchFromGitHub {
owner = "go-logr";
repo = "logr";
rev = "v1.4.3";
sha256 = "1m9v04wkrbm89vhlzfm99vxg3lnjsqqsqp871zv3r2sa92x38a88";
};
};
# github.com/gofrs/flock - file locking, pulled in by go-huggingface's hub
gofrs-flock = depot.nix.buildGo.external {
path = "github.com/gofrs/flock";
src = pkgs.fetchFromGitHub {
owner = "gofrs";
repo = "flock";
rev = "v0.13.0";
sha256 = "18wggkw7x22x6aldcdvss292qqn9jpqj46jxa12njg83x30ymamp";
};
deps = [ golang-x-sys-unix ];
};
# k8s.io/klog/v2 - logging used by gomlx internals.
#
# Note the repo is kubernetes/klog on GitHub even though the import path is
# k8s.io/klog/v2; path must match the import, src is where it actually lives.
klog = depot.nix.buildGo.external {
path = "k8s.io/klog/v2";
src = pkgs.fetchFromGitHub {
owner = "kubernetes";
repo = "klog";
rev = "v2.140.0";
sha256 = "0jg7kckh6q204kj0cdg9yp32y224csfda60bdrbrs78m9j8y196q";
};
deps = [ go-logr ];
};
# ══════════════════════════════════════════════════════════
# Protobuf
# ══════════════════════════════════════════════════════════
# google.golang.org/protobuf - required because the .onnx file format *is*
# protobuf: onnx-gomlx/internal/protos is 4.5k lines of generated .pb.go.
#
# This is the one dependency that exercises buildGo's //go:embed support:
# internal/editiondefaults embeds editions_defaults.binpb. It works (verified)
# because buildGo.external passes source paths inside the unpacked tree, so
# buildGo.package's `dirOf (head srcs)` lands in the directory that also holds
# the .binpb, where the `go list` invocation can find it. No change to the
# analyser was needed, despite it not collecting EmbedFiles itself.
protobuf = depot.nix.buildGo.external {
path = "google.golang.org/protobuf";
src = pkgs.fetchFromGitHub {
owner = "protocolbuffers";
repo = "protobuf-go";
rev = "v1.36.11";
sha256 = "02v902sl3g325id0670h5s6ixwsxb7i3yfmhyfq00wa3jxzkgv7g";
};
};
# ══════════════════════════════════════════════════════════
# GoMLX: the inference stack
# ══════════════════════════════════════════════════════════
# github.com/gomlx/compute - tensor backend. The pure-Go implementation lives
# in internal/gobackend and is registered via an init() side effect, so
# consumers must blank-import compute/gobackend to make "go" available.
gomlx-compute = depot.nix.buildGo.external {
path = "github.com/gomlx/compute";
src = pkgs.fetchFromGitHub {
owner = "gomlx";
repo = "compute";
rev = "v0.1.2";
sha256 = "00pja00w8m8r3l7vjsx4vshfj1wwgr8a31450n53vq3j9rzbx8g3";
};
deps = [
pkg-errors
klog
golang-x-exp.constraints
];
};
# github.com/gomlx/gomlx - graph construction, tensors and NN layers.
gomlx = depot.nix.buildGo.external {
path = "github.com/gomlx/gomlx";
src = pkgs.fetchFromGitHub {
owner = "gomlx";
repo = "gomlx";
rev = "v0.28.2";
sha256 = "0ihawgibqiignwci7z07gn7871j7d4036bfrzsq89482m80plzf9";
};
deps = [
gomlx-compute
gomlx-compute.distributed
gomlx-compute.dtypes
gomlx-compute.dtypes.bfloat16
gomlx-compute.dtypes.float16
gomlx-compute.dtypes.gotype
gomlx-compute.shapes
gomlx-compute.support.envutil
gomlx-compute.support.humanize
gomlx-compute.support.xslices
pkg-errors
klog
google-uuid
golang-x-exp.constraints
];
};
# github.com/gomlx/onnx-gomlx - translates an ONNX graph into GoMLX ops.
onnx-gomlx = depot.nix.buildGo.external {
path = "github.com/gomlx/onnx-gomlx";
src = pkgs.fetchFromGitHub {
owner = "gomlx";
repo = "onnx-gomlx";
rev = "v0.5.2";
sha256 = "17cdczss7c75ri14xk12x54ird20bg4immpjhh01qkh15naqi9ip";
};
deps = [
gomlx.core.graph
gomlx.core.tensors
gomlx.core.tensors.images
gomlx.ml.layers.activation
gomlx.ml.layers.attention
gomlx.ml.layers.attention.pos
gomlx.ml.layers.lstm
gomlx.ml.layers.norm
gomlx.ml.model
gomlx.ml.nn
gomlx.support.sets
gomlx-compute
gomlx-compute.dtypes
gomlx-compute.dtypes.float16
gomlx-compute.dtypes.gotype
gomlx-compute.gobackend
gomlx-compute.shapes
gomlx-exceptions
protobuf.proto
protobuf.reflect.protoreflect
protobuf.runtime.protoimpl
pkg-errors
];
};
# github.com/gomlx/go-huggingface - used only for tokenizers/hftokenizer,
# which parses tokenizer.json. Its `hub` package (a model downloader) comes
# along because hftokenizer.New takes a *hub.Repo, even though fedisearch
# calls NewFromContent and downloads models itself.
go-huggingface = depot.nix.buildGo.external {
path = "github.com/gomlx/go-huggingface";
src = pkgs.fetchFromGitHub {
owner = "gomlx";
repo = "go-huggingface";
rev = "v0.4.1";
sha256 = "1khdfpblpvbx279xilq7a7n25lk4jbkyzrw65zrcd3bcviv2kb3d";
};
deps = [
gomlx-compute.support.humanize
pkg-errors
gofrs-flock
google-uuid
golang-x-text.unicode.norm
];
};
}
|