1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
|
#pragma once
///@file
#include "lix/libexpr/nixexpr.hh"
#include "lix/libexpr/pos-idx.hh"
#include "lix/libutil/logging.hh"
namespace nix::parser {
struct IndStringLine {
// String containing only the leading whitespace of the line. May be empty.
std::string_view indentation;
// Position of the line start (before the indentation)
PosIdx pos;
// Whether the line contains anything besides indentation and line break
bool hasContent = false;
std::vector<
std::pair<
PosIdx,
std::variant<std::unique_ptr<Expr>, std::string_view>
>
> parts = {};
};
struct State
{
NixSymbolTable & symbols;
PosTable & positions;
SourcePath basePath;
PosTable::Origin origin;
const FeatureSettings & featureSettings;
bool hasWarnedAboutBadLineEndings = false; // State to only warn on first occurrence
void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
void overridesFound(const PosIdx pos);
void badLineEndingFound(const PosIdx pos, bool warnOnly);
void badFirstLineIndStringFound(const PosIdx pos);
void badSingleLineIndStringFound(const PosIdx pos);
void badEscapeFound(const PosIdx pos, char found, bool isIndented);
void nulFound(const PosIdx pos);
void recSetMergeFound(const AttrPath & attrPath, const PosIdx pos);
void recSetDynamicAttrFound(const PosIdx pos);
void orIdentifierFound(const PosIdx pos);
void orArgumentFound(const PosIdx pos);
void whitespaceBetweenTokensRequired(const PosIdx pos);
void addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos);
void mergeAttrs(AttrPath & attrPath, ExprSet * source, ExprSet * target);
void validateLambdaAttrs(AttrsPattern & pattern, PosIdx pos = noPos);
std::unique_ptr<Expr> stripIndentation(const PosIdx pos, std::vector<IndStringLine> && line);
/* Creates an ExprVar or an ExprVarRoot depending on the feature settings.
* The symbol is synthetic, but for the purpose of error handling the pos is required
* and should point to the expression where the var is used
*/
inline std::unique_ptr<ExprVar> mkInternalVar(PosIdx pos, Symbol name);
// lazy positioning means we don't get byte offsets directly, in.position() would work
// but also requires line and column (which is expensive)
PosIdx at(const auto & in)
{
return positions.add(origin, in.begin() - in.input().begin());
}
PosIdx atEnd(const auto & in)
{
return positions.add(origin, in.end() - in.input().begin());
}
};
std::unique_ptr<ExprVar> State::mkInternalVar(PosIdx pos, Symbol name) {
return std::make_unique<ExprVar>(pos, name, !featureSettings.isEnabled(Dep::ShadowInternalSymbols));
}
inline void State::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos)
{
throw ParseError({
.msg = HintFmt("attribute '%1%' already defined at %2%",
showAttrPath(symbols, attrPath), positions[prevPos]),
.pos = positions[pos]
});
}
inline void State::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos)
{
throw ParseError({
.msg = HintFmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]),
.pos = positions[pos]
});
}
inline void State::overridesFound(const PosIdx pos) {
// Added 2024-09-18 as a warning, updated and made hard error 2025-11-27
// See the documentation on deprecated features for more details.
throw ParseError({
.msg = HintFmt(
"%s attributes are deprecated and will be removed in the future. Use %s to silence this warning.",
"__overrides",
"--extra-deprecated-features rec-set-overrides"
),
.pos = positions[pos],
});
}
// Both added 2026-01-30. Probably won't turn this one into an error for a while,
// as this has quite a lot of use in the wild. But it's clearly wrong code,
// so we should warn users about it.
// See the documentation on deprecated features for more details.
inline void State::badSingleLineIndStringFound(const PosIdx pos)
{
logWarning({
.msg = HintFmt(
"Whitespace in a ''-string will be stripped even if the string only has a single line, which is most likely not the intent of the code. To fix this, remove the whitespace or replace the string with \" instead. Use %s to silence this warning.",
"--extra-deprecated-features broken-string-indentation"
),
.pos = positions[pos],
});
}
inline void State::badFirstLineIndStringFound(const PosIdx pos)
{
logWarning({
.msg = HintFmt(
"Whitespace calculations for indentation stripping in a multiline ''-string include the first line, so putting text on it will effectively disable all indentation stripping. To fix this, simply break the line right after the string starts. Use %s to silence this warning.",
"--extra-deprecated-features broken-string-indentation"
),
.pos = positions[pos],
});
}
// Added 2024-12-12, equally used in the wild.
inline void State::badEscapeFound(const PosIdx pos, char found, bool isIndented)
{
auto escape = std::string(isIndented ? "''\\" : "\\");
auto interpolEscape = std::string(isIndented ? "''${" : "\\${");
HintFmt msg = HintFmt(
"%s is an ill-defined escape. You can drop the %s and simply write %s instead. "
"Use %s to silence this warning.",
escape + found,
escape,
found,
"--extra-deprecated-features broken-string-escape"
);
/* Special case some common escapes to provide better messages */
if (found == '$' || found == '{') {
/* Someone possibly tried to escape an interpolation but used the wrong sequence.
* We don't have the full context within this function to know for sure (extracting that information
* would require changing some parser rules), but we can at least add a hint about this to the default
* message.
*/
msg = HintFmt(
"%s is an ill-defined escape. You can drop the %s and simply write %s instead. "
"If you meant to escape an interpolation, write %s instead. "
"Use %s to silence this warning.",
escape + found,
escape,
found,
interpolEscape,
"--extra-deprecated-features broken-string-escape"
);
} else if (found == '\r' || found == '\n') {
/* Someone tried to escape a line break */
msg = HintFmt(
"%s at the end of a line is an ill-defined escape. Escaping line endings has no effect. "
"You can either drop the trailing %s, or use an explicit string concatenation instead. "
"Use %s to silence this warning.",
escape,
escape,
"--extra-deprecated-features broken-string-escape"
);
} else if (!isprint(found) || isspace(found)) {
/* Generic error message for all non-printable escape characters */
msg = HintFmt(
"Found an ill-defined escape. You can drop the %s, as it has no effect. %s"
"Use %s to silence this warning.",
escape,
isIndented
? "(Note that an escaped space at the beginning of a string line may influence the string's "
"indentation, however deliberately using this in strings is not supported.) "
: "",
"--extra-deprecated-features broken-string-escape"
);
}
logWarning({
.msg = msg,
.pos = positions[pos],
});
}
// Added 2025-02-05. This is unlikely to ever occur in the wild, given how broken it is
inline void State::badLineEndingFound(const PosIdx pos, bool warnOnly)
{
ErrorInfo ei = {
.msg = HintFmt(
"CR (`\\r`) and CRLF (`\\r\\n`) line endings are not supported. Please inspect the file and normalize it to use LF (`\\n`) line endings instead. Use %s to silence this warning.",
"--extra-deprecated-features cr-line-endings"
),
.pos = positions[pos],
};
// Within strings we should throw because it is a correctness issue, outside of
// strings it only harmlessly fucks up line numbers in error messages so warning is sufficient.
if (warnOnly) {
if (!hasWarnedAboutBadLineEndings)
logWarning(ei);
hasWarnedAboutBadLineEndings = true;
} else
throw ParseError(ei);
}
// Added 2025-02-05.
inline void State::nulFound(const PosIdx pos)
{
throw ParseError({
.msg = HintFmt(
"NUL bytes (`\\0`) are currently not well supported, because internally strings are NUL-terminated, which may lead to unexpected truncation. Use %s to disable this error.",
"--extra-deprecated-features nul-bytes"
),
.pos = positions[pos],
});
}
// Added 2025-11-23
inline void State::recSetMergeFound(const AttrPath & attrPath, const PosIdx pos)
{
throw ParseError({
.msg = HintFmt(
"attribute '%s' cannot be merged, because one set is marked as recursive and the other "
"isn't. Use %s to disable this error and make the expression parse as-is with "
"implementation-defined semantics.",
showAttrPath(symbols, attrPath),
"--extra-deprecated-features rec-set-merges"
),
.pos = positions[pos],
});
}
// Added 2025-11-24
inline void State::recSetDynamicAttrFound(const PosIdx pos)
{
logWarning({
.msg = HintFmt(
"dynamic attributes are not allowed within recursive attrsets, because they would be "
"evaluated separately from the other recursive attributes. Use %s to disable this "
"error.",
"--extra-deprecated-features rec-set-dynamic-attrs"
),
.pos = positions[pos],
});
}
// Added 2026-01-30
inline void State::orIdentifierFound(const PosIdx pos)
{
logWarning({
.msg = HintFmt(
"using %s as an identifier is deprecated because it cannot be used in most places (try "
"%s). Use %s to disable this warning.",
"or",
"let or = 1; in or",
"--extra-deprecated-features or-as-identifier"
),
.pos = positions[pos],
});
}
// Added 2026-01-30
inline void State::orArgumentFound(const PosIdx pos)
{
logWarning({
.msg = HintFmt(
"using %s as an argument is deprecated because it is parsed with the wrong precedence "
"which may cause unexpected behavior. Use %s to disable this warning.",
"or",
"--extra-deprecated-features or-as-identifier"
),
.pos = positions[pos],
});
}
// Added 2026-01-30
inline void State::whitespaceBetweenTokensRequired(const PosIdx pos)
{
throw ParseError(
{.msg = HintFmt(
"whitespace between function arguments or list elements is required here. Use %s to "
"disable this error",
"--extra-deprecated-features tokens-no-whitespace"
),
.pos = positions[pos]}
);
}
inline void State::addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos)
{
AttrPath::iterator i;
// All attrpaths have at least one attr
assert(!attrPath.empty());
// Walk the attrpath up to the parent of the attribute we want to insert, moving `attrs` along
// and creating new empty intermediate attrsets as necessary.
for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
AttrName & attr = *i;
if (attr.isDynamic()) {
/* We don't want to insert dynamic attributes into recursive sets, because that has
* fucky semantics */
if (ExprSet * set = dynamic_cast<ExprSet *>(attrs);
!featureSettings.isEnabled(Dep::RecSetDynamicAttrs) && set && set->recursive)
{
recSetDynamicAttrFound(pos);
}
// Simply insert an empty attrset (but dynamic)
auto & next = attrs->dynamicAttrs.emplace_back(std::move(i->expr), std::make_unique<ExprSet>(), pos);
attrs = static_cast<ExprSet *>(next.valueExpr.get());
} else if (ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
j != attrs->attrs.end())
{
// Try to walk down the next attribute, throw duplicate error if not possible
auto & [foundName, foundDef] = *j;
if (foundDef.kind == ExprAttrs::AttrDef::Kind::Inherited) {
attrPath.erase(i + 1, attrPath.end());
return dupAttr(attrPath, pos, foundDef.pos);
}
ExprSet * foundAttrs = dynamic_cast<ExprSet *>(foundDef.e.get());
if (!foundAttrs) {
attrPath.erase(i + 1, attrPath.end());
return dupAttr(attrPath, pos, foundDef.pos);
}
attrs = foundAttrs;
} else {
// Simply insert an empty attrset
auto next = attrs->attrs.emplace(
std::piecewise_construct,
std::tuple(attr.symbol),
std::tuple(std::make_unique<ExprSet>(), pos)
);
// Before inserting new attrs, check for __override and throw an error
// (the error will initially be a warning to ease migration)
if (!featureSettings.isEnabled(Dep::RecSetOverrides) && attr.symbol == symbols.sym___overrides) {
if (auto set = dynamic_cast<ExprSet *>(attrs); set && set->recursive) {
overridesFound(pos);
}
}
attrs = static_cast<ExprSet *>(next.first->second.e.get());
}
}
// Expr insertion.
// ==========================
AttrName & attr = *i;
if (attr.isDynamic()) {
/* We don't want to insert dynamic attributes into recursive sets, because that has
* fucky semantics */
if (ExprSet * set = dynamic_cast<ExprSet *>(attrs);
!featureSettings.isEnabled(Dep::RecSetDynamicAttrs) && set && set->recursive)
{
recSetDynamicAttrFound(pos);
}
attrs->dynamicAttrs.emplace_back(std::move(attr.expr), std::move(e), pos);
} else if (ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(attr.symbol);
j != attrs->attrs.end())
{
// This attr path is already defined. However, if both
// e and the expr pointed by the attr path are two attribute sets,
// we want to merge them.
// Otherwise, throw an error.
auto & [foundName, foundDef] = *j;
auto * insertAttrs = dynamic_cast<ExprSet *>(e.get());
auto * foundAttrs = dynamic_cast<ExprSet *>(foundDef.e.get());
if (!foundAttrs || !insertAttrs) {
return dupAttr(attrPath, pos, foundDef.pos);
}
mergeAttrs(attrPath, insertAttrs, foundAttrs);
} else {
// This attr path is not defined. Let's create it.
// Before inserting new attrs, check for __override and throw an error
// (the error will initially be a warning to ease migration)
if (!featureSettings.isEnabled(Dep::RecSetOverrides) && attr.symbol == symbols.sym___overrides) {
if (auto set = dynamic_cast<ExprSet *>(attrs); set && set->recursive) {
overridesFound(pos);
}
}
// Also check for recursive sets to insert into and throw an error
// We check on the attrpath length because `x = null;` in `rec { x = null; }` is allowed but
// not `{ x = rec {}; x.y = null; }`.
// (Note that `rec { x.y = null; }` is not affected by this condition because `attrs`
// currently points to `x` (non-rec`) and not to the outer set.)
if (!featureSettings.isEnabled(Dep::RecSetMerges) && attrPath.size() > 1) {
if (auto set = dynamic_cast<ExprSet *>(attrs); set && set->recursive) {
recSetMergeFound(attrPath, e->pos);
}
}
e->setName(attr.symbol);
attrs->attrs.emplace(
std::piecewise_construct, std::tuple(attr.symbol), std::tuple(std::move(e), pos)
);
}
}
/* mutably merge source into target. attrPath is only for error messages */
inline void State::mergeAttrs(AttrPath & attrPath, ExprSet * source, ExprSet * target)
{
// Before merging, we check that either both or neither are marked as `rec` and throw an error
// otherwise
if (!featureSettings.isEnabled(Dep::RecSetMerges) && source->recursive != target->recursive) {
recSetMergeFound(attrPath, source->pos);
}
if (source->inheritFromExprs && !target->inheritFromExprs) {
target->inheritFromExprs = std::make_unique<std::list<std::unique_ptr<Expr>>>();
}
for (auto & [insertKey, insertDef] : source->attrs) {
if (auto collision = target->attrs.find(insertKey); collision != target->attrs.end()) {
// Attr already defined in target, recurse merge if possible otherwise error.
auto * collisionInsert = dynamic_cast<ExprSet *>(insertDef.e.get());
auto * collisionTarget = dynamic_cast<ExprSet *>(collision->second.e.get());
if (!collisionInsert || !collisionTarget) {
attrPath.push_back(AttrName(insertDef.pos, insertKey));
return dupAttr(attrPath, insertDef.pos, collision->second.pos);
}
// Push insertKey to the attrPath for error propagation (pop afterwards), then recurse
// merge
attrPath.push_back(AttrName(insertDef.pos, insertKey));
mergeAttrs(attrPath, collisionInsert, collisionTarget);
attrPath.pop_back();
}
if (insertDef.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
auto & sel = dynamic_cast<ExprSelect &>(*insertDef.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
from.displ += target->inheritFromExprs->size();
}
target->attrs.emplace(insertKey, std::move(insertDef));
}
std::ranges::move(source->dynamicAttrs, std::back_inserter(target->dynamicAttrs));
if (source->inheritFromExprs) {
std::ranges::move(*source->inheritFromExprs, std::back_inserter(*target->inheritFromExprs));
}
}
inline void State::validateLambdaAttrs(AttrsPattern & formals, PosIdx pos)
{
std::sort(formals.formals.begin(), formals.formals.end(),
[] (const auto & a, const auto & b) {
return std::tie(a.name, a.pos) < std::tie(b.name, b.pos);
});
std::optional<std::pair<Symbol, PosIdx>> duplicate;
for (size_t i = 0; i + 1 < formals.formals.size(); i++) {
if (formals.formals[i].name != formals.formals[i + 1].name)
continue;
std::pair thisDup{formals.formals[i].name, formals.formals[i + 1].pos};
duplicate = std::min(thisDup, duplicate.value_or(thisDup));
}
if (duplicate)
throw ParseError({
.msg = HintFmt("duplicate formal function argument '%1%'", symbols[duplicate->first]),
.pos = positions[duplicate->second]
});
if (formals.name && formals.has(formals.name))
throw ParseError({
.msg = HintFmt("duplicate formal function argument '%1%'", symbols[formals.name]),
.pos = positions[pos]
});
}
inline std::unique_ptr<Expr> State::stripIndentation(
const PosIdx pos,
std::vector<IndStringLine> && lines)
{
/* If the only line is whitespace-only, directly return empty string.
* The rest of the code relies on the final string not being empty.
*/
if (lines.size() == 1 && lines.front().parts.empty()) {
return std::make_unique<ExprString>(pos, "");
}
/* If the last line only contains whitespace, trim it to not cause excessive whitespace.
* (Other whitespace-only lines get stripped only of the common indentation, and excess
* whitespace becomes part of the string.)
*/
if (lines.back().parts.empty()) {
lines.back().indentation = {};
}
/* Figure out the minimum indentation. Note that by design
whitespace-only lines are not taken into account. */
size_t minIndent = 1000000;
for (auto & line : lines) {
if (line.hasContent) {
minIndent = std::min(minIndent, line.indentation.size());
}
}
/* Strip spaces from each line. */
for (auto & line : lines) {
line.indentation.remove_prefix(std::min(minIndent, line.indentation.size()));
}
/* Concat the parts together again */
std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> parts;
/* Accumulator for merging intermediates */
PosIdx merged_pos;
std::string merged = "";
auto push_merged = [&] (PosIdx i_pos, std::string_view str) {
if (merged.empty()) {
merged_pos = i_pos;
}
merged += str;
};
auto flush_merged = [&] () {
if (!merged.empty()) {
parts.emplace_back(merged_pos, std::make_unique<ExprString>(pos, std::string(merged)));
merged.clear();
}
};
for (auto && [li, line] : enumerate(lines)) {
push_merged(line.pos, line.indentation);
for (auto & val : line.parts) {
auto &[i_pos, item] = val;
std::visit(overloaded{
[&](std::string_view str) {
push_merged(i_pos, str);
},
[&](std::unique_ptr<Expr> expr) {
flush_merged();
parts.emplace_back(i_pos, std::move(expr));
},
}, std::move(item));
}
}
flush_merged();
/* If this is a single string, then don't do a concatenation.
* (If it's a single expression, still do the ConcatStrings to properly force it being a string.)
*/
if (parts.size() == 1 && dynamic_cast<ExprString *>(parts[0].second.get())) {
return std::move(parts[0].second);
}
return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts));
}
}
|