1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
|
import dataclasses
import logging
import re
from enum import StrEnum
from functools import cache
from pathlib import Path
from typing import Any, NamedTuple, ClassVar
from collections.abc import Generator
import tomllib
from testlib.fixtures.file_helper import AssetSymlink, CopyFile, FileDeclaration, Fileish
from testlib.utils import is_value_of_type, test_base_folder, get_global_asset
from tomllib import TOMLDecodeError
LANG_TEST_ID_PATTERN = "{folder_name}:{test_name}"
class LangTestRunner(StrEnum):
"""
A list of possible runners for the lang tests on
The string value is the one present in the filenames / test.toml file as runner
"""
EVAL_OKAY = "eval-okay"
EVAL_FAIL = "eval-fail"
PARSE_OKAY = "parse-okay"
PARSE_FAIL = "parse-fail"
@classmethod
def as_regex_selector(cls) -> str:
"""
Returns a regex to match any available runner name into a named capturing group called "runner_name"
:return: a string containing a matching regex
"""
# For some reason ruff doesn't detect that .value is the string already and not a function
# it seems to work when accessing `LangTestRunner` but not when using `cls`
return rf"(?P<runner_name>{'|'.join([runner.value for runner in cls])})" # type: ignore
INVALID_TESTER_NAME = (
f"invalid runner name: '%s', must be one of {[runner.value for runner in LangTestRunner]!r}"
)
"""
Base message for invalid runner, to use across collection
"""
SUFFIX_REGEX = r"-[\w-]+?"
NAME_PATTERN_GENERIC_EXP = re.compile(
rf"{LangTestRunner.as_regex_selector()}(?P<suffix>{SUFFIX_REGEX})?"
)
NAME_PATTERN_IN_FILE = rf"in({SUFFIX_REGEX})?.nix"
class InFile(NamedTuple):
name: str
suffix: str
@classmethod
def parse(cls, name_str: str) -> "InFile":
"""
Parses the given name into a InFile object
:param name_str: string to parse
:raises ValueError: When The given string was invalid
:return: InFile named tuple, on successful parse
"""
match_ = re.fullmatch(NAME_PATTERN_IN_FILE, name_str)
if match_ is None:
msg = f"invalid in-file name {name_str!r}"
raise ValueError(msg)
return cls(match_.group(0), match_.group(1) or "")
class LangTest:
ids: ClassVar[set[str]] = set()
"""
Set of all existing Ids
"""
def __init__(
self,
name: str,
folder_name: str,
runner: LangTestRunner,
in_file: InFile,
flags: list[str] | None = None,
extra_files: list[str] | None = None,
global_assets: list[str] | None = None,
):
"""
Internal class to represent a lang test
:param name: name of the explicit test (e.g. "eval-depr" or "eval-allow-depr")
:param folder_name: the folder / group of tests this one originates from (e.g. "nul_bytes")
:param runner: which runner to run this test on (e.g. EVAL_FAIL or PARSE_OKAY)
:param in_file: what the input file is
:param flags: additional flags provided for nix
:param extra_files: any additional files which should be copied into the tests directory
:raises ValueError(id, msg): when the id constructed from the parameters is not unique
"""
self.runner = runner
self.flags = flags or []
self.folder = folder_name
self.extra_files = extra_files or []
self.global_assets = global_assets or []
self.in_file_name = in_file.name
self.suffix = in_file.suffix
self.test_name = f"{name}{self.suffix}"
self.id = LANG_TEST_ID_PATTERN.format(folder_name=self.folder, test_name=self.test_name)
if self.id in LangTest.ids:
msg = f"id {self.id!r} is not unique. Please set the 'name' attribute manually"
raise ValueError(self.id, msg)
LangTest.ids.add(self.id)
def _get_files(self) -> FileDeclaration:
"""
Internal function to turn the metadata into a FileDeclaration used for parametrization
:return: FileDeclaration object containing all files required for the test
"""
files: dict[str, Fileish] = {
"in.nix": CopyFile(f"{self.folder}/{self.in_file_name}"),
"lib.nix": CopyFile("lib.nix"),
"out.exp": AssetSymlink(f"{self.folder}/{self.test_name}.out.exp"),
"err.exp": AssetSymlink(f"{self.folder}/{self.test_name}.err.exp"),
}
for file in self.extra_files:
# Make sure to add the extra-files requested by the test.toml
files[file] = CopyFile(f"{self.folder}/{file}")
for asset in self.global_assets:
# Add required global assets like `config.nix`
files[asset] = get_global_asset(asset)
return files
def to_params(self) -> tuple[FileDeclaration, list[str], str]:
"""
Converts the LangTest to the parameters required for parametrization of the test runners
:return: a Tuple of the FileDeclaration (used by the `files` fixture), list of flags and a unique id
"""
return self._get_files(), self.flags, self.id
class InvalidLangTest:
def __init__(self, name: str, reasons: list[str]):
"""
Metadata class for invalid test configuration, used to later fail a test with the given reasons
:param name: name of the test which is configured badly
:param reasons: a list of reasons as to why the configuration is invalid
"""
self.name = name
self.reasons = reasons
def _group_lang_tests(tests: list[LangTest]) -> dict[LangTestRunner, list[LangTest]]:
"""
groups the given list of tests by their runner
:param tests: list of tests to group
:return: grouped tests in the following order: EVAL_OKAY, EVAL_FAIL, PARSE_OKAY, PARSE_FAIL
"""
grouped_tests = {runner_name: [] for runner_name in LangTestRunner}
for test in tests:
grouped_tests[test.runner].append(test)
return grouped_tests
@dataclasses.dataclass
class LangTestDefinition:
"""
A parsed object of a singular `test` section from the `test.toml` file
"""
runner: LangTestRunner
name: str
flags: list[str]
extra_files: list[str]
global_assets: list[str]
matrix: bool
in_: list[InFile]
@classmethod
def parse(cls, dict_: dict[str, Any], in_file_names: list[str]) -> "LangTestDefinition":
"""
Parses the given dict to a LangTestDefinition object.
:param dict_: content of a singular test section within the `test.toml` file
:param in_file_names: a list of all existing in files, used as a default for matrix tests
:return: LangTestDefinition object containing the parsed information
:raises ValueError(test_name, [*reasons]): when any information was invalid
"""
issues = []
runner_name = dict_.pop("runner", "")
try:
runner = LangTestRunner(runner_name)
except ValueError:
issues.append(INVALID_TESTER_NAME % runner_name)
runner = None
extra_files = dict_.pop("extra-files", [])
if not is_value_of_type(extra_files, list[str]):
issues.append(
f"invalid value type for 'extra-files': {extra_files}, expected a list of strings"
)
global_assets = dict_.pop("global-assets", [])
if not is_value_of_type(global_assets, list[str]):
issues.append(
f"invalid value type for 'global-assets': {global_assets}, expected a list of strings"
)
flags = dict_.pop("flags", [])
if not is_value_of_type(flags, list[str]):
issues.append(f"invalid value type for 'flags': {flags}, expected a list of strings")
test_name = dict_.pop("name", runner_name)
is_matrix = dict_.pop("matrix", False)
if not is_value_of_type(is_matrix, bool):
issues.append(f"invalid type for 'matrix': {is_matrix}, expected a boolean")
in_file_def = dict_.pop("in", None)
if is_matrix:
in_correct_type = in_file_def is None or is_value_of_type(in_file_def, list[str])
in_file_names = in_file_def or in_file_names
else:
in_correct_type = is_value_of_type(in_file_def, str | None)
in_file_names = [in_file_def] if in_file_def is not None else ["in.nix"]
in_files: list[InFile] = []
if not in_correct_type or len(in_file_names) == 0:
issues.append(
f"invalid type for 'in': {in_file_def!r}, expected a{' list of' if is_matrix else ''} string"
)
else:
# Only check naming if the type is actually correct
for i, name in enumerate(in_file_names):
try:
in_files.append(InFile.parse(name))
except ValueError as e:
issues.append(f"{e.args[0]} at position {i} for 'in'")
if dict_:
issues.append(f"unexpected arguments: {list(dict_.keys())!r}")
if issues:
raise ValueError(test_name, issues)
return cls(
runner=runner,
name=test_name,
flags=flags,
extra_files=extra_files,
global_assets=global_assets,
matrix=is_matrix,
in_=in_files,
)
def build(self, folder: str) -> tuple[list[LangTest], list[InvalidLangTest]]:
"""
Builds this LangTestDefinition into LangTests.
:param folder: test group folder name this Definition was created from
:return: list of valid LangTests and a list of InvalidLangTests (created when the id wasn't unique)
"""
tests = []
invalid = []
for file in self.in_:
try:
tests.append(
LangTest(
self.name,
folder,
self.runner,
file,
self.flags,
self.extra_files,
self.global_assets,
)
)
except ValueError as e:
id_, *reasons = e.args
invalid.append(InvalidLangTest(id_, reasons))
return tests, invalid
def parse_toml(
toml_content: dict[str, Any], in_files: list[str]
) -> Generator[LangTestDefinition | ValueError, None, None]:
"""
Parses the given toml content to LangTestDefinitions.
:param toml_content: content of the `test.toml` file
:param in_files: a list of all in files, used as a default for matrix tests
:returns: Generator yielding `LangTestDefinition`s for successful parses and ValueError(test_name, [*reasons]) for parse failures
"""
test_attr_name = "test"
issues = []
test_dicts = toml_content.pop(test_attr_name, [])
if not test_dicts:
issues.append(f"key {test_attr_name!r} not found or empty")
test_dicts = []
elif not is_value_of_type(test_dicts, list[dict[str, Any]]):
issues.append(f"Invalid type for {test_attr_name!r}. Expected an Array of Tables")
test_dicts = []
if len(toml_content) > 0:
issues.append(
f"unexpected key(s) {list(toml_content.keys())}; expected only {test_attr_name!r}"
)
for declaration in test_dicts:
try:
yield LangTestDefinition.parse(declaration, in_files)
except ValueError as e:
yield e
if issues:
yield ValueError("", issues)
def _collect_toml_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]:
"""
Collects all tests, declared by a `test.toml` file within the given folder
:param folder: base folder to collect tests in
:return: a list of valid test configurations and a list of invalid test configurations
"""
parent_name = folder.name
invalid_tests: list[InvalidLangTest] = []
tests: list[LangTest] = []
# files starting with "_" will be ignored, e.g. "__pycache__"
all_files = {f.name for f in folder.iterdir() if not f.name.startswith("_")}
in_files = [f for f in all_files if re.fullmatch(rf"in({SUFFIX_REGEX})?\.nix", f) is not None]
# used to make sure all files are actually used
unused_files = all_files - {"test.toml"}
try:
infos: dict[str, Any] = tomllib.loads((folder / "test.toml").read_text())
except TOMLDecodeError as e:
return [], [InvalidLangTest(parent_name, [f"couldn't parse toml: {e!r}"])]
for test in parse_toml(infos, in_files):
test_name = test.name if isinstance(test, LangTestDefinition) else test.args[0]
full_name = LANG_TEST_ID_PATTERN.format(folder_name=parent_name, test_name=test_name)
if isinstance(test, ValueError):
invalid_tests.append(InvalidLangTest(full_name, test.args[1]))
continue
new_tests, new_invalids = test.build(parent_name)
tests += new_tests
invalid_tests += new_invalids
unused_files -= set(test.extra_files)
for t in new_tests:
unused_files -= {t.in_file_name, f"{t.test_name}.out.exp", f"{t.test_name}.err.exp"}
# Only throw issues about unused files when the group is otherwise valid
# this is done to avoid having unused files showing up due to invalid configurations,
# which should be fixed first and the unused files would just clutter the error messages in that case
if len(unused_files) > 0 and not invalid_tests:
invalid_tests.append(
InvalidLangTest(
parent_name, [f"the following files weren't referenced: {unused_files!r}"]
)
)
return tests, invalid_tests
def _collect_generic_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]:
"""
Collects all tests within the given folder, if no `test.toml` file is presented for a more explicit test configuration
:param folder: base folder to collect tests in
:return: a list of valid test configurations and a list of invalid test configurations
"""
parent_name = folder.name
tests: list[LangTest] = []
invalid_tests: list[InvalidLangTest] = []
all_files = set(folder.iterdir())
unused = {f.name for f in all_files if not f.name.startswith("_")}
collected: set[str] = set()
for file in all_files:
file: Path
if file.suffix == ".exp":
# we cannot use `file.stem` here, as it only removes the last suffix. i.e.
# `"eval-okay.out.exp".stem` => "eval-okay.out"
# `"eval-okay.out.exp".split(".")[0]` => "eval-okay"
# or
# `"parse-fail-some-name.err.exp".stem` => "parse-fail-some-name.err"
# `"parse-fail-some-name.err.exp".split(".")[0]` => "parse-fail-some-name"
test_name = file.name.rsplit(".", 2)[0]
if test_name in collected:
# Skipping collection to avoid duplicate collection
# when both `.err.exp` and `.out.exp` are provided
# in case of non-error traces
unused -= {file.name}
continue
collected |= {test_name}
full_name = LANG_TEST_ID_PATTERN.format(folder_name=parent_name, test_name=test_name)
match = re.fullmatch(NAME_PATTERN_GENERIC_EXP, test_name)
if match is None:
if re.match(LangTestRunner.as_regex_selector(), test_name) is None:
reason = INVALID_TESTER_NAME % test_name
else:
reason = f"incorrectly formatted test name: {test_name!r}"
invalid_tests.append(InvalidLangTest(full_name, [reason]))
continue
runner_name, suffix = match.groups()
runner = LangTestRunner(runner_name)
suffix = suffix or ""
tests.append(
LangTest(runner_name, folder.name, runner, InFile(f"in{suffix}.nix", suffix))
)
unused -= {file.name, f"in{suffix or ''}.nix"}
# Only throw issues about unused files when the group is otherwise valid
# this is done to avoid having unused files showing up due to invalid configurations,
# which should be fixed first and the unused files would just clutter the error messages in that case
if len(unused) > 0 and not invalid_tests:
invalid_tests.append(
InvalidLangTest(parent_name, [f"the following files weren't referenced: {unused!r}"])
)
return tests, invalid_tests
def _collect_test_group(folder: Path) -> tuple[list[LangTest], list[InvalidLangTest]]:
"""
Collects all tests within the given test group
:param folder: where the test group are located
:return: a list of valid test configurations and a list of invalid test configurations
"""
test_declaration = folder / "test.toml"
if test_declaration.exists():
return _collect_toml_test_group(folder)
return _collect_generic_test_group(folder)
def _collect_all_tests() -> tuple[list[LangTest], list[InvalidLangTest]]:
"""
Collects all LangTests present in the functional2/lang folder
:return: a list of valid test configurations and a list of invalid test configurations
"""
logger = logging.getLogger("lang-test-collector")
lang_folder = test_base_folder / "functional2/lang"
tests: list[LangTest] = []
invalid_tests: list[InvalidLangTest] = []
for node in lang_folder.iterdir():
node: Path
# skip files, as test groups are folders and custom tests will be collected by pytest
# these are files like this `lang_util.py` or the `lib.nix` etc
if node.is_file() or node.name == "assets" or node.name.startswith("_"):
continue
# ignore test groups, which have a py file, as those are set up fully custom
# and expected to be collected by pytest and not by us
if list(node.glob("*.py")):
logger.info("skipping %s as it contains a py file, assuming custom tests", node)
continue
t, i = _collect_test_group(node)
tests += t
invalid_tests += i
return tests, invalid_tests
@cache
def fetch_all_lang_tests() -> tuple[dict[LangTestRunner, list[LangTest]], list[InvalidLangTest]]:
"""
Collects all LangTests declared within `functional2/lang` and groups them by runner / type
:return: valid tests as a mapping of runner -> tests for said runner, InvalidLangTests
"""
tests, invalid_tests = _collect_all_tests()
return _group_lang_tests(tests), invalid_tests
|