test/integration/fixtures.rad 12.9 KiB raw
1
//! Compiler test fixtures in `test/tests`: compiling one to an image or to
2
//! binary IL, checking the IL snapshots of all of them, running the ones
3
//! that have an expected exit status, and checking the diagnostics of the
4
//! ones the compiler must reject.
5
6
use std::arch::rv64::image;
7
use std::fmt;
8
use std::fs;
9
use std::io;
10
use std::mem;
11
use std::process;
12
use std::process::Command;
13
use std::process::Outcome;
14
use std::process::Output;
15
use std::process::Target;
16
use std::sys;
17
use super::harness;
18
use super::harness::Expect;
19
use super::harness::Harness;
20
21
/// IL snapshot checker that runs each fixture with a `.ril` file.
22
constant RUNNER: *[u8] = "test/runner.rv64";
23
24
/// Marker of a rejection fixture. Each line `//! rejects: LINE:COL: MESSAGE`
25
/// expects one error at that position of the fixture with that message.
26
export constant REJECTS: *[u8] = "//! rejects:";
27
28
/// Part of a compiler output line that marks an error diagnostic.
29
constant ERROR: *[u8] = ": error: ";
30
31
/// What the checks of one fixture need and where their results are.
32
record Case: Copy {
33
    /// Fixture source path.
34
    source: *[u8],
35
    /// Fixture source text.
36
    content: *[u8],
37
    /// Path of the fixture executable.
38
    image: *[u8],
39
    /// Index of the IL check in the batch, if the fixture has a snapshot.
40
    snapshot: ?u32,
41
    /// Output of the IL check.
42
    report: *[u8],
43
    /// Expected exit status as written in the source, if any.
44
    returns: ?*[u8],
45
    /// Index of the execution in the batch, if the executable exists.
46
    execution: ?u32,
47
    /// Expected stored size of the read-write data as written, if any.
48
    rwData: ?*[u8],
49
    /// Whether the fixture must be rejected, so that its compilation runs in
50
    /// the batch.
51
    rejected: bool,
52
    /// Compiler output of the rejected compilation.
53
    diagnostics: *[u8],
54
}
55
56
/// Collect the compiler arguments that compile a fixture to the executable
57
/// `output`, when it ends in `.rv64`, or otherwise to binary IL in the
58
/// directory `output`, which must then hold `test/test.ril.bin`. Returns the
59
/// file the compilation must produce, after removing it.
60
export fn prepare(h: &mut Harness, source: *[u8], output: *[u8]) -> *[u8]
61
    throws (sys::Error)
62
{
63
    let content = try h.load(source);
64
    let base = harness::stripExt(source);
65
    let isImage = mem::endsWith(output, ".rv64");
66
    let produced = output if isImage else h.join(&[output, "/test/test.ril.bin"]);
67
    if isImage {
68
        h.add(&["-o", output]);
69
    } else {
70
        try harness::makeDirs(output);
71
        h.add(&["-ril", output]);
72
    }
73
    if harness::marker(content, "//! stack-check") <> nil {
74
        h.add(&["-stack-check"]);
75
    }
76
    // A fixture with submodules or assembly is a package directory that
77
    // holds `test.rad`; any other is one module, which depends on the
78
    // packages that it imports.
79
    let directory = harness::directory(source);
80
    if try harness::exists(h.join(&[directory, "/test.manifest"])) {
81
        h.add(&[directory]);
82
    } else {
83
        h.add(&["-name", "test", source]);
84
    }
85
    try fs::remove(produced) catch err {
86
        if err <> sys::Error::NotFound {
87
            throw err;
88
        }
89
    };
90
    return produced;
91
}
92
93
/// Compile a fixture as [`prepare`] says. The compiler's output is not
94
/// captured.
95
export fn compile(h: &mut Harness, args: *[*[u8]]) throws (sys::Error) {
96
    let produced = try prepare(h, args[0], args[1]);
97
    if h.compile(args[0], harness::INHERIT) {
98
        h.expect(h.join(&[args[0], " output"]), produced, Expect::Present);
99
    }
100
}
101
102
/// Print the verdict of one check of a fixture: `ok`, or `FAILED` with the
103
/// reason.
104
fn verdict(h: &mut Harness, source: *[u8], what: *[u8], reason: ?*[u8]) {
105
    io::print("test ");
106
    io::print(source);
107
    io::print(what);
108
    io::print(" ... ");
109
    let detail = reason else {
110
        io::print("ok\n");
111
        h.pass();
112
        return;
113
    };
114
    io::print("FAILED (");
115
    io::print(detail);
116
    io::print(")\n");
117
    set h.failed += 1;
118
}
119
120
/// Check the exit status of the execution of a fixture against the status
121
/// written in its source.
122
fn checkReturns(h: &mut Harness, c: Case, expected: *[u8], outcome: ?Outcome) {
123
    let want = try fmt::parseInt(expected) catch {
124
        return verdict(h, c.source, "", h.join(&["invalid expected status: ", expected]));
125
    };
126
    let actual = outcome else {
127
        return verdict(h, c.source, "", h.join(&["binary not found: ", c.image]));
128
    };
129
    if actual == Outcome::Exited(want.magnitude as i32) {
130
        return verdict(h, c.source, "", nil);
131
    }
132
    verdict(h, c.source, "", h.join(&["expected: ", expected, ", got: ", h.describe(actual)]));
133
}
134
135
/// Check the stored size of the read-write data of a fixture executable
136
/// against the size written in its source.
137
fn checkRwData(h: &mut Harness, c: Case, expected: *[u8]) {
138
    let what = " rw.data";
139
    let want = try fmt::parseInt(expected) catch {
140
        return verdict(h, c.source, what, h.join(&["invalid expected size: ", expected]));
141
    };
142
    let header = try h.load(c.image) catch {
143
        return verdict(h, c.source, what, h.join(&["binary not found: ", c.image]));
144
    };
145
    let layout = try image::parse(header) catch {
146
        return verdict(h, c.source, what, "invalid image header");
147
    };
148
    if layout.rwData.size as u64 == want.magnitude {
149
        return verdict(h, c.source, what, nil);
150
    }
151
    let got = h.num(layout.rwData.size);
152
    verdict(h, c.source, what, h.join(&["expected: ", expected, ", got: ", got]));
153
}
154
155
/// Count the errors in the compiler `output`. With an `expected` line, count
156
/// only the error lines equal to it.
157
fn countErrors(output: *[u8], expected: ?*[u8]) -> u32 {
158
    let mut count: u32 = 0;
159
    let mut lines = mem::split(output, '\n');
160
    while let line = lines.next() {
161
        if mem::find(line, ERROR) == nil {
162
            continue;
163
        }
164
        if let text = expected {
165
            if not mem::eq(line, text) {
166
                continue;
167
            }
168
        }
169
        set count += 1;
170
    }
171
    return count;
172
}
173
174
/// Count the lines of `content` equal to `line`.
175
fn countLines(content: *[u8], line: *[u8]) -> u32 {
176
    let mut count: u32 = 0;
177
    let mut lines = mem::split(content, '\n');
178
    while let other = lines.next() {
179
        if mem::eq(other, line) {
180
            set count += 1;
181
        }
182
    }
183
    return count;
184
}
185
186
/// Print the error lines of the compiler `output` to standard error.
187
fn printErrors(output: *[u8]) {
188
    let mut lines = mem::split(output, '\n');
189
    while let line = lines.next() {
190
        if mem::find(line, ERROR) <> nil {
191
            io::printError("    ");
192
            io::printError(line);
193
            io::printError("\n");
194
        }
195
    }
196
}
197
198
/// Check the rejected compilation of a fixture against its `//! rejects:`
199
/// lines. Each line `//! rejects: LINE:COL: MESSAGE` must match one error of
200
/// the fixture as the compiler prints it, without the path and `error:`, and
201
/// the compiler must report no other errors.
202
fn checkRejects(h: &mut Harness, c: Case) {
203
    let what = " rejects";
204
    let output = h.text(c.diagnostics);
205
    let mut reason: ?*[u8] = nil;
206
    let mut want: u32 = 0;
207
    let mut lines = mem::split(c.content, '\n');
208
    while let line = lines.next() {
209
        if not mem::startsWith(line, REJECTS) {
210
            continue;
211
        }
212
        let rest = &line[REJECTS.len ..];
213
        let mut start: u32 = 0;
214
        while start < rest.len and rest[start] == ' ' {
215
            set start += 1;
216
        }
217
        let spec = &rest[start ..];
218
        let gap = mem::find(spec, ": ") else {
219
            return verdict(h, c.source, what, h.join(&["invalid expectation: ", spec]));
220
        };
221
        // A line listed twice expects the error twice, so with the total
222
        // below the errors are exactly those listed.
223
        let expected = h.join(&[c.source, ":", &spec[.. gap], ERROR, &spec[gap + 2 ..]]);
224
        let times = countLines(c.content, line);
225
        let got = countErrors(output, expected);
226
        if reason == nil and got <> times {
227
            set reason = h.join(&[
228
                "expected ", h.num(times), " of error ", spec, ", got ", h.num(got)
229
            ]);
230
        }
231
        set want += 1;
232
    }
233
    let total = countErrors(output, nil);
234
    if reason == nil and total <> want {
235
        set reason = h.join(&["expected ", h.num(want), " errors, got ", h.num(total)]);
236
    }
237
    if reason <> nil {
238
        printErrors(output);
239
    }
240
    verdict(h, c.source, what, reason);
241
}
242
243
/// Directory of the compiler test fixtures.
244
export constant TESTS: *[u8] = "test/tests";
245
246
/// Return the root modules of the Radiance fixtures in [`TESTS`]: its `.rad`
247
/// files, and `test.rad` in each of its package directories, which hold
248
/// `test.manifest`.
249
export fn fixtures(h: &mut Harness) -> *[*[u8]] throws (sys::Error) {
250
    let mut result: *mut [*[u8]] = &mut [];
251
    try! result.extend(try h.listFiles(TESTS, ".rad"), h.arena);
252
    let mut packages: *mut [*[u8]] = &mut [];
253
    for path in try h.children(TESTS) {
254
        if try harness::exists(h.join(&[path, "/test.manifest"])) {
255
            try! packages.append(h.join(&[path, "/test.rad"]), h.arena);
256
        }
257
    }
258
    harness::sort(&mut packages[..]);
259
    try! result.extend(&packages[..], h.arena);
260
    return &result[..];
261
}
262
263
/// Run the checks of the fixtures: the IL snapshot of each fixture that has
264
/// one, the exit status of each that declares one, the stored size of its
265
/// read-write data when declared, and the diagnostics of each that must be
266
/// rejected. All children run in one batch; results are printed in the order
267
/// of the fixtures. With no `tests`, all fixtures in `test/tests` are checked.
268
export fn bin(h: &mut Harness, tests: *[*[u8]]) throws (sys::Error) {
269
    if not try harness::exists(RUNNER) {
270
        io::printError(h.join(&["error: runner binary not found: ", RUNNER, "\n"]));
271
        io::printError("hint: run 'make test' first\n");
272
        set h.failed += 1;
273
        return;
274
    }
275
    try h.begin("bin");
276
    let mut sources: *mut [*[u8]] = &mut [];
277
    for source in tests {
278
        try! sources.append(source, h.arena);
279
    }
280
    if tests.len == 0 {
281
        try! sources.extend(try fixtures(h), h.arena);
282
        try! sources.extend(try h.listFiles(TESTS, ".ras"), h.arena);
283
    }
284
    let mut cases: *mut [Case] = &mut [];
285
    let mut batch: *mut [Command] = &mut [];
286
    for source in &sources[..] {
287
        let image = h.join(&[harness::stripExt(source), ".rv64"]);
288
        let content = try h.load(source) catch {
289
            verdict(h, source, "", "source not readable");
290
            continue;
291
        };
292
        let mut c = Case {
293
            source,
294
            content,
295
            image,
296
            snapshot: nil,
297
            report: h.path(&[h.num(cases.len), ".log"]),
298
            returns: harness::marker(content, "//! returns:"),
299
            execution: nil,
300
            rwData: harness::marker(content, "//! rw-data-size:"),
301
            rejected: harness::marker(content, REJECTS) <> nil,
302
            diagnostics: h.path(&[h.num(cases.len), ".rejects.log"]),
303
        };
304
        if c.rejected {
305
            try prepare(h, source, h.path(&[h.num(cases.len), ".rv64"]));
306
            let compile = h.cc(h.take(), Output::File(c.diagnostics));
307
            try! batch.append(compile, h.arena);
308
        }
309
        if try harness::exists(h.join(&[harness::stripExt(source), ".ril"])) {
310
            let target = Target::Hosted(process::Memory {
311
                memory: 0, data: 0, stack: 1024
312
            });
313
            set c.snapshot = batch.len;
314
            let check = h.cmd(
315
                RUNNER,
316
                &[source],
317
                target,
318
                harness::RUN_STEPS,
319
                Output::File(
320
                    c.report
321
                )
322
            );
323
            try! batch.append(check, h.arena);
324
        }
325
        if c.returns <> nil and try harness::exists(image) {
326
            set c.execution = batch.len;
327
            let exec = h.cmd(
328
                image, &[], harness::HOSTED, harness::RUN_STEPS, Output::Discard
329
            );
330
            try! batch.append(exec, h.arena);
331
        }
332
        try! cases.append(c, h.arena);
333
    }
334
    let outcomes = h.runAll(&batch[..]);
335
    for c in &cases[..] {
336
        if let index = c.snapshot {
337
            io::print(h.text(c.report));
338
            if outcomes[index] == harness::OK {
339
                h.pass();
340
            } else {
341
                set h.failed += 1;
342
            }
343
        }
344
        if let expected = c.returns {
345
            let mut outcome: ?Outcome = nil;
346
            if let index = c.execution {
347
                set outcome = outcomes[index];
348
            }
349
            checkReturns(h, c, expected, outcome);
350
        }
351
        if c.returns == nil or c.execution <> nil {
352
            if let expected = c.rwData {
353
                checkRwData(h, c, expected);
354
            }
355
        }
356
        if c.rejected {
357
            checkRejects(h, c);
358
        }
359
    }
360
    io::print("\ntest result: ");
361
    io::print("ok. " if h.failed == 0 else "FAILED. ");
362
    let passed = h.passed;
363
    let failed = h.failed;
364
    io::print(h.join(&[h.num(passed), " passed; ", h.num(failed), " failed\n"]));
365
}