test/acceptance/metrics 5.4 KiB raw
1
#!/usr/bin/env python3
2
"""Require the native acceptance matrix and report observed metadata work."""
3
4
import pathlib
5
import re
6
import sys
7
8
9
def report(path):
10
    """Read completed fixture records without accepting partial test output."""
11
    lines = path.read_text().splitlines()
12
    completed = set(lines)
13
    samples = {}
14
    waits = {}
15
    pending = None
16
    pending_wait = None
17
    invariants = False
18
    cycles_invariants = False
19
    cycles_pending = None
20
    cycles_wait = None
21
    for line in lines:
22
        if line == "smp final ownership and interval invariants passed":
23
            if invariants:
24
                raise ValueError("duplicate SMP invariant completion")
25
            invariants = True
26
        elif match := re.fullmatch(r"smp metadata instructions: 0x([0-9a-f]+)", line):
27
            pending = int(match[1], 16)
28
        elif match := re.fullmatch(r"smp acquisition instructions: 0x([0-9a-f]+)", line):
29
            pending_wait = int(match[1], 16)
30
        elif match := re.fullmatch(r"smp: ([128])-hart execution passed", line):
31
            if pending is None or pending <= 0:
32
                raise ValueError("SMP completion has no positive metadata measurement")
33
            if not invariants:
34
                raise ValueError("SMP completion has no final invariant check")
35
            invariants = False
36
            name = f"SMP {match[1]} harts"
37
            if name in samples:
38
                raise ValueError("duplicate SMP completion")
39
            samples[name] = pending
40
            waits[name] = pending_wait
41
            pending = None
42
            pending_wait = None
43
        elif line == "cycles final recovery and invariants passed":
44
            if cycles_invariants:
45
                raise ValueError("duplicate cycles invariant completion")
46
            cycles_invariants = True
47
        elif match := re.fullmatch(r"cycles metadata instructions: ([0-9a-f]+)", line):
48
            cycles_pending = int(match[1], 16)
49
        elif match := re.fullmatch(r"cycles acquisition instructions: ([0-9a-f]+)", line):
50
            cycles_wait = int(match[1], 16)
51
        elif match := re.fullmatch(r"cycles: ([128])-hart load, spawn, yield, fault, and recovery passed", line):
52
            if not cycles_invariants or cycles_pending is None or cycles_pending <= 0:
53
                raise ValueError("cycles completion has no final invariants or metadata measurement")
54
            name = f"cycles {match[1]} harts"
55
            if name in samples:
56
                raise ValueError("duplicate cycles completion")
57
            samples[name] = cycles_pending
58
            waits[name] = cycles_wait
59
            cycles_invariants = False
60
            cycles_pending = None
61
            cycles_wait = None
62
        elif match := re.fullmatch(r"runtime (admission )?metadata instructions: ([0-9a-f]+)", line):
63
            name = "runtime admission" if match[1] else "runtime completion"
64
            if name in samples:
65
                raise ValueError("duplicate runtime metadata measurement")
66
            samples[name] = int(match[2], 16)
67
        elif match := re.fullmatch(r"runtime (admission )?acquisition instructions: ([0-9a-f]+)", line):
68
            name = "runtime admission" if match[1] else "runtime completion"
69
            if name in waits:
70
                raise ValueError("duplicate runtime acquisition measurement")
71
            waits[name] = int(match[2], 16)
72
    names = {"SMP 1 harts", "SMP 2 harts", "SMP 8 harts", "runtime admission", "runtime completion"}
73
    names.update(f"cycles {harts} harts" for harts in (1, 2, 8))
74
    if samples.keys() != names or any(value <= 0 for value in samples.values()):
75
        raise ValueError("metadata measurements are incomplete")
76
    if waits.keys() != names or any(value is None or value <= 0 for value in waits.values()):
77
        raise ValueError("acquisition measurements are incomplete")
78
    required = {
79
        "dispatch: 1-hart execution passed",
80
        "termination: 1-hart execution passed",
81
        "runtime: scheduled cancellation, retry, and publication passed",
82
        "bootstrap: root exit preserves allocation and creation authority",
83
        "mmio: 1-hart execution passed",
84
    }
85
    for harts in (1, 2, 8):
86
        required.add(f"scheduling: {harts}-hart root handoff, four user-space yields, and checked shutdown passed")
87
    for profile, counts in (("dispatch", (1,)), ("smp", (1, 2, 8)), ("termination", (1,)),
88
                            ("runtime", (1,)), ("bootstrap", (1,)), ("mmio", (1,)),
89
                            ("scheduling", (1, 2, 8)), ("cycles", (1, 2, 8))):
90
        for harts in counts:
91
            required.add(f"replay: {profile} {harts}-hart output matched")
92
    if missing := required - completed:
93
        raise ValueError("native completions are missing: " + "; ".join(sorted(missing)))
94
    for name, value in sorted(samples.items()):
95
        print(f"{name}: {value} maximum metadata instructions")
96
        print(f"{name}: {waits[name]} maximum acquisition instructions")
97
    print(f"SMP/runtime/cycles maximum observed metadata work: {max(samples.values())} instructions")
98
    print(f"SMP/runtime/cycles maximum observed acquisition work: {max(waits.values())} instructions")
99
100
101
def main():
102
    """Report a log produced by the acceptance runner."""
103
    if len(sys.argv) != 2:
104
        raise ValueError("usage: metrics test-log")
105
    report(pathlib.Path(sys.argv[1]))
106
107
108
if __name__ == "__main__":
109
    try:
110
        main()
111
    except (OSError, ValueError) as error:
112
        print(f"kernel acceptance: {error}", file=sys.stderr)
113
        sys.exit(1)