/// Largest observed instruction count from ticket request to lock acquisition.

static MAXIMUM_WAIT: u64 = 0;

    let requested = retired();

    let wait = start - requested;

    if wait > MAXIMUM_WAIT {

        set MAXIMUM_WAIT = wait;

    }

/// Read the maximum measured ticket acquisition work under serialization.

/// This includes the acquisition of the reporting call itself.

export fn maximumWait() -> u64 {

    let guard = enter();

    let value = MAXIMUM_WAIT;

    leave(guard);

    return value;

}

#!/usr/bin/env python3

"""Require the native acceptance matrix and report observed metadata work."""

import pathlib

import re

import sys

def report(path):

    """Read completed fixture records without accepting partial test output."""

    lines = path.read_text().splitlines()

    completed = set(lines)

    samples = {}

    waits = {}

    pending = None

    pending_wait = None

    invariants = False

    cycles_invariants = False

    cycles_pending = None

    cycles_wait = None

    for line in lines:

        if line == "smp final ownership and interval invariants passed":

            if invariants:

                raise ValueError("duplicate SMP invariant completion")

            invariants = True

        elif match := re.fullmatch(r"smp metadata instructions: 0x([0-9a-f]+)", line):

            pending = int(match[1], 16)

        elif match := re.fullmatch(r"smp acquisition instructions: 0x([0-9a-f]+)", line):

            pending_wait = int(match[1], 16)

        elif match := re.fullmatch(r"smp: ([128])-hart execution passed", line):

            if pending is None or pending <= 0:

                raise ValueError("SMP completion has no positive metadata measurement")

            if not invariants:

                raise ValueError("SMP completion has no final invariant check")

            invariants = False

            name = f"SMP {match[1]} harts"

            if name in samples:

                raise ValueError("duplicate SMP completion")

            samples[name] = pending

            waits[name] = pending_wait

            pending = None

            pending_wait = None

        elif line == "cycles final recovery and invariants passed":

            if cycles_invariants:

                raise ValueError("duplicate cycles invariant completion")

            cycles_invariants = True

        elif match := re.fullmatch(r"cycles metadata instructions: ([0-9a-f]+)", line):

            cycles_pending = int(match[1], 16)

        elif match := re.fullmatch(r"cycles acquisition instructions: ([0-9a-f]+)", line):

            cycles_wait = int(match[1], 16)

        elif match := re.fullmatch(r"cycles: ([128])-hart load, spawn, yield, fault, and recovery passed", line):

            if not cycles_invariants or cycles_pending is None or cycles_pending <= 0:

                raise ValueError("cycles completion has no final invariants or metadata measurement")

            name = f"cycles {match[1]} harts"

            if name in samples:

                raise ValueError("duplicate cycles completion")

            samples[name] = cycles_pending

            waits[name] = cycles_wait

            cycles_invariants = False

            cycles_pending = None

            cycles_wait = None

        elif match := re.fullmatch(r"runtime (admission )?metadata instructions: ([0-9a-f]+)", line):

            name = "runtime admission" if match[1] else "runtime completion"

            if name in samples:

                raise ValueError("duplicate runtime metadata measurement")

            samples[name] = int(match[2], 16)

        elif match := re.fullmatch(r"runtime (admission )?acquisition instructions: ([0-9a-f]+)", line):

            name = "runtime admission" if match[1] else "runtime completion"

            if name in waits:

                raise ValueError("duplicate runtime acquisition measurement")

            waits[name] = int(match[2], 16)

    names = {"SMP 1 harts", "SMP 2 harts", "SMP 8 harts", "runtime admission", "runtime completion"}

    names.update(f"cycles {harts} harts" for harts in (1, 2, 8))

    if samples.keys() != names or any(value <= 0 for value in samples.values()):

        raise ValueError("metadata measurements are incomplete")

    if waits.keys() != names or any(value is None or value <= 0 for value in waits.values()):

        raise ValueError("acquisition measurements are incomplete")

    required = {

        "dispatch: 1-hart execution passed",

        "termination: 1-hart execution passed",

        "runtime: scheduled cancellation, retry, and publication passed",

        "bootstrap: root exit preserves allocation and creation authority",

        "mmio: 1-hart execution passed",

    }

    for harts in (1, 2, 8):

        required.add(f"scheduling: {harts}-hart root handoff, four user-space yields, and checked shutdown passed")

    for profile, counts in (("dispatch", (1,)), ("smp", (1, 2, 8)), ("termination", (1,)),

                            ("runtime", (1,)), ("bootstrap", (1,)), ("mmio", (1,)),

                            ("scheduling", (1, 2, 8)), ("cycles", (1, 2, 8))):

        for harts in counts:

            required.add(f"replay: {profile} {harts}-hart output matched")

    if missing := required - completed:

        raise ValueError("native completions are missing: " + "; ".join(sorted(missing)))

    for name, value in sorted(samples.items()):

        print(f"{name}: {value} maximum metadata instructions")

        print(f"{name}: {waits[name]} maximum acquisition instructions")

    print(f"SMP/runtime/cycles maximum observed metadata work: {max(samples.values())} instructions")

    print(f"SMP/runtime/cycles maximum observed acquisition work: {max(waits.values())} instructions")

def main():

    """Report a log produced by the acceptance runner."""

    if len(sys.argv) != 2:

        raise ValueError("usage: metrics test-log")

    report(pathlib.Path(sys.argv[1]))

if __name__ == "__main__":

    try:

        main()

    except (OSError, ValueError) as error:

        print(f"kernel acceptance: {error}", file=sys.stderr)

        sys.exit(1)

#!/bin/sh

# Build and test the sibling emulator and the full kernel before reporting acceptance.

set -eu

mkdir -p bin/acceptance

if ! make -C ../kernel-emulator test > bin/acceptance/emulator.log 2>&1; then

    cat bin/acceptance/emulator.log

    exit 1

fi

if ! KERNEL_REPLAY=1 make test RAD_EMULATOR=../kernel-emulator/bin/emulator > bin/acceptance/kernel.log 2>&1; then

    cat bin/acceptance/kernel.log

    exit 1

fi

python3 test/acceptance/sizes > bin/acceptance/sizes.log

python3 test/acceptance/metrics bin/acceptance/kernel.log > bin/acceptance/metrics.log

cat bin/acceptance/sizes.log bin/acceptance/metrics.log

printf 'kernel acceptance: emulator and complete kernel matrix passed\n'

#!/usr/bin/env python3

"""Check kernel source limits and report the built image's fixed memory."""

import pathlib

import struct

import sys

def source_sizes(root):

    """Count physical source lines and combine files that form one module."""

    totals = {"runtime": 0, "tests": 0, "tools": 0}

    modules = {}

    for path in sorted(root.rglob("*")):

        if path.suffix not in (".rad", ".ras"):

            continue

        relative = path.relative_to(root)

        category = "runtime"

        if "tests" in relative.parts or path.stem == "tests":

            category = "tests"

        elif relative.parts[0] == "tools":

            category = "tools"

        lines = len(path.read_bytes().splitlines())

        totals[category] += lines

        if category != "tests":

            module = str(relative.with_suffix(""))

            modules[module] = modules.get(module, 0) + lines

    if not totals["runtime"]:

        raise ValueError("kernel runtime sources are missing")

    for module, lines in sorted(modules.items()):

        print(f"module {module}: {lines} source lines")

        if lines >= 1000:

            raise ValueError(f"module {module} must have fewer than 1000 lines")

    for category, lines in totals.items():

        print(f"kernel {category}: {lines} source lines")

    if totals["runtime"] > 12000:

        raise ValueError("kernel runtime exceeds 12000 source lines")

def image_sizes(path):

    """Validate the native image extents and report each fixed memory segment."""

    with path.open("rb") as stream:

        header = stream.read(64)

    if len(header) != 64:

        raise ValueError("native image header is truncated")

    magic, version, entry = struct.unpack_from("<IIQ", header)

    if magic != 0x30444152 or version != 2:

        raise ValueError("expected a version 2 native image")

    segments = []

    payload = 64

    for index, name in enumerate(("code", "read-only", "writable")):

        address, initialized, memory = struct.unpack_from("<QII", header, 16 + index * 16)

        alignment = 4 if index == 0 else 8

        if initialized > memory or address + memory > 1 << 64:

            raise ValueError(f"invalid {name} segment extent")

        if memory and address % alignment:

            raise ValueError(f"misaligned {name} segment")

        segments.append((address, initialized, memory))

        payload += initialized

        print(f"image {name}: {memory} fixed bytes, {initialized} initialized bytes")

    code, initialized, _ = segments[0]

    if entry % 4 or initialized % 4 or not code <= entry < code + initialized:

        raise ValueError("entry is outside initialized instructions")

    for index, (start, _, size) in enumerate(segments):

        for other, _, length in segments[index + 1:]:

            if size and length and start < other + length and other < start + size:

                raise ValueError("native image segments overlap")

    if path.stat().st_size != payload:

        raise ValueError("native image payload size does not match its header")

    print(f"image total: {sum(segment[2] for segment in segments)} fixed bytes")

def main():

    """Check the worktree, or explicit source and image paths for a fixture."""

    if len(sys.argv) not in (1, 3):

        raise ValueError("usage: sizes [kernel-directory native-image]")

    root = pathlib.Path(sys.argv[1] if len(sys.argv) == 3 else "kernel")

    image = pathlib.Path(sys.argv[2] if len(sys.argv) == 3 else "bin/kernel.rv64")

    source_sizes(root)

    image_sizes(image)

if __name__ == "__main__":

    try:

        main()

    except (OSError, ValueError) as error:

        print(f"kernel size check: {error}", file=sys.stderr)

        sys.exit(1)

//! A private worker instance that exits normally or raises a CPU fault.

use support;

export mod abi;

export mod sys;

/// Worker startup values supplied by its controller.

record Env: Copy {

    /// Readable argument memory.

    argsPointer: *opaque,

    /// Zero selects normal exit; one selects a fault.

    argsSize: u64,

    /// Installed event capability.

    eventsHandle: u64,

    /// Shared event-ring address.

    eventsPointer: u64,

}

/// Check private dependency state before the selected terminal operation.

@default fn main(env: *Env) {

    assert support::advance() == 1;

    if env.argsSize == 0 {

        sys::exit(9);

    }

    assert false;

}

//! Concurrent private package graphs and complete worker resource recovery.

use std::mem;

use kernel::abi;

use kernel::frames;

use kernel::slots;

use kernel::backing;

use kernel::pages;

use kernel::capability;

use kernel::registry;

use kernel::loader;

use kernel::domains;

use kernel::budgets;

use kernel::dispatch;

use kernel::boot;

use kernel::sync;

use kernel::remote;

use kernel::events;

use kernel::devices;

use kernel::platform;

use kernel::invariants;

use kernel::dispatchinput;

/// Bootstrap resource capabilities.

unsafe static TABLE: capability::Table = undefined;

/// Fixture publication barrier.

static READY: u64 = 0;

/// Surviving event consumer.

unsafe static PARENT: abi::Ref = undefined;

/// Controller identities indexed by hart.

unsafe static CHILDREN: [abi::Ref; 8] = undefined;

/// Bootstrap-owned argument page addresses indexed by hart.

static ARGUMENTS: [u64; 8] = [0; 8];

/// Free frames before controller creation.

static FREE: u32 = 0;

/// Reusable page, backing, domain, and context slots before controller creation.

static SLOTS: [u32; 4] = [0; 4];

/// Validated physical page mapping.

fn memory(address: u64) -> *mut u8;

/// Current kernel package-state table.

fn kernelGp() -> u64;

/// Native checker entry.

fn completion() -> u64;

/// Write one diagnostic byte.

fn put(byte: u8);

/// Finish the native fixture.

fn finish();

/// Count reusable frames in short metadata sections.

unsafe fn freeFrames() -> u32 {

    let mut count: u32 = 0;

    let mut first: u32 = 0;

    while first < pages::STORE.backings.pool.count {

        let mut end = first + 128;

        if end > pages::STORE.backings.pool.count {

            set end = pages::STORE.backings.pool.count;

        }

        let guard = sync::enter();

        for i in first..end {

            if frames::available(&pages::STORE.backings.pool, i) {

                set count += 1;

            }

        }

        sync::leave(guard);

        set first = end;

    }

    return count;

}

/// Count reusable object slots after all allocation transactions have stopped.

fn freeSlots(entries: &[slots::Slot]) -> u32 {

    let mut count: u32 = 0;

    let mut first: u32 = 0;

    while first < entries.len {

        let mut end = first + 128;

        if end > entries.len {

            set end = entries.len;

        }

        let guard = sync::enter();

        for i in first..end {

            assert entries[i].state <> slots::State::Reserved;

            if entries[i].state == slots::State::Free {

                set count += 1;

            }

        }

        sync::leave(guard);

        set first = end;

    }

    return count;

}

/// Read object capacity at the quiescent boundaries of the workload.

unsafe fn capacity() -> [u32; 4] {

    return [freeSlots(&pages::STORE.slots[..]), freeSlots(&pages::STORE.backings.slots[..]),

        freeSlots(&domains::STORE.slots[..]), freeSlots(&domains::STORE.contextSlots[..])];

}

/// Copy one trusted package into bootstrap-owned memory.

unsafe fn input(authority: abi::Handle, bytes: &[u8]) -> abi::Handle {

    let handle = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (bytes.len as u64 + 4095) / 4096);

    let page = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, handle)).object);

    assert try! mem::copy(@sliceOf(memory(page.base), page.count * 4096), bytes) == bytes.len;

    return handle;

}

/// Publish fixture state before any hart enters dispatch.

export unsafe fn start(last: bool) {

    if last {

        setup();

        sync::storeRelease(&mut READY, 1);

    }

    while sync::loadAcquire(&READY) == 0 {

    }

}

/// Prepare one controller per hart and a later surviving checker.

unsafe fn setup() {

    let pending = try! slots::reserve(&mut domains::STORE.slots[..]);

    let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);

    capability::initialize(&mut TABLE, owner);

    try! backing::registerDomain(&mut pages::STORE.backings, owner);

    let authority = try! capability::install(&mut TABLE, capability::Entry {

        kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE | abi::ALLOCATE),

    });

    let mut image = abi::Handle(0);

    for bytes in [&dispatchinput::support[..], &dispatchinput::spin[..]] {

        let source = input(authority, &bytes[..]);

        set image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,

            loader::Request { authority, source, offset: 0, length: bytes.len as u64 });

    }

    let source = input(authority, &dispatchinput::job[..]);

    let parentHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);

    set PARENT = (try! capability::get(&TABLE, parentHandle)).object;

    let parent = try! domains::get(&domains::STORE, PARENT);

    set domains::STORE.records[PARENT.index].state = domains::Lifecycle::Active;

    let frame = &mut domains::STORE.contexts[parent.initial.index].frame;

    set frame.pc = completion(); set frame.status = 0x1880;

    set frame.registers[2] = domains::STORE.contexts[parent.initial.index].kernelStack.end;

    set frame.registers[3] = kernelGp();

    let mut arguments: [abi::Handle; 8] = undefined;

    let mut handles: [abi::Handle; 8] = undefined;

    let mut contexts: [abi::Ref; 8] = undefined;

    let mut count: u64 = 0;

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) == 0 {

            continue;

        }

        set count += 1;

        set arguments[hart] = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);

        set ARGUMENTS[hart] = (try! pages::get(&pages::STORE, (try! capability::get(&TABLE, arguments[hart])).object)).base;

    }

    set FREE = freeFrames();

    set SLOTS = capacity();

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) == 0 {

            continue;

        }

        let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);

        let object = (try! capability::get(&TABLE, handle)).object;

        let mut child = try! domains::get(&domains::STORE, object);

        set CHILDREN[hart] = object; set handles[hart] = handle; set contexts[hart] = child.initial;

        try! domains::reparent(&mut domains::STORE, &TABLE, handle, parentHandle);

        let self = try! capability::install(&mut child.memory.table, capability::Entry {

            kind: abi::Kind::Domain, object, rights: abi::Rights(abi::CREATE | abi::ALLOCATE),

        });

        let stack = try! pages::allocate(&mut pages::STORE, &mut child.memory.table, self, 4);

        let storage = try! pages::get(&pages::STORE, (try! capability::get(&child.memory.table, stack)).object);

        let args = @sliceOf(memory(ARGUMENTS[hart]) as *mut u64, 6);

        set args[0] = *(try! pages::grant(&mut pages::STORE, &TABLE, &mut child.memory.table, source, abi::READ as u64));

        set args[1] = dispatchinput::job.len as u64;

        for i in 0..devices::STORE.count {

            if devices::STORE.regions[i].kind == platform::Kind::Clint {

                set args[2] = *(try! devices::install(&devices::STORE, &mut child.memory.table, i));

            }

        }

        assert args[2] <> 0;

        set args[3] = *(try! pages::grant(&mut pages::STORE, &TABLE, &mut child.memory.table, arguments[hart], (abi::READ | abi::WRITE | abi::GRANT) as u64));

        set args[4] = 10000000 * count;

        try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + storage.count as u64 * 4096, ARGUMENTS[hart], 56);

    }

    for byte in "cycles controllers ready\n" {

        put(byte);

    }

    let now = dispatch::clock();

    let start = now + 1000000 * count;

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) == 0 {

            continue;

        }

        let window = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, start, 0xffffffffffffffff);

        let tail = try! budgets::split(&mut budgets::STORE, &mut TABLE, window, start + 800000000 * count, now);

        let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,

            budgets::Binding { budget: window, domain: handles[hart], context: contexts[hart] }, now);

        if hart == 0 {

            let final = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,

                budgets::Binding { budget: tail, domain: parentHandle, context: parent.initial }, now);

        }

    }

}

/// Check all terminal records, shared code identity, and complete transient recovery.

export unsafe fn verify() {

    for byte in "cycles checking completion\n" {

        put(byte);

    }

    let guard = sync::enter();

    let mut parent = try! domains::get(&domains::STORE, PARENT);

    let job = registry::find(&registry::STORE, &"job"[..]) else panic "worker package required";

    let package = try! registry::get(&registry::STORE, job);

    let mut seen: u64 = 0;

    while let event = try! events::pop(&mut parent.memory.ring) {

        assert event.kind == events::CHILD_EXIT and event.code == 68;

        let mut found = false;

        for hart in 0..8 {

            if (boot::PLATFORM.harts & (1 << hart)) == 0 {

                continue;

            }

            if event.value == abi::id(CHILDREN[hart]) {

                assert (seen & (1 << hart)) == 0;

                set seen |= 1 << hart; set found = true;

                let args = @sliceOf(memory(ARGUMENTS[hart]) as *mut u64, 7);

                assert args[5] == package.codeAddress and args[6] == 4;

            }

        }

        assert found;

    }

    assert seen == boot::PLATFORM.harts;

    try! events::refresh(&mut domains::STORE.events, PARENT, &parent.memory.ring);

    assert loader::STATE.live[job.index] and not loader::STATE.busy;

    let retained = loader::STATE.resident[job.index].code.count + loader::STATE.resident[job.index].metadata.count;

    sync::leave(guard);

    let mut done = false;

    for attempt in 0..100000 {

        let guard = sync::enter();

        set done = domains::STORE.dead == 0 and pages::STORE.backings.pool.retiredCount == 0;

        if done {

            sync::leave(guard);

            break;

        }

        let request = try! dispatch::request(0, remote::Action::Reschedule);

        sync::leave(guard);

        assert try! dispatch::awaitRequest(0, request) == 0;

    }

    assert done;

    assert freeFrames() + retained == FREE;

    let recovered = capacity();

    for i in 0..4 {

        assert recovered[i] == SLOTS[i];

    }

    invariants::check(&TABLE);

    for byte in "cycles final recovery and invariants passed\n" {

        put(byte);

    }

    let maximum = sync::maximum();

    assert maximum > 0 and maximum < 1000000;

    for byte in "cycles metadata instructions: " {

        put(byte);

    }

    number(maximum);

    put(10);

    let waiting = sync::maximumWait();

    assert waiting > 0;

    for byte in "cycles acquisition instructions: " {

        put(byte);

    }

    number(waiting);

    put(10);

    finish();

}

/// Write a fixed-width diagnostic value.

fn number(value: u64) {

    let digits = "0123456789abcdef";

    for i in 0..16 {

        put(digits[((value >> ((15 - i) as u64 * 4)) & 15) as u32]);

    }

}

//! Start all initialized harts after the fixture publishes its shared domains.

.text;

    call @kernel::boot::initialize;

    call @kernel::dispatchcheck::start;

    call @kernel::boot::run;

    ebreak;

.export @kernel::dispatchcheck::memory;

.export @kernel::dispatchcheck::kernelGp;

.export @kernel::dispatchcheck::completion;

.export @kernel::dispatchcheck::finish;

.export @kernel::dispatchcheck::put;

@kernel::dispatchcheck::memory

    ret;

@kernel::dispatchcheck::kernelGp

    mv %a0 %gp;

    ret;

@kernel::dispatchcheck::completion

    la %a0 @complete;

    ret;

@complete

    call @kernel::dispatchcheck::verify;

    ebreak;

@kernel::dispatchcheck::put

    li %t0 0x10000000;

    sb %a0 0(%t0);

    ret;

@kernel::dispatchcheck::finish

    li %t0 0x10001000;

    li %t1 0x5555;

    sw %t1 0(%t0);

    ebreak;

#!/bin/sh

# Compile a dependency, controllers, and a worker loaded after dispatch starts.

set -eu

emulator=${RAD_EMULATOR:-emulator}

harts=${1:-1 2 8}

work=$(mktemp -d)

trap 'rm -rf "$work"' EXIT HUP INT TERM

for package in spin job; do

    mkdir "$work/$package"

    cp "test/cycles/$package.rad" "$work/$package.rad"

    cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/$package/"

    set -- -pkg support -mod test/cycles/support.rad -pkg "$package" -mod "$work/$package.rad" \

        -mod "$work/$package/abi.rad" -mod "$work/$package/sys.rad"

    if [ "$package" = spin ]; then

        cp kernel/mmio.rad "$work/spin/"

        set -- "$@" -mod "$work/spin/mmio.rad"

    fi

    "$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev "$@" -entry "$package" -ril "$work"

done

mkdir "$work/kernel"

cp kernel/kernel.rad "$work/kernel.rad"

printf '\nexport mod dispatchcheck;\nexport mod dispatchinput;\nexport mod invariants;\n' >> "$work/kernel.rad"

cp kernel/kernel/*.rad "$work/kernel/"

cp test/cycles/kernel/dispatchcheck.rad test/smp/kernel/invariants.rad "$work/kernel/"

for package in support spin job; do

    length=$(wc -c < "$work/$package.ril")

    printf '/// Complete trusted %s package.\nexport static %s: [u8; %s] = [\n' "$package" "$package" "$length" >> "$work/kernel/dispatchinput.rad"

    od -An -v -tu1 "$work/$package.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"

    printf '];\n' >> "$work/kernel/dispatchinput.rad"

done

sh test/acceptance/compile "$emulator" "$work"

cat test/cycles/machine.ras kernel/kernel/*.ras > "$work/cycles.ras"

"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \

    -- "$work/std.ril" "$work/kernel.ril" "$work/cycles.ras" "$work/cycles.rv64"

for count in $harts; do

    if [ "${KERNEL_REPLAY:-0}" = 1 ]; then

        sh test/acceptance/replay cycles "$emulator" "$work/cycles.rv64" "$count" -max-steps=10000000000

    else

        "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=10000000000 -run "$work/cycles.rv64"

    fi

    printf 'cycles: %s-hart load, spawn, yield, fault, and recovery passed\n' "$count"

done

//! Concurrent load, spawn, yield, fault, and teardown cycles.

use support;

export mod abi;

export mod sys;

export mod mmio;

/// Ordered shared-memory access boundary for the single event consumer.

@intrinsic fn memoryFence();

/// Controller resources and retained completion metadata.

record Arguments: Copy {

    /// Readable binary worker package.

    source: abi::Handle,

    /// Exact package byte length.

    length: u64,

    /// Read-only clock capability.

    clock: abi::Handle,

    /// Page that owns this argument record.

    page: abi::Handle,

    /// Reserved worker interval in clock ticks.

    quantum: u64,

    /// Shared worker code base reported to the checker.

    codeBase: u64,

    /// Number of fully checked worker lifetimes.

    completed: u64,

}

/// Published event wire record.

record Event: Copy {

    /// Notification kind.

    kind: u16,

    /// Reserved zero bits.

    reserved: u16,

    /// Terminal status or fault code.

    code: u32,

    /// Generation-bearing child identity.

    value: u64,

}

/// Shared single-consumer event ring.

record Ring: Copy {

    /// Published event slots.

    data: [Event; 256],

    /// Consumer progress.

    head: u32,

    /// Producer progress.

    tail: u32,

    /// Ring index mask.

    mask: u32,

}

/// Kernel startup pointers for one controller.

record Env: Copy {

    /// Writable arguments retained by the parent.

    argsPointer: *unsafe mut Arguments,

    /// Argument record byte size.

    argsSize: u64,

    /// Installed event capability.

    eventsHandle: u64,

    /// Shared event-ring pointer.

    eventsPointer: *unsafe mut Ring,

}

/// Repeat publication lookup and private worker lifetimes on this hart.

@default unsafe fn main(env: *Env) {

    assert env.argsSize == @sizeOf(Arguments) as u64;

    let mut args = env.argsPointer;

    let initial = sys::currentContext();

    for turn in 0..4 {

        assert support::advance() == turn as u64 + 1;

        let mut image = abi::Handle(0);

        while image == abi::Handle(0) {

            set image = try sys::imageLoad(abi::Handle(0), args.source, 0, args.length) catch error {

                assert error == abi::Error::Busy;

                continue;

            };

        }

        let loaded = try! sys::queryImage(image);

        if turn == 0 {

            set args.codeBase = loaded.codeBase;

        }

        assert loaded.codeBase == args.codeBase;

        let child = try! sys::domainCreate(abi::Handle(0), image);

        let identity = sys::queryDomain(child).id;

        let stack = try! sys::pageAllocate(abi::Handle(0), 4);

        let extent = sys::queryPage(stack);

        let moved = try! sys::capabilityTransfer(stack, child, abi::RIGHTS_MASK as u64);

        let shared = try! sys::capabilityGrant(args.page, child, abi::READ as u64);

        try! sys::domainActivate(child, extent.base + extent.count * 4096, args as *unsafe opaque, (turn % 2) as u64);

        let current = sys::currentContext();

        assert current.context == initial.context and current.hart == initial.hart;

        let later = try! sys::budgetSplit(abi::Handle(current.budget), mmio::read64(args.clock, 0) + args.quantum);

        try! sys::yield(child);

        assert sys::currentContext().budget == *later;

        let mut ring = env.eventsPointer;

        let head = ring.head;

        let tail = ring.tail;

        memoryFence();

        assert head <> tail and ring.mask == 255;

        let event = ring.data[head & ring.mask];

        assert event.value == identity and event.reserved == 0;

        if turn % 2 == 0 {

            assert event.kind == 4 and event.code == 9;

        }

        else {

            assert event.kind == 3 and event.code == 3;

        }

        memoryFence();

        set ring.head = head + 1;

        try! sys::capabilityDrop(image);

        set args.completed = turn as u64 + 1;

    }

    sys::exit(68);

}

//! Shared library code with private state in each domain graph.

/// Per-instance invocation count.

static COUNT: u64 = 0;

/// Advance this domain's private library state.

export fn advance() -> u64 {

    set COUNT += 1;

    return COUNT;

}

    let admissionWait = sync::maximumWait();

    assert admissionWait > 0;

    for byte in "runtime admission acquisition instructions: " {

        put(byte);

    }

    number(admissionWait); put(10);

    let waiting = sync::maximumWait();

    assert waiting > 0;

    for byte in "runtime acquisition instructions: " {

        put(byte);

    }

    number(waiting); put(10);

use kernel::invariants;

    invariants::check(&TABLE);

    for byte in "smp final ownership and interval invariants passed\n" {

        put(byte);

    }

    let waiting = sync::maximumWait();

    assert waiting > 0;

    for byte in "smp acquisition instructions: 0x" {

        put(byte);

    }

    number(waiting);

    put(10);

//! Final ownership and interval checks over published kernel objects.

use kernel::abi;

use kernel::boot;

use kernel::slots;

use kernel::capability;

use kernel::domains;

use kernel::budgets;

use kernel::interrupts;

use kernel::sync;

/// Select authoritative capability storage, including the fixture's boot owner.

/// The bootstrap table and domain memory must outlive the metadata transaction.

unsafe fn table(owner: abi::Ref, bootstrap: &capability::Table) -> *unsafe capability::Table {

    if owner == bootstrap.owner {

        return bootstrap as *unsafe capability::Table;

    }

    let domain = try! domains::get(&domains::STORE, owner);

    assert domain.state <> domains::Lifecycle::Dead;

    assert domain.memory.table.owner == owner;

    return &domain.memory.table;

}

/// Check live references, exclusive execution ownership, and disjoint CPU windows.

/// Each budget or IRQ check uses a separate bounded metadata transaction.

export unsafe fn check(bootstrap: &capability::Table) {

    let guard = sync::enter();

    let mut occupied: u32 = 0;

    for i in 0..domains::STORE.contextSlots.len {

        if domains::STORE.contextSlots[i].state <> slots::State::Live {

            continue;

        }

        let context = domains::STORE.contexts[i];

        let owner = try! domains::get(&domains::STORE, context.owner);

        assert owner.state <> domains::Lifecycle::Dead;

        if let hart = context.hart {

            assert hart < 8 and (boot::PLATFORM.harts & (1 << hart)) <> 0;

            assert (occupied & (1 << hart)) == 0;

            set occupied |= 1 << hart;

        }

    }

    sync::leave(guard);

    for i in 0..budgets::STORE.slots.len {

        let guard = sync::enter();

        if budgets::STORE.slots[i].state == slots::State::Live {

            let object = abi::Ref { index: i, generation: budgets::STORE.slots[i].generation };