test: emulator seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test

KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras

# Native preemption across user code, idle, and retained machine work.

.PHONY: dispatch-test

dispatch-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64

	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run

export mod dispatch;

use super::dispatch;

/// Handle timer boundaries and stop on unsupported traps.

    if trap::classify(frame.cause) == trap::Cause::Timer {

        dispatch::interrupt(frame, hart);

        return;

    }

    try! dispatch::initialize(&PLATFORM, hart as u32, stateTable as u64);

/// Enter dispatch after this hart completes platform initialization.

export unsafe fn run() { dispatch::start(&mut HARTS[dispatch::hart()]); }

    call @kernel::boot::run;

    ebreak;

//! Hart-local selection of runnable CPU authority and timer boundaries.

use super::abi;

use super::limits;

use super::slots;

use super::budgets;

use super::domains;

use super::platform;

use super::trap;

use super::range;

/// Dispatch ownership and idle state retained by one physical hart.

export record State: Copy {

    /// Physical hart whose windows this state selects.

    hart: u32,

    /// Context currently executing or interrupted on this hart.

    current: ?abi::Ref,

    /// Budget selected for the current context.

    budget: ?abi::Ref,

    /// M-mode frame that enters the hart's idle loop.

    idle: trap::Frame,

    /// Firmware stack reserved for idle trap handling.

    stack: range::Range,

}

/// Validated RV64 CLINT registers owned by one hart.

export record Timer: Copy {

    /// Shared monotonic mtime register.

    clock: u64,

    /// This hart's mtimecmp register.

    compare: u64,

}

/// Timer and dispatch state owned by one physical hart.

record Runtime: Copy {

    /// Current context and retained idle frame.

    state: State,

    /// Validated timer registers for this hart.

    timer: Timer,

}

/// Runtime slots initialized before each hart enables timer interrupts.

unsafe static HARTS: [Runtime; limits::HARTS] = undefined;

/// Read the executing physical hart identifier.

export fn hart() -> u32;

/// Address of the M-mode idle loop.

fn idleAddress() -> u64;

/// Enable machine timer delivery while global interrupts remain disabled.

fn enable();

/// Initialize this hart's idle frame and validated timer before installing dispatch.

export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {

    let device = try timer(machine, id);

    let stack = machine.stacks[id];

    let mut idle = trap::Frame { registers: [0; 32], pc: idleAddress(), status: 0x1880, cause: 0, value: 0 };

    set idle.registers[2] = stack.end;

    set idle.registers[3] = kernelGp;

    set HARTS[id] = Runtime {

        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,

    };

    arm(device, 0xffffffffffffffff);

}

/// Reevaluate authority after a timer interrupt with shared metadata serialized.

export unsafe fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) {

    let id = hart();

    assert id < limits::HARTS;

    let clock = now(HARTS[id].timer);

    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);

    arm(HARTS[id].timer, choice.deadline);

}

/// Enter the first authorized context or the hart's retained idle frame.

export unsafe fn start(anchor: &mut trap::Hart) -> ! {

    let id = hart();

    assert id < limits::HARTS;

    let mut frame = HARTS[id].state.idle;

    interrupt(&mut frame, anchor);

    enable();

    trap::resume(&frame);

}

/// Read one validated, naturally aligned 64-bit device register.

fn read(address: u64) -> u64;

/// Write one validated, naturally aligned 64-bit device register.

fn write(address: u64, value: u64);

/// Select one CLINT extent and derive the selected hart's timer registers.

export fn timer(machine: &platform::Platform, hart: u32) -> Timer throws (abi::Error) {

    if hart >= limits::HARTS or (machine.harts & (1 << hart)) == 0 { throw abi::Error::InvalidArg; }

    let mut result: ?Timer = nil;

    for i in 0..machine.deviceCount {

        let device = machine.devices[i];

        if device.kind <> platform::Kind::Clint { continue; }

        if result <> nil or device.memory.end < device.memory.start

            or device.memory.end - device.memory.start < 0xc000 or (device.memory.start & 7) <> 0 {

            throw abi::Error::InvalidArg;

        }

        set result = Timer { clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8 };

    }

    let selected = result else { throw abi::Error::InvalidArg; };

    return selected;

}

/// Read the platform's monotonic clock in mtime ticks.

export fn now(timer: Timer) -> u64 { return read(timer.clock); }

/// Program the next boundary while this hart's timer handling is serialized.

export fn arm(timer: Timer, deadline: u64) { write(timer.compare, deadline); }

/// Runnable context and the next boundary that requires a dispatch decision.

export record Choice: Copy {

    /// Current exclusive interval, or nil while idle.

    budget: ?abi::Ref,

    /// Context authorized to execute now, or nil while idle.

    context: ?abi::Ref,

    /// Platform mtime tick for the next timer interrupt.

    deadline: u64,

}

/// Test whether a live context belongs to the active domain owning its budget.

fn runnable(store: &domains::Store, owner: abi::Ref, context: abi::Ref) -> bool {

    if not slots::matches(&store.slots[..], owner, slots::State::Live) { return false; }

    if store.records[owner.index].state <> domains::Lifecycle::Active { return false; }

    if not slots::matches(&store.contextSlots[..], context, slots::State::Live) { return false; }

    return store.contexts[context.index].owner == owner;

}

/// Select the unique runnable window and the earliest relevant timer boundary.

/// The caller serializes budget and domain metadata while taking this snapshot.

export fn select(windows: &budgets::Store, contexts: &domains::Store, hart: u32, now: u64)

    -> Choice throws (abi::Error)

{

    if hart >= limits::HARTS { throw abi::Error::InvalidArg; }

    let mut result = Choice { budget: nil, context: nil, deadline: 0xffffffffffffffff };

    for i in 0..limits::BUDGETS {

        if windows.slots[i].state <> slots::State::Live { continue; }

        let window = windows.windows[i];

        if window.hart <> hart or window.end <= now { continue; }

        let context = window.context else { continue; };

        if not runnable(contexts, window.owner, context) { continue; }

        if window.start > now {

            if window.start < result.deadline { set result.deadline = window.start; }

            continue;

        }

        assert result.budget == nil;

        set result.budget = abi::Ref { index: i, generation: windows.slots[i].generation };

        set result.context = context;

        if window.end < result.deadline { set result.deadline = window.end; }

    }

    return result;

}

/// Save the interrupted context and install the selected frame and kernel stack.

/// Interrupts remain disabled and shared metadata stays serialized through exchange.

export fn exchange(state: &mut State, windows: &budgets::Store, contexts: &mut domains::Store,

    frame: &mut trap::Frame, anchor: &mut trap::Hart, now: u64) -> Choice throws (abi::Error)

{

    let choice = try select(windows, contexts, state.hart, now);

    if let current = state.current {

        assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);

        set contexts.contexts[current.index].frame = *frame;

    }

    if let next = choice.context {

        let context = contexts.contexts[next.index];

        assert context.kernelStack.start < context.kernelStack.end;

        set *frame = context.frame;

        set anchor.stackBottom = context.kernelStack.start;

        set anchor.stackTop = context.kernelStack.end;

    } else {

        set *frame = state.idle;

        set anchor.stackBottom = state.stack.start;

        set anchor.stackTop = state.stack.end;

    }

    set state.current = choice.context;

    set state.budget = choice.budget;

    return choice;

}

//! Naturally aligned RV64 CLINT access with device ordering.

.text;

.export @kernel::dispatch::read;

.export @kernel::dispatch::write;

.export @kernel::dispatch::hart;

.export @kernel::dispatch::idleAddress;

.export @kernel::dispatch::enable;

// The executing hart selects its private dispatch runtime slot.

@kernel::dispatch::hart

    csrr %a0 mhartid;

    ret;

// Saved idle frames enter this loop with machine interrupts enabled by mret.

@kernel::dispatch::idleAddress

    la %a0 @dispatchIdle;

    ret;

@dispatchIdle

    wfi;

    j @dispatchIdle;

// Global interrupt enable is restored from the selected frame by mret.

@kernel::dispatch::enable

    csrr %t0 mie;

    ori %t0 %t0 128;

    csrw mie %t0;

    ret;

// The validated CLINT mapping contains a naturally aligned mtime register.

@kernel::dispatch::read

    fence iorw iorw;

    ld %a0 0(%a0);

    fence iorw iorw;

    ret;

// One RV64 store updates the owning hart's complete mtimecmp value.

@kernel::dispatch::write

    fence iorw iorw;

    sd %a1 0(%a0);

    fence iorw iorw;

    ret;

use super::range;

/// Retained kernel stack capacity for one suspended execution context.

export constant KERNEL_STACK_PAGES: u32 = 64;

    /// Physical frames retained until this context stops.

    kernelFrames: frames::Run,

    /// Exclusive stack bounds used by trap entry and suspended M-mode calls.

    kernelStack: range::Range,

    /// Retained stack allocation for the initial context.

    kernelFrames: frames::Run,

    /// Initial context's mapped stack bounds.

    kernelStack: range::Range,

    let pendingStack = try frames::reserve(pool, KERNEL_STACK_PAGES) catch err {

        try! frames::release(pool, graph.frames);

        try! frames::release(pool, allocation); throw err;

    };

    let kernelFrames = frames::commit(pendingStack);

    let kernelStack = try! frames::extent(pool, kernelFrames);

    zero(kernelStack.start, kernelFrames.count);

    return Prepared { allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack };

    try! frames::release(pool, prepared.kernelFrames);

    set store.contexts[initial.index] = Context {

        owner: object, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack, frame,

    };

export mod dispatch;

use kernel::range;

        owner: child, kernelFrames: frames::Run { first: 0, count: 0 }, kernelStack: range::Range { start: 0, end: 0 },

        frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 },

//! Dispatch selection at CPU-window boundaries and idle gaps.

use std::testing;

use kernel::abi;

use kernel::slots;

use kernel::domains;

use kernel::budgets;

use kernel::capability;

use kernel::dispatch;

use kernel::platform;

use kernel::range;

use kernel::trap;

/// Platform resources used to validate CLINT address selection.

unsafe static PLATFORM: platform::Platform = undefined;

/// Domain lifetimes and saved contexts used for selection.

unsafe static DOMAINS: domains::Store = undefined;

/// Exclusive intervals presented to dispatch.

unsafe static BUDGETS: budgets::Store = undefined;

/// Owner of the test CPU capabilities.

unsafe static TABLE: capability::Table = undefined;

/// Install one live context and two separated windows on hart zero.

unsafe fn initialize() {

    domains::initialize(&mut DOMAINS);

    budgets::initialize(&mut BUDGETS);

    let owner = abi::Ref { index: 0, generation: 1 };

    capability::initialize(&mut TABLE, owner);

    set DOMAINS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };

    set DOMAINS.records[0].state = domains::Lifecycle::Active;

    set DOMAINS.contextSlots[0] = slots::Slot { generation: 1, state: slots::State::Live };

    set DOMAINS.contexts[0].owner = owner;

    let first = try! budgets::seed(&mut BUDGETS, &mut TABLE, 0, 10, 40);

    let gap = try! budgets::split(&mut BUDGETS, &mut TABLE, first, 20, 0);

    let last = try! budgets::split(&mut BUDGETS, &mut TABLE, gap, 30, 0);

    let a = try! capability::get(&TABLE, first);

    let b = try! capability::get(&TABLE, last);

    set BUDGETS.windows[a.object.index].context = owner;

    set BUDGETS.windows[b.object.index].context = owner;

}

/// Dispatch selects only the half-open interval and wakes at the next runnable start.

@test unsafe fn boundaries() throws (testing::TestError) {

    initialize();

    let before = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);

    try testing::expect(before.context == nil and before.deadline == 10);

    let first = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);

    try testing::expect(first.context == TABLE.owner and first.deadline == 20);

    let gap = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 20);

    try testing::expect(gap.context == nil and gap.deadline == 30);

    let last = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 30);

    try testing::expect(last.context == TABLE.owner and last.deadline == 40);

    let ended = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 40);

    try testing::expect(ended.context == nil and ended.deadline == 0xffffffffffffffff);

    let other = try! dispatch::select(&BUDGETS, &DOMAINS, 1, 10);

    try testing::expect(other.context == nil and other.deadline == 0xffffffffffffffff);

}

/// Pending domains and stale context generations cannot become runnable through a budget.

@test unsafe fn lifetimes() throws (testing::TestError) {

    initialize();

    set DOMAINS.records[0].state = domains::Lifecycle::Pending;

    let pending = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);

    try testing::expect(pending.context == nil);

    set DOMAINS.records[0].state = domains::Lifecycle::Active;

    set DOMAINS.contextSlots[0].generation = 2;

    let stale = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);

    try testing::expect(stale.context == nil);

}

/// Each online hart has a separate comparison word and shares one clock register.

@test unsafe fn timerRegisters() throws (testing::TestError) {

    set PLATFORM.harts = 0x81;

    set PLATFORM.deviceCount = 1;

    set PLATFORM.devices[0] = platform::Device {

        kind: platform::Kind::Clint, memory: range::Range { start: 0x2000000, end: 0x2010000 },

    };

    let first = try! dispatch::timer(&PLATFORM, 0);

    let last = try! dispatch::timer(&PLATFORM, 7);

    try testing::expect(first.clock == 0x200bff8 and first.compare == 0x2004000);

    try testing::expect(last.clock == first.clock and last.compare == first.compare + 56);

    let mut failures: u32 = 0;

    try dispatch::timer(&PLATFORM, 1) catch err {

        try testing::expect(err == abi::Error::InvalidArg); set failures += 1;

    };

    set PLATFORM.devices[0].memory.end = 0x200bff8;

    try dispatch::timer(&PLATFORM, 0) catch err {

        try testing::expect(err == abi::Error::InvalidArg); set failures += 1;

    };

    try testing::expect(failures == 2);

}

/// Trap callback supplied to the frame-exchange fixture.

fn handler(frame: &mut trap::Frame, hart: &mut trap::Hart) {}

/// Exchange preserves an M-mode continuation while another context uses its own stack.

@test unsafe fn continuations() throws (testing::TestError) {

    initialize();

    let mut frame = trap::Frame { registers: [0; 32], pc: 0x40, status: 0x80, cause: 0, value: 0 };

    set DOMAINS.contexts[0].frame = frame;

    set DOMAINS.contexts[0].kernelStack = range::Range { start: 0x8000, end: 0xc000 };

    set DOMAINS.contextSlots[1] = slots::Slot { generation: 1, state: slots::State::Live };

    set DOMAINS.contexts[1].owner = TABLE.owner;

    set DOMAINS.contexts[1].frame = frame;

    set DOMAINS.contexts[1].kernelStack = range::Range { start: 0xc000, end: 0x10000 };

    set BUDGETS.windows[1].context = abi::Ref { index: 1, generation: 1 };

    let mut idle = frame;

    set idle.pc = 0x100;

    set idle.status = 0x1880;

    set idle.registers[2] = 0x4000;

    let mut state = dispatch::State {

        hart: 0, current: nil, budget: nil, idle,

        stack: range::Range { start: 0x2000, end: 0x4000 },

    };

    let mut anchor = trap::Hart {

        stackTop: 0x4000, stackBottom: 0x2000, kernelGp: 0x20000, handler,

        savedT0: 0, savedT1: 0, savedSp: 0,

    };

    let first = try! dispatch::exchange(&mut state, &BUDGETS, &mut DOMAINS, &mut frame, &mut anchor, 10);

    try testing::expect(frame.pc == 0x40 and anchor.stackTop == 0xc000);

    set frame.pc = 0x88;

    set frame.status = 0x1880;

    set frame.registers[2] = 0xb000;

    set frame.registers[5] = 43;

    let second = try! dispatch::exchange(&mut state, &BUDGETS, &mut DOMAINS, &mut frame, &mut anchor, 20);

    try testing::expect(anchor.stackBottom == 0xc000 and anchor.stackTop == 0x10000);

    set frame.pc = 0x77;

    let resumed = try! dispatch::exchange(&mut state, &BUDGETS, &mut DOMAINS, &mut frame, &mut anchor, 30);

    try testing::expect(frame.pc == 0x88 and frame.status == 0x1880);

    try testing::expect(frame.registers[2] == 0xb000 and frame.registers[5] == 43);

    try testing::expect(DOMAINS.contexts[1].frame.pc == 0x77 and anchor.stackTop == 0xc000);

    let ended = try! dispatch::exchange(&mut state, &BUDGETS, &mut DOMAINS, &mut frame, &mut anchor, 40);

    try testing::expect(ended.context == nil and frame.pc == idle.pc and frame.registers[2] == 0x4000);

    try testing::expect(anchor.stackTop == 0x4000 and anchor.stackBottom == 0x2000);

}

    try testing::expect(context.kernelFrames.count == domains::KERNEL_STACK_PAGES);

    try testing::expect(context.kernelStack.end - context.kernelStack.start == domains::KERNEL_STACK_PAGES as u64 * 4096);

    try testing::expect((context.kernelStack.end & 15) == 0);

    set BACKINGS.pool.count += 1;

    try failed(abi::Error::OutOfMemory);

export fn resume(frame: &Frame) -> !;

    let slotCap = tokenCapacity(sourceKind, source, pool);

/// Bound symbol and fixup counts by lexical tokens, including one spare slot.

unsafe fn tokenCapacity(kind: scanner::SourceKind, source: *[u8], pool: *unsafe mut strings::Pool) -> u32 {

    let mut scan = scanner::scanner(kind, source, pool);

    let mut count = SOURCE_CAP_PADDING;

    while true {

        let token = scanner::next(&mut scan);

        if token.kind == scanner::TokenKind::Eof { break; }

        set count += 1;

        if token.kind == scanner::TokenKind::Invalid { break; }

    }

    return count;

}

/// Source buffer for comment capacity tests.

static SOURCE: [u8; 4096] = [0; 4096];

/// Long comments consume source space while leaving symbol and fixup demand small.

@test unsafe fn commentStorage() throws (testing::TestError) {

    let source = &mut SOURCE[..];

    for i in 0..source.len { set source[i] = 32; }

    set source[0] = '/'; set source[1] = '/';

    let text = "\n.text;\nret;\n";

    for byte, i in text { set source[source.len - text.len + i] = byte; }

    let program = try assembleSource(&SOURCE[..]);

    try testing::expect(program.text.len == 1 and program.text[0] == encode::ret());

}

cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras > "$work/boot.ras"

cat test/boot/trap.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras > "$work/trap.ras"

//! Native timer preemption across user execution, idle, and machine continuations.

use std::mem;

use std::arch::rv64::shared;

use kernel::abi;

use kernel::slots;

use kernel::backing;

use kernel::pages;

use kernel::capability;

use kernel::registry;

use kernel::loader;

use kernel::domains;

use kernel::budgets;

use kernel::dispatch;

use kernel::boot;

use kernel::dispatchinput;

/// Bootstrap authority used to create fixture domains and CPU windows.

unsafe static TABLE: capability::Table = undefined;

/// Domain handles used for budget binding.

unsafe static HANDLES: [abi::Handle; 3] = undefined;

/// Initial contexts for the user, machine worker, and completion check.

unsafe static CONTEXTS: [abi::Ref; 3] = undefined;

/// View a validated allocated source page.

fn memory(address: u64) -> *mut u8;

/// Current kernel package-state base.

fn kernelGp() -> u64;

/// Native entry for retained M-mode work.

fn worker() -> u64;

/// Native entry for the final assertion callback.

fn completion() -> u64;

/// Complete the machine fixture with a passing status.

fn finish();

/// Prepare real private domains before enabling dispatch on hart zero.

export unsafe fn setup() {

    let pending = try! slots::reserve(&mut domains::STORE.slots[..]);

    let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);

    capability::initialize(&mut TABLE, owner);

    try! backing::registerDomain(&mut pages::STORE.backings, owner);

    let authority = try! capability::install(&mut TABLE, capability::Entry {

        kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE | abi::ALLOCATE),

    });

    let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);

    let sourceEntry = try! capability::get(&TABLE, source);

    let page = try! pages::get(&pages::STORE, sourceEntry.object);

    let bytes = @sliceOf(memory(page.base), 4096);

    let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);

    let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,

        loader::Request { authority, source, offset: 0, length: length as u64 });

    for i in 0..3 {

        set HANDLES[i] = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);

        let cap = try! capability::get(&TABLE, HANDLES[i]);

        let child = try! domains::get(&domains::STORE, cap.object);

        set CONTEXTS[i] = child.initial;

        set domains::STORE.records[cap.object.index].state = domains::Lifecycle::Active;

        if i > 0 {

            set domains::STORE.contexts[child.initial.index].frame.status = 0x1880;

            set domains::STORE.contexts[child.initial.index].frame.registers[2] = domains::STORE.contexts[child.initial.index].kernelStack.end;

            set domains::STORE.contexts[child.initial.index].frame.registers[3] = kernelGp();

        }

    }

    let cap = try! capability::get(&TABLE, HANDLES[0]);

    let mut user = try! domains::get(&domains::STORE, cap.object);

    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);