radiance · commit

Makefile +2 -2

		lib/std.test.rv64.s \
		lib/std.test.rv64.o

# Kernel modules and tests use a separate package and test entry point.
KERNEL_SRC := kernel/kernel.rad $(shell find kernel/kernel -name '*.rad' ! -name 'tests.rad' ! -path '*/tests/*' 2>/dev/null)
KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras
KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras kernel/kernel/plic.ras
KERNEL_TEST_SRC := kernel/kernel/tests.rad $(shell find kernel/kernel/tests -name '*.rad' 2>/dev/null)
KERNEL := -pkg kernel $(patsubst %,-mod %,$(sort $(KERNEL_SRC)))
KERNEL_TEST := $(BIN_DIR)/kernel.test.rv64

kernel-test: emulator $(KERNEL_TEST)

	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run

# Concurrent user contexts and metadata transactions on every online hart.
.PHONY: smp-test
smp-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run smp "1 2 8"
	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run smp "1 2 8" 7

kernel/kernel.rad +2 -0

export mod platform;
export mod trap;
export mod capability;
export mod transactions;
export mod events;
export mod interrupts;
export mod plic;
export mod registry;
export mod frames;
export mod backing;
export mod pages;
export mod loader;

kernel/kernel/abi.rad +6 -0

    base: u64,
    /// Number of contiguous physical frames.
    count: u64,
}

/// Physical source returned by QueryInterrupt.
export record InterruptInfo: Copy {
    /// PLIC source number; zero does not name an interrupt.
    number: u32,
}

/// Domain identity and lifetime returned by QueryDomain.
export record DomainInfo: Copy {
    /// Generation-bearing domain identity.
    id: u64,
    /// Historical creator identity.

kernel/kernel/boot.rad +5 -1

use super::domains;
use super::budgets;
use super::dispatch;
use super::timers;
use super::remote;
use super::interrupts;
use super::plic;
use std::arch::rv64::shared::catalog;

/// Platform data published by hart zero before secondary initialization.
export unsafe static PLATFORM: platform::Platform = undefined;
/// Release/acquire publication flag for PLATFORM.

unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Handle timer boundaries and user calls, and stop on unsupported traps.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
    let cause = trap::classify(frame.cause);
    if cause == trap::Cause::Timer or cause == trap::Cause::Software {
    if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
        dispatch::interrupt(frame, hart);
        return;
    }
    if cause == trap::Cause::UserCall and trap::fromUser(frame) {
        dispatch::call(frame, hart);

        try! pages::initialize(&mut pages::STORE, &PLATFORM);
        domains::initialize(&mut domains::STORE);
        budgets::initialize(&mut budgets::STORE);
        timers::initialize(&mut timers::STORE);
        remote::initialize(&mut remote::STORE, PLATFORM.harts);
        interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
        try! plic::initialize(&PLATFORM);
        assert entryCount == 2;
        registry::initialize(&mut registry::STORE);
        let bootCatalog = @sliceOf(entries, entryCount);
        try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
        let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";

kernel/kernel/calls.rad +27 -5

use super::loader;
use super::dispatch;
use super::timers;
use super::sync;
use super::remote;
use super::interrupts;
use super::plic;

/// Resolve the owning hart for an operation that can change CPU authority.
unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
    match operation {
        case 11, 12, 70, 71, 73 => {},

    }
    if entry.kind == abi::Kind::Budget {
        if not moving { throw abi::Error::Denied; }
        return try budgets::transfer(&mut budgets::STORE, &mut domain.memory.table, &mut destination.memory.table, handle, rights, now);
    }
    if entry.kind == abi::Kind::Interrupt {
        if not moving { throw abi::Error::Denied; }
        return try interrupts::transfer(&mut interrupts::STORE, &mut domain.memory.table,
            &mut destination.memory.table, handle, rights);
    }
    try resident(entry);
    let pending = try transactions::reserve(&mut destination.memory.table, owner, handle, entry, rights, moving);
    if moving { return transactions::transfer(&mut domain.memory.table, &mut destination.memory.table, pending); }
    return transactions::grant(&mut destination.memory.table, pending);
}

    let entry = try transactions::dropping(&domain.memory.table, handle, domain.events);
    let table: &mut capability::Table = &mut domain.memory.table;
    match entry.kind {
        case abi::Kind::Page => try pages::drop(&mut pages::STORE, table, handle),
        case abi::Kind::Budget => try budgets::drop(&mut budgets::STORE, table, handle),
        case abi::Kind::Interrupt => {
            let number = try interrupts::query(&interrupts::STORE, table, handle);
            plic::mask(number);
            try! interrupts::drop(&mut interrupts::STORE, table, handle);
        },
        else => {
            try resident(entry);
            let removed = try! capability::invalidate(table, handle);
        },
    }
}

/// Map a checked, aligned caller record address to words.
fn memory(address: u64) -> *mut u64;
/// Map a checked four-byte-aligned result buffer to its source-number word.
fn memory32(address: u64) -> *mut u32;

/// Test whether one nonwrapping extent contains the complete result range.
fn contains(base: u64, size: u64, address: u64, length: u64) -> bool {
    return address >= base and address - base <= size and length <= size - (address - base);
}

/// Validate aligned Page or private package-data storage with the given rights.
/// Package-state tables and kernel metadata cannot hold caller-supplied records.
unsafe fn buffer(domain: &domains::Domain, address: u64, size: u64, rights: abi::Rights) -> bool {
    if (address & 7) <> 0 or size > 0xffffffffffffffff - address { return false; }
unsafe fn buffer(domain: &domains::Domain, address: u64, size: u64, rights: abi::Rights, alignment: u64) -> bool {
    assert alignment == 4 or alignment == 8;
    if (address & (alignment - 1)) <> 0 or size > 0xffffffffffffffff - address { return false; }
    if pages::accessible(&pages::STORE, &domain.memory.table, address, size, rights) { return true; }
    for i in 0..limits::PACKAGES {
        if registry::STORE.slots[i].state <> slots::State::Live { continue; }
        let base = domain.graph.table[i];
        if base == 0 { continue; }

}

/// Copy a fixed query record only after validating its exact buffer size.
unsafe fn reply(domain: &domains::Domain, arguments: &[u64], words: &[u64]) throws (abi::Error) {
    let size = words.len as u64 * 8;
    if arguments[2] <> size or arguments[3] <> 0 or not buffer(domain, arguments[1], size, abi::Rights(abi::WRITE)) {
    if arguments[2] <> size or arguments[3] <> 0 or not buffer(domain, arguments[1], size, abi::Rights(abi::WRITE), 8) {
        throw abi::Error::InvalidArg;
    }
    let output = @sliceOf(memory(arguments[1]), words.len);
    for i in 0..words.len { set output[i] = words[i]; }
}

            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
            let page = try pages::get(&pages::STORE, entry.object);
            try reply(&domain, arguments, &[page.base, page.count as u64]);
            return 0;
        },
        case 46 => {
            let number = try interrupts::query(&interrupts::STORE, &domain.memory.table, handle);
            if arguments[2] <> @sizeOf(abi::InterruptInfo) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], 4, abi::Rights(abi::WRITE), 4) { throw abi::Error::InvalidArg; }
            set *memory32(arguments[1]) = number;
            return 0;
        },
        case 47 => {
            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
            let target = try domains::get(&domains::STORE, entry.object);
            let mut parent: u64 = 0;
            if let object = target.parent { set parent = abi::id(object); }

            try reply(&domain, arguments, &[abi::id(object), entry, package.codeAddress, package.code.len as u64 * 4, package.memory as u64]);
            return 0;
        },
        case 60 => {
            if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ), 8) {
                throw abi::Error::InvalidArg;
            }
            let words = @sliceOf(memory(arguments[1]), 4);
            let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
            return abi::id(try domains::contextCreate(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

        case 51 => return try runtimeLoad(&mut domain, loader::Request {
            authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
        }, guard),
        case 60 => {
            if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ), 8) {
                sync::leave(guard); throw abi::Error::InvalidArg;
            }
            let words = @sliceOf(memory(arguments[1]), 4);
            let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
            let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

kernel/kernel/calls.ras +2 -0

//! Typed access to a checked user record buffer.
.text;
.export @kernel::calls::memory;
.export @kernel::calls::memory32;

// The caller checked alignment, extent, and the required domain access rights.
@kernel::calls::memory
@kernel::calls::memory32
    ret;

kernel/kernel/dispatch.rad +5 -2

use super::range;
use super::calls;
use super::timers;
use super::sync;
use super::remote;
use super::plic;

/// Dispatch ownership and idle state retained by one physical hart.
export record State: Copy {
    /// Physical hart whose windows this state selects.
    hart: u32,

fn stackPointer() -> u64;
/// Trap anchor of the hart currently executing this continuation.
fn localAnchor() -> *mut trap::Hart;
/// Address of the M-mode idle loop.
fn idleAddress() -> u64;
/// Enable timer and software interrupt delivery with global interrupts disabled.
/// Enable machine interrupt sources with global interrupts disabled.
fn enable();

/// Initialize this hart's idle frame and validated timer before installing dispatch.
export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {
    let device = try timer(machine, id);

    assert pointer >= stack.start and pointer < stack.end;
    let guard = sync::enter();
    writeSoftware(HARTS[id].timer.software, 0);
    let previous = HARTS[id].state.current;
    save(&mut HARTS[id].state, &mut domains::STORE, frame);
    if trap::classify(frame.cause) == trap::Cause::External { plic::claim(id); }
    let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
    let mut count: u32 = 0;
    for i in 0..limits::REMOTE_PER_HART {
        let delivery = try! remote::take(&mut remote::STORE, id) else { break; };
        match delivery.request.action {

        set deliveries[count] = delivery;
        set count += 1;
    }
    let clock = now(HARTS[id].timer);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
    let notified = timeout.harts | plic::service(clock);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    let mut deadline = choice.deadline;
    if timeout.deadline < deadline { set deadline = timeout.deadline; }
    for target in 0..limits::HARTS {
        if target <> id and (timeout.harts & (1 << target)) <> 0 { signal(target); }
        if target <> id and (notified & (1 << target)) <> 0 { signal(target); }
    }
    arm(HARTS[id].timer, deadline);
    if previous <> choice.context {
        if let context = previous { signalContext(context, clock); }
    }

kernel/kernel/dispatch.ras +2 -1

    j @dispatchIdle;

// Global interrupt enable is restored from the selected frame by mret.
@kernel::dispatch::enable
    csrr %t0 mie;
    ori %t0 %t0 136;
    li %t1 2184;
    or %t0 %t0 %t1;
    csrw mie %t0;
    ret;

// The validated CLINT mapping contains a naturally aligned mtime register.
@kernel::dispatch::read

kernel/kernel/events.rad +18 -0

    head: u32,
    /// Authoritative producer counter.
    tail: u32,
    /// Terminal domain per occupied slot; generation zero denotes an ordinary event.
    subjects: [abi::Ref; CAPACITY],
    /// Interrupt source per occupied slot, or zero for another event kind.
    sources: [u32; CAPACITY],
}

/// Lifetime state of a terminal-event record.
export union State: Copy {
    /// Available for a new domain's terminal reservation.

/// Publish one event after capacity and receiver validation.
fn publish(queue: &mut Queue, ring: &mut Ring, event: Event, subject: abi::Ref) {
    let index = queue.tail & (CAPACITY - 1);
    set ring.data[index] = event;
    set queue.subjects[index] = subject;
    let mut source: u32 = 0;
    if event.kind == INTERRUPT { set source = event.code; }
    set queue.sources[index] = source;
    set queue.tail = next(queue.tail);
    sync::storeRelease32(&mut ring.tail, queue.tail);
}

/// Send an ordinary event, returning false when its reserved occupancy limit is reached.
export fn send(store: &mut Store, receiver: abi::Ref, ring: &mut Ring, event: Event) -> bool throws (abi::Error) {
    if event.reserved <> 0 or (event.kind <> INTERRUPT and event.kind <> TIMEOUT and event.kind <> WAKEUP) {
        throw abi::Error::InvalidArg;
    }
    if event.kind == INTERRUPT and event.code == 0 { throw abi::Error::InvalidArg; }
    try refresh(store, receiver, ring);
    if event.kind == INTERRUPT and interruptPending(store, receiver, event.code) { return true; }
    let queue = &mut store.queues[receiver.index];
    if distance(queue.tail, queue.head) >= ORDINARY { return false; }
    publish(queue, ring, event, abi::Ref { index: 0, generation: 0 });
    return true;
}

/// Test trusted IRQ metadata after the receiver's consumer progress is refreshed.
export fn interruptPending(store: &Store, receiver: abi::Ref, source: u32) -> bool {
    if source == 0 or not live(store, receiver) { return false; }
    let queue = &store.queues[receiver.index];
    for i in 0..distance(queue.tail, queue.head) {
        let index = ((queue.head & (CAPACITY - 1)) + i) & (CAPACITY - 1);
        if queue.sources[index] == source { return true; }
    }
    return false;
}

/// Read one published event and release its slot. Callers serialize consumers.
export fn pop(ring: &mut Ring) -> ?Event throws (abi::Error) {
    let head = sync::loadAcquire32(&ring.head);
    let tail = sync::loadAcquire32(&ring.tail);
    let count = distance(tail, head);

kernel/kernel/interrupts.rad added +137 -0

1	+	//! Exclusive IRQ ownership and retained notification state.
2	+	use super::abi;
3	+	use super::limits;
4	+	use super::slots;
5	+	use super::capability;
6	+	use super::transactions;
7	+	use super::domains;
8	+	use super::events;
9	+
10	+	/// Notification state while hardware delivery is serialized.
11	+	export union State: Copy {
12	+	/// No retained notification prevents the source from being enabled.
13	+	Idle,
14	+	/// The claimed source needs an event in its current receiver.
15	+	Pending,
16	+	/// A queued event must be consumed before the source is enabled.
17	+	Queued,
18	+	}
19	+
20	+	/// One physical source; its table index is the PLIC number minus one.
21	+	export record Interrupt: Copy {
22	+	/// Domain holding the sole interrupt capability.
23	+	owner: abi::Ref,
24	+	/// Sole live capability in the owner's table.
25	+	handle: abi::Handle,
26	+	/// Retained notification and rearming state.
27	+	state: State,
28	+	}
29	+
30	+	/// Fixed source identities and their exclusive receivers.
31	+	export record Store: Copy {
32	+	/// Number of physical sources reported by the platform.
33	+	count: u32,
34	+	/// Source object generations, indexed by PLIC number minus one.
35	+	slots: [slots::Slot; limits::INTERRUPTS],
36	+	/// Published ownership and notification state.
37	+	records: [Interrupt; limits::INTERRUPTS],
38	+	}
39	+
40	+	/// Global interrupt ownership serialized by the metadata lock.
41	+	export unsafe static STORE: Store = undefined;
42	+
43	+	/// Initialize the bounded physical-source profile before issuing capabilities.
44	+	export fn initialize(store: &mut Store, count: u32) {
45	+	assert count <= limits::INTERRUPTS;
46	+	set store.count = count;
47	+	slots::initialize(&mut store.slots[..]);
48	+	}
49	+
50	+	/// Resolve a source object after checking its generation.
51	+	export fn get(store: &Store, object: abi::Ref) -> Interrupt throws (abi::Error) {
52	+	if object.index >= store.count or not slots::matches(&store.slots[..], object, slots::State::Live) {
53	+	throw abi::Error::BadHandle;
54	+	}
55	+	return store.records[object.index];
56	+	}
57	+
58	+	/// Resolve the sole capability owner with the required rights.
59	+	fn owned(store: &Store, table: &capability::Table, handle: abi::Handle, rights: u16) -> abi::Ref throws (abi::Error) {
60	+	let entry = try capability::lookup(table, handle, abi::Kind::Interrupt, abi::Rights(rights));
61	+	let source = try get(store, entry.object);
62	+	if source.owner <> table.owner or source.handle <> handle { throw abi::Error::BadHandle; }
63	+	return entry.object;
64	+	}
65	+
66	+	/// Issue one source during trusted resource initialization.
67	+	export fn seed(store: &mut Store, table: &mut capability::Table, number: u32) -> abi::Handle throws (abi::Error) {
68	+	if number == 0 or number > store.count { throw abi::Error::InvalidArg; }
69	+	let index = number - 1;
70	+	if store.slots[index].state <> slots::State::Free { throw abi::Error::Busy; }
71	+	let pending = try slots::reserve(&mut table.slots[..]);
72	+	let object = abi::Ref { index, generation: store.slots[index].generation };
73	+	let handle = try! abi::handle(abi::Kind::Interrupt, slots::reference(&pending));
74	+	set store.slots[index].state = slots::State::Reserved;
75	+	set store.records[index] = Interrupt { owner: table.owner, handle, state: State::Idle };
76	+	let live = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
77	+	return capability::publish(table, pending, capability::Entry {
78	+	kind: abi::Kind::Interrupt, object: live, rights: abi::Rights(abi::READ \| abi::TRANSFER),
79	+	});
80	+	}
81	+
82	+	/// Read the source number through its sole capability.
83	+	export fn query(store: &Store, table: &capability::Table, handle: abi::Handle) -> u32 throws (abi::Error) {
84	+	return (try owned(store, table, handle, 0)).index + 1;
85	+	}
86	+
87	+	/// Move exclusive ownership after reserving the receiver's capability slot.
88	+	export fn transfer(store: &mut Store, source: &mut capability::Table, target: &mut capability::Table,
89	+	handle: abi::Handle, rights: u64) -> abi::Handle throws (abi::Error)
90	+	{
91	+	let object = try owned(store, source, handle, abi::TRANSFER);
92	+	let entry = try! capability::get(source, handle);
93	+	let pending = try transactions::reserve(target, source.owner, handle, entry, rights, true);
94	+	let result = transactions::transfer(source, target, pending);
95	+	set store.records[object.index].owner = target.owner;
96	+	set store.records[object.index].handle = result;
97	+	if store.records[object.index].state == State::Queued { set store.records[object.index].state = State::Pending; }
98	+	return result;
99	+	}
100	+
101	+	/// Invalidate a source capability after its hardware route has been masked.
102	+	export fn drop(store: &mut Store, table: &mut capability::Table, handle: abi::Handle) throws (abi::Error) {
103	+	let object = try owned(store, table, handle, 0);
104	+	let removed = try! capability::invalidate(table, handle);
105	+	try! slots::release(&mut store.slots[..], object);
106	+	}
107	+
108	+	/// Retain a claimed source without scanning domain capability tables.
109	+	export fn claim(store: &mut Store, number: u32) -> bool {
110	+	if number == 0 or number > store.count { return false; }
111	+	let index = number - 1;
112	+	if store.slots[index].state <> slots::State::Live { return false; }
113	+	if store.records[index].state == State::Idle { set store.records[index].state = State::Pending; }
114	+	return true;
115	+	}
116	+
117	+	/// Deliver retained work or acknowledge consumption; true permits hardware rearming.
118	+	export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, number: u32) -> bool throws (abi::Error) {
119	+	if number == 0 or number > store.count { throw abi::Error::InvalidArg; }
120	+	let index = number - 1;
121	+	if store.slots[index].state <> slots::State::Live { return false; }
122	+	let source = store.records[index];
123	+	let mut domain = try domains::get(receivers, source.owner) catch { return false; };
124	+	if domain.state == domains::Lifecycle::Dead { return false; }
125	+	try events::refresh(&mut receivers.events, source.owner, &domain.memory.ring);
126	+	if source.state == State::Queued {
127	+	if events::interruptPending(&receivers.events, source.owner, number) { return false; }
128	+	set store.records[index].state = State::Idle;
129	+	} else if source.state == State::Pending {
130	+	if try domains::notify(receivers, source.owner,
131	+	events::Event { kind: events::INTERRUPT, reserved: 0, code: number, value: 0 }) {
132	+	set store.records[index].state = State::Queued;
133	+	}
134	+	return false;
135	+	}
136	+	return true;
137	+	}

kernel/kernel/platform.rad +90 -7

    devices: [Device; limits::DEVICES],
    /// Number of MMIO devices.
    deviceCount: u32,
    /// Bit mask of online physical hart IDs.
    harts: u32,
    /// PLIC source count, or zero when no supported controller is present.
    irqSources: u32,
    /// Machine-mode PLIC context index for each online hart.
    irqContexts: [?u32; limits::HARTS],
    /// Timer ticks per second.
    timebase: u32,
    /// Kernel stack ranges, indexed by physical hart ID.
    stacks: [range::Range; limits::HARTS],
    /// Number of stack ranges in the boot contract.

    registers: ?fdt::Span,
    /// Device compatibility strings, if present.
    compatible: ?fdt::Span,
    /// Device type string, if present.
    deviceType: ?fdt::Span,
    /// Interrupt-controller reference identity, or zero when absent.
    phandle: u32,
    /// Cells in one interrupt specifier.
    interruptCells: u32,
    /// PLIC source count, or zero when absent.
    sources: u32,
    /// PLIC context references and local interrupt numbers.
    interrupts: ?fdt::Span,
    /// Node status permits use.
    enabled: bool,
    /// This node or its parent is the reserved-memory container.
    reserved: bool,
}

/// Read a CPU node's bounded physical hart identifier.
fn cpu(bytes: &[u8], node: Node) -> u32 throws (Error) {
    let registers = node.registers else { throw Error::Invalid; };
    if node.sizeCells <> 0 or node.addressCells < 1 or node.addressCells > 2
        or registers.end - registers.start <> node.addressCells * 4 { throw Error::Invalid; }
    let hart = try fdt::integer(bytes, registers.start, node.addressCells * 4) catch err { throw Error::Tree(err); };
    if hart >= limits::HARTS as u64 { throw Error::Invalid; }
    return hart as u32;
}

/// Associate a CPU-local interrupt controller with its parent hart.
fn controller(bytes: &[u8], node: Node, parent: Node, references: &mut [u32]) throws (Error) {
    let kind = parent.deviceType else { throw Error::Invalid; };
    if not string(bytes, kind, &"cpu"[..]) or node.phandle == 0 or node.interruptCells <> 1 { throw Error::Invalid; }
    let hart = try cpu(bytes, parent);
    if references[hart] <> 0 { throw Error::Invalid; }
    for reference in &references[..] { if reference == node.phandle { throw Error::Invalid; } }
    set references[hart] = node.phandle;
}

/// Resolve PLIC context ordinals after all CPU controller references are known.
fn plic(bytes: &[u8], node: Node, references: &[u32], platform: &mut Platform) throws (Error) {
    if node.sources == 0 or node.sources > limits::INTERRUPTS or node.interruptCells <> 1 { throw Error::Invalid; }
    let contexts = node.interrupts else { throw Error::Invalid; };
    let length = contexts.end - contexts.start;
    if length == 0 or length % 8 <> 0 or length / 8 > limits::HARTS * 2 { throw Error::Invalid; }
    let registers = node.registers else { throw Error::Invalid; };
    if registers.end - registers.start <> (node.addressCells + node.sizeCells) * 4 { throw Error::Invalid; }
    let memory = try extent(bytes, registers.start, node.addressCells, node.sizeCells);
    let required = 0x200008 + (length as u64 / 8 - 1) * 0x1000;
    if (memory.start & 3) <> 0 or memory.end - memory.start < required { throw Error::Invalid; }
    for index in 0..length / 8 {
        let offset = contexts.start + index * 8;
        let reference = try fdt::integer(bytes, offset, 4) catch err { throw Error::Tree(err); };
        let cause = try fdt::integer(bytes, offset + 4, 4) catch err { throw Error::Tree(err); };
        let mut hart: ?u32 = nil;
        for id in 0..limits::HARTS { if references[id] <> 0 and references[id] as u64 == reference { set hart = id; } }
        let id = hart else { throw Error::Invalid; };
        if cause == 9 or cause == 0xffffffff { continue; }
        if cause <> 11 { throw Error::Invalid; }
        if (platform.harts & (1 << id)) == 0 { continue; }
        if platform.irqContexts[id] <> nil { throw Error::Invalid; }
        set platform.irqContexts[id] = index;
    }
    for hart in 0..limits::HARTS {
        if (platform.harts & (1 << hart)) <> 0 and platform.irqContexts[hart] == nil { throw Error::Invalid; }
    }
    set platform.irqSources = node.sources;
}

/// Compare a complete byte span with a name.
fn equal(bytes: &[u8], span: fdt::Span, value: &[u8]) -> bool {
    if span.end - span.start <> value.len { return false; }
    for i in 0..value.len { if bytes[span.start + i] <> value[i] { return false; } }
    return true;

fn finish(bytes: &[u8], node: Node, platform: &mut Platform) throws (Error) {
    if not node.enabled { return; }
    let registers = node.registers else { return; };
    if let typeName = node.deviceType {
        if string(bytes, typeName, &"cpu"[..]) {
            if node.sizeCells <> 0 or node.addressCells < 1 or node.addressCells > 2
                or registers.end - registers.start <> node.addressCells * 4 { throw Error::Invalid; }
            let hart = try fdt::integer(bytes, registers.start, node.addressCells * 4) catch err { throw Error::Tree(err); };
            if hart >= limits::HARTS as u64 or (platform.harts & (1 << hart as u32)) <> 0 { throw Error::Invalid; }
            set platform.harts |= 1 << hart as u32;
            let hart = try cpu(bytes, node);
            if (platform.harts & (1 << hart)) <> 0 { throw Error::Invalid; }
            set platform.harts |= 1 << hart;
            return;
        }
    }
    let width = (node.addressCells + node.sizeCells) * 4;
    if width == 0 or (registers.end - registers.start) % width <> 0 { throw Error::Invalid; }


/// Extract native platform resources. Discard the output if decoding fails.
export unsafe fn decode(bytes: &[u8], platform: &mut Platform) throws (Error) {
    set platform.ramCount = 0; set platform.reservedCount = 0; set platform.deviceCount = 0;
    set platform.harts = 0; set platform.timebase = 0; set platform.stackCount = 0;
    set platform.irqSources = 0;
    for hart in 0..limits::HARTS { set platform.irqContexts[hart] = nil; }
    let mut references: [u32; limits::HARTS] = [0; limits::HARTS];
    let mut interruptController: ?Node = nil;
    let tree = try fdt::header(bytes) catch err { throw Error::Tree(err); };
    let mut at = tree.reservations.start;
    while at < tree.reservations.end {
        try protect(platform, try extent(bytes, at, 2, 2)); set at += 16;
    }

                    set reserved = parent.reserved;
                    set enabled = parent.enabled;
                }
                if state.depth == 2 and equal(bytes, name, &"reserved-memory"[..]) { set reserved = true; }
                set nodes[state.depth - 1] = Node { name, addressCells, sizeCells, childAddress: 2, childSize: 1,
                    registers: nil, compatible: nil, deviceType: nil, enabled, reserved };
                    registers: nil, compatible: nil, deviceType: nil, phandle: 0, interruptCells: 0,
                    sources: 0, interrupts: nil, enabled, reserved };
            },
            case fdt::Event::Property { name, value } => {
                if equal(bytes, name, &"#address-cells"[..]) {
                    let count = try scalar(bytes, value);
                    if count > 2 { throw Error::Invalid; } set nodes[state.depth - 1].childAddress = count;

                    let count = try scalar(bytes, value);
                    if count > 2 { throw Error::Invalid; } set nodes[state.depth - 1].childSize = count;
                } else if equal(bytes, name, &"reg"[..]) { set nodes[state.depth - 1].registers = value; }
                else if equal(bytes, name, &"compatible"[..]) { set nodes[state.depth - 1].compatible = value; }
                else if equal(bytes, name, &"device_type"[..]) { set nodes[state.depth - 1].deviceType = value; }
                else if equal(bytes, name, &"phandle"[..]) { set nodes[state.depth - 1].phandle = try scalar(bytes, value); }
                else if equal(bytes, name, &"#interrupt-cells"[..]) { set nodes[state.depth - 1].interruptCells = try scalar(bytes, value); }
                else if equal(bytes, name, &"riscv,ndev"[..]) { set nodes[state.depth - 1].sources = try scalar(bytes, value); }
                else if equal(bytes, name, &"interrupts-extended"[..]) { set nodes[state.depth - 1].interrupts = value; }
                else if equal(bytes, name, &"status"[..]) {
                    set nodes[state.depth - 1].enabled = nodes[state.depth - 1].enabled and (string(bytes, value, &"okay"[..]) or string(bytes, value, &"ok"[..]));
                } else if equal(bytes, name, &"ranges"[..]) and value.start <> value.end { throw Error::Invalid; }
                else if state.depth == 2 and equal(bytes, nodes[state.depth - 1].name, &"cpus"[..]) and equal(bytes, name, &"timebase-frequency"[..]) {
                    set platform.timebase = try scalar(bytes, value);

                        if (stack.start & 15) <> 0 or (stack.end & 15) <> 0 { throw Error::Invalid; }
                        set platform.stacks[i] = stack; try protect(platform, stack);
                    }
                }
            },
            case fdt::Event::EndNode => try finish(bytes, nodes[state.depth], platform),
            case fdt::Event::EndNode => {
                let node = nodes[state.depth];
                if let compatible = node.compatible {
                    if string(bytes, compatible, &"riscv,cpu-intc"[..]) {
                        if state.depth == 0 { throw Error::Invalid; }
                        try controller(bytes, node, nodes[state.depth - 1], &mut references[..]);
                    }
                    if node.enabled and string(bytes, compatible, &"riscv,plic0"[..]) {
                        if interruptController <> nil { throw Error::Invalid; }
                        set interruptController = node;
                    }
                }
                try finish(bytes, node, platform);
            },
            case fdt::Event::End => break,
        }
    }
    if platform.ramCount == 0 or platform.harts == 0 or platform.timebase == 0 { throw Error::Invalid; }
    if let node = interruptController { try plic(bytes, node, &references[..], platform); }
    for hart in 0..limits::HARTS {
        if (platform.harts & (1 << hart)) == 0 { continue; }
        if hart >= platform.stackCount or not inRam(platform, platform.stacks[hart]) { throw Error::Invalid; }
        for other in 0..hart {
            if (platform.harts & (1 << other)) <> 0 and overlaps(platform.stacks[hart], platform.stacks[other]) {

kernel/kernel/plic.rad added +107 -0

1	+	//! PLIC register access and bounded external-interrupt delivery.
2	+	use super::abi;
3	+	use super::limits;
4	+	use super::platform;
5	+	use super::sync;
6	+	use super::slots;
7	+	use super::interrupts;
8	+	use super::domains;
9	+	use super::budgets;
10	+
11	+	/// Validated registers for the kernel's machine interrupt receiver.
12	+	export record Mapping: Copy {
13	+	/// Physical controller base.
14	+	base: u64,
15	+	/// Supported nonzero source count.
16	+	count: u32,
17	+	/// PLIC machine context assigned to physical hart zero.
18	+	context: u32,
19	+	}
20	+
21	+	/// Controller state published before secondary harts start.
22	+	static CONTROLLER: ?Mapping = nil;
23	+
24	+	/// Map a validated aligned MMIO address to one device word.
25	+	fn pointer(address: u64) -> *unsafe mut u32;
26	+
27	+	/// Select the controller and validate its routed context's register extent.
28	+	export fn mapping(machine: &platform::Platform) -> ?Mapping throws (abi::Error) {
29	+	if machine.irqSources == 0 { return nil; }
30	+	if machine.irqSources > limits::INTERRUPTS or (machine.harts & 1) == 0 { throw abi::Error::InvalidArg; }
31	+	let context = machine.irqContexts[0] else { throw abi::Error::InvalidArg; };
32	+	if context >= limits::HARTS * 2 { throw abi::Error::InvalidArg; }
33	+	let mut result: ?Mapping = nil;
34	+	for i in 0..machine.deviceCount {
35	+	let device = machine.devices[i];
36	+	if device.kind <> platform::Kind::Plic { continue; }
37	+	let required = 0x200008 + context as u64 * 0x1000;
38	+	if result <> nil or (device.memory.start & 3) <> 0 or device.memory.end < device.memory.start
39	+	or device.memory.end - device.memory.start < required { throw abi::Error::InvalidArg; }
40	+	set result = Mapping { base: device.memory.start, count: machine.irqSources, context };
41	+	}
42	+	if result == nil { throw abi::Error::InvalidArg; }
43	+	return result;
44	+	}
45	+
46	+	/// Mask all machine contexts and initialize one priority class before dispatch.
47	+	export unsafe fn initialize(machine: &platform::Platform) throws (abi::Error) {
48	+	set CONTROLLER = try mapping(machine);
49	+	let device = CONTROLLER else { return; };
50	+	for hart in 0..limits::HARTS {
51	+	let context = machine.irqContexts[hart] else { continue; };
52	+	for word in 0..(device.count + 32) / 32 {
53	+	sync::write32(pointer(device.base + 0x2000 + context as u64 * 0x80 + word as u64 * 4), 0);
54	+	}
55	+	sync::write32(pointer(device.base + 0x200000 + context as u64 * 0x1000), 0);
56	+	}
57	+	for number in 1..device.count + 1 { sync::write32(pointer(device.base + number as u64 * 4), 1); }
58	+	}
59	+
60	+	/// Change one source enable bit while the metadata lock serializes the word.
61	+	unsafe fn enable(device: Mapping, number: u32, enabled: bool) {
62	+	assert number > 0 and number <= device.count;
63	+	let address = pointer(device.base + 0x2000 + device.context as u64 * 0x80 + (number / 32) as u64 * 4);
64	+	let before = sync::read32(address as *unsafe u32);
65	+	let bit = 1 << (number % 32);
66	+	let mut after = before & ~bit;
67	+	if enabled { set after \|= bit; }
68	+	if after <> before { sync::write32(address, after); }
69	+	}
70	+
71	+	/// Mask a source before its exclusive capability is invalidated.
72	+	export unsafe fn mask(number: u32) {
73	+	let device = CONTROLLER else { return; };
74	+	enable(device, number, false);
75	+	}
76	+
77	+	/// Claim one source, retain its work, complete it while enabled, then mask it.
78	+	export unsafe fn claim(hart: u32) {
79	+	let device = CONTROLLER else { return; };
80	+	assert hart == 0;
81	+	let address = pointer(device.base + 0x200004 + device.context as u64 * 0x1000);
82	+	let number = sync::read32(address as *unsafe u32);
83	+	if number == 0 { return; }
84	+	assert number <= device.count;
85	+	let retained = interrupts::claim(&mut interrupts::STORE, number);
86	+	sync::write32(address, number);
87	+	enable(device, number, false);
88	+	}
89	+
90	+	/// Deliver retained notifications, rearm consumed sources, and report runnable harts.
91	+	export unsafe fn service(now: u64) -> u32 {
92	+	let device = CONTROLLER else { return 0; };
93	+	let mut harts: u32 = 0;
94	+	for number in 1..device.count + 1 {
95	+	let index = number - 1;
96	+	let mut pending = false;
97	+	if interrupts::STORE.slots[index].state == slots::State::Live {
98	+	set pending = interrupts::STORE.records[index].state == interrupts::State::Pending;
99	+	}
100	+	let enabled = try interrupts::service(&mut interrupts::STORE, &mut domains::STORE, number) catch { false };
101	+	enable(device, number, enabled);
102	+	if pending and interrupts::STORE.records[index].state == interrupts::State::Queued {
103	+	set harts \|= budgets::wakeTargets(&budgets::STORE, &domains::STORE, interrupts::STORE.records[index].owner, now);
104	+	}
105	+	}
106	+	return harts;
107	+	}

kernel/kernel/plic.ras added +7 -0

1	+	//! Typed access to validated, naturally aligned PLIC registers.
2	+	.text;
3	+	.export @kernel::plic::pointer;
4	+
5	+	// Register extents and alignment are checked before device access.
6	+	@kernel::plic::pointer
7	+	ret;

kernel/kernel/sys.rad +10 -0

        try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
        return info;
    }
}

/// Read a physical interrupt source. An invalid handle is a program error.
export fn queryInterrupt(handle: abi::Handle) -> abi::InterruptInfo {
    unsafe {
        let mut info: abi::InterruptInfo = undefined;
        try! result(ecall(abi::Operation::QueryInterrupt as u32, *handle as i64,
            (&mut info) as i64, @sizeOf(abi::InterruptInfo) as i64, 0));
        return info;
    }
}

/// Read domain identity and lifetime. An invalid handle is a program error.
export fn queryDomain(handle: abi::Handle) -> abi::DomainInfo {
    unsafe {
        let mut info: abi::DomainInfo = undefined;
        try! result(ecall(abi::Operation::QueryDomain as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::DomainInfo) as i64, 0));

kernel/kernel/tests.rad +1 -0

export mod frames;
export mod backing;
export mod pages;
export mod transactions;
export mod events;
export mod interrupts;
export mod registry;
export mod loader;
export mod instances;
export mod domains;
export mod budgets;

kernel/kernel/tests/calls.rad +33 -0

use kernel::registry;
use kernel::limits;
use kernel::trap;
use kernel::budgets;
use kernel::remote;
use kernel::interrupts;

/// Interrupt queries write exactly four bytes at four-byte-aligned addresses.
@test unsafe fn interruptRecord() throws (testing::TestError) {
    let owner = initialize();
    interrupts::initialize(&mut interrupts::STORE, 8);
    let handle = try! interrupts::seed(&mut interrupts::STORE, &mut MEMORY.table, 7);
    let page = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
    let base = (&RAM[OFFSET]) as u64;
    set RAM[OFFSET] = 0xaaaaaaaa55555555;
    set RAM[OFFSET + 1] = 0x1122334455667788;
    let result = try! calls::invoke(owner, 46, &[*handle, base + 4, 4, 0], 0);
    assert result == 0 and @sizeOf(abi::InterruptInfo) == 4;
    assert RAM[OFFSET] == 0x0000000755555555 and RAM[OFFSET + 1] == 0x1122334455667788;
    let mut rejected: u32 = 0;
    for args in &[
        [*handle, base + 2, 4, 0], [*handle, base + 4, 8, 0],
        [*handle, base + 4096, 4, 0], [*handle, base + 4, 4, 1],
    ] {
        try calls::invoke(owner, 46, &args[..], 0) catch error {
            assert error == abi::Error::InvalidArg; set rejected += 1;
        };
    }
    assert rejected == 4 and RAM[OFFSET] == 0x0000000755555555;
    let boundary = try! calls::invoke(owner, 46, &[*handle, base + 4092, 4, 0], 0);
    assert boundary == 0 and RAM[OFFSET + 511] == 0x0000000700000000;
    let decoded = try! abi::decode(abi::Handle(page));
    set MEMORY.table.entries[decoded.object.index].rights = abi::Rights(abi::READ);
    try calls::invoke(owner, 46, &[*handle, base + 4, 4, 0], 0) catch error {
        assert error == abi::Error::InvalidArg; set rejected += 1;
    };
    assert rejected == 5;
}

/// Deferred budget operations recheck authority and expiration at execution time.
@test unsafe fn deferredBudget() throws (testing::TestError) {
    let owner = initialize();
    let handle = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 1, 10, 100);

kernel/kernel/tests/events.rad +49 -0


/// Kernel event metadata workspace.
unsafe static STORE: events::Store = undefined;
/// Consumer-visible ring for the receiver.
unsafe static RING: events::Ring = undefined;
/// Separate receiver used to check per-domain IRQ coalescing.
unsafe static OTHER: events::Ring = undefined;

/// Return the receiver's current domain reference.
fn receiver() -> abi::Ref { return abi::Ref { index: 0, generation: 1 }; }

/// Reset metadata and open one queue.

/// Build an ordinary wakeup with a recognizable payload.
fn wake(value: u64) -> events::Event {
    return events::Event { kind: events::WAKEUP, reserved: 0, code: 17, value };
}

/// Duplicate IRQ notifications retain one queue entry even at ordinary capacity.
@test unsafe fn interruptCoalescing() throws (testing::TestError) {
    initialize();
    let irq = events::Event { kind: events::INTERRUPT, reserved: 0, code: 5, value: 0 };
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    set RING.data[0].code = 99;
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == 1;
    for i in 1..events::ORDINARY { assert try! events::send(&mut STORE, receiver(), &mut RING, wake(i as u64)); }
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == events::ORDINARY;
    assert not try! events::send(&mut STORE, receiver(), &mut RING,
        events::Event { kind: events::INTERRUPT, reserved: 0, code: 6, value: 0 });
    let consumed = try! events::pop(&mut RING) else panic "missing interrupt";
    assert consumed.code == 99;
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == events::ORDINARY + 1;
}

/// Pending IRQ identity is scoped to the receiver and survives counter wrap.
@test unsafe fn interruptReceivers() throws (testing::TestError) {
    initialize();
    let other = abi::Ref { index: 1, generation: 1 };
    try! events::open(&mut STORE, other, &mut OTHER);
    set STORE.queues[0].head = 0xfffffffe;
    set STORE.queues[0].tail = 0xfffffffe;
    set RING.head = 0xfffffffe;
    set RING.tail = 0xfffffffe;
    let irq = events::Event { kind: events::INTERRUPT, reserved: 0, code: 5, value: 0 };
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert try! events::send(&mut STORE, receiver(), &mut RING,
        events::Event { kind: events::INTERRUPT, reserved: 0, code: 6, value: 0 });
    assert try! events::send(&mut STORE, other, &mut OTHER, irq);
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == 0 and STORE.queues[1].tail == 1;
    assert events::interruptPending(&STORE, receiver(), 5);
    assert events::interruptPending(&STORE, other, 5);
    let first = try! events::pop(&mut RING) else panic "missing wrapped IRQ";
    assert first.code == 5;
    try! events::refresh(&mut STORE, receiver(), &RING);
    assert not events::interruptPending(&STORE, receiver(), 5);
    assert events::interruptPending(&STORE, receiver(), 6);
    assert events::interruptPending(&STORE, other, 5);
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == 1;
}

/// Fill ordinary capacity while preserving the reserved lifecycle capacity.
@test unsafe fn saturation() throws (testing::TestError) {
    initialize();
    for i in 0..events::ORDINARY {
        try testing::expect(try! events::send(&mut STORE, receiver(), &mut RING, wake(i as u64)));

kernel/kernel/tests/interrupts.rad added +120 -0

1	+	//! Exclusive source ownership, deferred notification, and rearming.
2	+	use std::testing;
3	+	use kernel::abi;
4	+	use kernel::slots;
5	+	use kernel::capability;
6	+	use kernel::domains;
7	+	use kernel::events;
8	+	use kernel::interrupts;
9	+	use kernel::limits;
10	+
11	+	/// Physical source metadata under test.
12	+	unsafe static STORE: interrupts::Store = undefined;
13	+	/// Receiver lifetimes and trusted event metadata.
14	+	unsafe static DOMAINS: domains::Store = undefined;
15	+	/// Source domain capability table and ring.
16	+	unsafe static SOURCE: domains::Memory = undefined;
17	+	/// Target domain capability table and ring.
18	+	unsafe static TARGET: domains::Memory = undefined;
19	+
20	+	/// Initialize two live receivers and eight physical sources.
21	+	unsafe fn initialize() {
22	+	interrupts::initialize(&mut STORE, 8);
23	+	domains::initialize(&mut DOMAINS);
24	+	open(&mut SOURCE);
25	+	open(&mut TARGET);
26	+	}
27	+
28	+	/// Publish one receiver backed by statically retained test memory.
29	+	unsafe fn open(memory: &mut domains::Memory) {
30	+	let pending = try! slots::reserve(&mut DOMAINS.slots[..]);
31	+	let owner = try! slots::commit(&mut DOMAINS.slots[..], pending);
32	+	capability::initialize(&mut memory.table, owner);
33	+	set DOMAINS.records[owner.index].state = domains::Lifecycle::Active;
34	+	set DOMAINS.records[owner.index].memory = (&mut memory) as unsafe mut domains::Memory;
35	+	try! events::open(&mut DOMAINS.events, owner, &mut memory.ring);
36	+	}
37	+
38	+	/// Transfer preserves queued events and moves pending service to the new owner.
39	+	@test unsafe fn ownership() throws (testing::TestError) {
40	+	initialize();
41	+	let first = try! interrupts::seed(&mut STORE, &mut SOURCE.table, 5);
42	+	assert try! interrupts::query(&STORE, &SOURCE.table, first) == 5;
43	+	assert interrupts::claim(&mut STORE, 5);
44	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 5);
45	+	assert STORE.records[4].state == interrupts::State::Queued;
46	+	let second = try! interrupts::transfer(&mut STORE, &mut SOURCE.table, &mut TARGET.table,
47	+	first, (abi::READ \| abi::TRANSFER) as u64);
48	+	assert STORE.records[4].state == interrupts::State::Pending;
49	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 5);
50	+	assert DOMAINS.events.queues[0].tail == 1 and DOMAINS.events.queues[1].tail == 1;
51	+	let returned = try! interrupts::transfer(&mut STORE, &mut TARGET.table, &mut SOURCE.table,
52	+	second, (abi::READ \| abi::TRANSFER) as u64);
53	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 5);
54	+	assert DOMAINS.events.queues[0].tail == 1;
55	+	let consumed = try! events::pop(&mut SOURCE.ring) else panic "missing source event";
56	+	assert consumed.kind == events::INTERRUPT and consumed.code == 5;
57	+	assert try! interrupts::service(&mut STORE, &mut DOMAINS, 5);
58	+	assert STORE.records[4].state == interrupts::State::Idle;
59	+	let old = try! capability::get(&SOURCE.table, returned);
60	+	try! interrupts::drop(&mut STORE, &mut SOURCE.table, returned);
61	+	assert not interrupts::claim(&mut STORE, 5);
62	+	let replacement = try! interrupts::seed(&mut STORE, &mut SOURCE.table, 5);
63	+	let fresh = try! capability::get(&SOURCE.table, replacement);
64	+	assert fresh.object.index == old.object.index and fresh.object.generation <> old.object.generation;
65	+	}
66	+
67	+	/// A full event queue retains the source until capacity and consumer progress permit rearming.
68	+	@test unsafe fn saturation() throws (testing::TestError) {
69	+	initialize();
70	+	let handle = try! interrupts::seed(&mut STORE, &mut SOURCE.table, 3);
71	+	let owner = SOURCE.table.owner;
72	+	for i in 0..events::ORDINARY {
73	+	assert try! events::send(&mut DOMAINS.events, owner, &mut SOURCE.ring,
74	+	events::Event { kind: events::WAKEUP, reserved: 0, code: i, value: 0 });
75	+	}
76	+	assert interrupts::claim(&mut STORE, 3);
77	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
78	+	assert STORE.records[2].state == interrupts::State::Pending;
79	+	assert interrupts::claim(&mut STORE, 3);
80	+	let freed = try! events::pop(&mut SOURCE.ring) else panic "full ring";
81	+	assert freed.kind == events::WAKEUP;
82	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
83	+	assert STORE.records[2].state == interrupts::State::Queued;
84	+	for i in 1..events::ORDINARY {
85	+	let event = try! events::pop(&mut SOURCE.ring) else panic "retained wakeup";
86	+	assert event.kind == events::WAKEUP;
87	+	}
88	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
89	+	let event = try! events::pop(&mut SOURCE.ring) else panic "retained interrupt";
90	+	assert event.code == 3 and event.kind == events::INTERRUPT;
91	+	assert try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
92	+	set DOMAINS.records[owner.index].state = domains::Lifecycle::Dead;
93	+	assert interrupts::claim(&mut STORE, 3);
94	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
95	+	}
96	+
97	+	/// Target exhaustion leaves the source handle and retained notification unchanged.
98	+	@test unsafe fn atomicTransfer() throws (testing::TestError) {
99	+	initialize();
100	+	let handle = try! interrupts::seed(&mut STORE, &mut SOURCE.table, 7);
101	+	assert interrupts::claim(&mut STORE, 7);
102	+	for i in 0..limits::HANDLES {
103	+	let filler = try! capability::install(&mut TARGET.table, capability::Entry {
104	+	kind: abi::Kind::Domain, object: SOURCE.table.owner, rights: abi::Rights(0),
105	+	});
106	+	}
107	+	let mut full = false;
108	+	try interrupts::transfer(&mut STORE, &mut SOURCE.table, &mut TARGET.table, handle, abi::READ as u64) catch error {
109	+	assert error == abi::Error::InvalidArg; set full = true;
110	+	};
111	+	assert full;
112	+	assert STORE.records[6].owner == SOURCE.table.owner and STORE.records[6].handle == handle;
113	+	assert STORE.records[6].state == interrupts::State::Pending;
114	+	assert try! interrupts::query(&STORE, &SOURCE.table, handle) == 7;
115	+	let mut invalid = false;
116	+	try interrupts::seed(&mut STORE, &mut SOURCE.table, 0) catch error {
117	+	assert error == abi::Error::InvalidArg; set invalid = true;
118	+	};
119	+	assert invalid and not interrupts::claim(&mut STORE, 0) and not interrupts::claim(&mut STORE, 9);
120	+	}

kernel/kernel/tests/platform.rad +90 -2

//! Platform resource discovery and reserved-memory exclusion fixtures.

use std::testing;
use kernel::platform;
use kernel::range;
use kernel::plic;
use kernel::abi;

/// Fixed FDT construction workspace with separate structure and string regions.
record Fixture: Copy {
    /// Entire blob.
    bytes: [u8; 4096],

    offset: u32,
    /// Next property-name byte.
    strings: u32,
    /// Stack range value offset for malformed-input checks.
    stack: u32,
    /// PLIC context-list payload for mapping rejection checks.
    contexts: u32,
    /// PLIC source-count payload for capacity checks.
    sources: u32,
    /// PLIC register extent payload for bounds checks.
    plic: u32,
}

/// Write a big-endian fixture field.
fn put(f: &mut Fixture, offset: u32, width: u32, value: u64) {
    for i in 0..width { set f.bytes[offset + i] = (value >> ((width - i - 1) * 8) as u64) as u8; }

    return offset;
}

/// Construct the supported one-hart boot contract.
fn build(f: &mut Fixture) {
    construct(f, false);
}

/// Construct a boot contract with optional supervisor and machine PLIC contexts.
fn construct(f: &mut Fixture, interrupts: bool) {
    for i in 0..f.bytes.len { set f.bytes[i] = 0; }
    set f.offset = 56; set f.strings = 3072;
    begin(f, &""[..]);
    scalar(f, &"#address-cells"[..], 2); scalar(f, &"#size-cells"[..], 2);
    begin(f, &"chosen"[..]);

    let ram = region(f, &"reg"[..], 0x80000000, 0x10000); word(f, 2);
    begin(f, &"cpus"[..]);
    scalar(f, &"#address-cells"[..], 1); scalar(f, &"#size-cells"[..], 0);
    scalar(f, &"timebase-frequency"[..], 10000000);
    begin(f, &"cpu@0"[..]); string(f, &"device_type"[..], &"cpu"[..]);
    scalar(f, &"reg"[..], 0); word(f, 2); word(f, 2);
    scalar(f, &"reg"[..], 0);
    if interrupts {
        begin(f, &"interrupt-controller"[..]);
        string(f, &"compatible"[..], &"riscv,cpu-intc"[..]);
        scalar(f, &"phandle"[..], 17); scalar(f, &"#interrupt-cells"[..], 1);
        let marker = property(f, &"interrupt-controller"[..], 0);
        word(f, 2);
    }
    word(f, 2); word(f, 2);
    begin(f, &"reserved-memory"[..]);
    scalar(f, &"#address-cells"[..], 2); scalar(f, &"#size-cells"[..], 2);
    let ranges = property(f, &"ranges"[..], 0);
    begin(f, &"image@80002000"[..]);
    let image = region(f, &"reg"[..], 0x80002000, 0x3000); word(f, 2); word(f, 2);
    begin(f, &"soc"[..]);
    scalar(f, &"#address-cells"[..], 2); scalar(f, &"#size-cells"[..], 2);
    let bus = property(f, &"ranges"[..], 0);
    begin(f, &"uart@10000000"[..]);
    string(f, &"compatible"[..], &"ns16550a"[..]);
    let uart = region(f, &"reg"[..], 0x10000000, 0x100); word(f, 2); word(f, 2);
    let uart = region(f, &"reg"[..], 0x10000000, 0x100); word(f, 2);
    if interrupts {
        begin(f, &"plic@c000000"[..]);
        string(f, &"compatible"[..], &"riscv,plic0"[..]);
        set f.plic = region(f, &"reg"[..], 0xc000000, 0x4000000);
        scalar(f, &"#interrupt-cells"[..], 1);
        let sources = property(f, &"riscv,ndev"[..], 4);
        set f.sources = sources; put(f, sources, 4, 31);
        let contexts = property(f, &"interrupts-extended"[..], 16);
        set f.contexts = contexts;
        put(f, contexts, 4, 17); put(f, contexts + 4, 4, 9);
        put(f, contexts + 8, 4, 17); put(f, contexts + 12, 4, 11);
        word(f, 2);
    }
    word(f, 2);
    word(f, 2); word(f, 9);
    let total = f.strings; let size = f.offset - 56;
    put(f, 0, 4, 0xd00dfeed); put(f, 4, 4, total as u64);
    put(f, 8, 4, 56); put(f, 12, 4, 3072); put(f, 16, 4, 40);
    put(f, 20, 4, 17); put(f, 24, 4, 16);
    put(f, 32, 4, (total - 3072) as u64); put(f, 36, 4, size as u64);
}

/// PLIC machine contexts follow reference-list order rather than physical hart ID.
@test unsafe fn interruptContexts() throws (testing::TestError) {
    let mut f: Fixture = undefined; construct(&mut f, true);
    let mut p: platform::Platform = undefined;
    try! platform::decode(&f.bytes[..f.strings], &mut p);
    assert p.irqSources == 31 and p.irqContexts[0] == 1;
    assert p.deviceCount == 2 and p.devices[1].kind == platform::Kind::Plic;
    let initialContexts = f.contexts;
    put(&mut f, initialContexts + 4, 4, 0xffffffff);
    try! platform::decode(&f.bytes[..f.strings], &mut p);
    assert p.irqContexts[0] == 1;
    for change in 0..7 {
        construct(&mut f, true);
        let sources = f.sources;
        let contexts = f.contexts;
        let registers = f.plic;
        match change {
            case 0 => put(&mut f, sources, 4, 0),
            case 1 => put(&mut f, sources, 4, 129),
            case 2 => put(&mut f, contexts + 8, 4, 99),
            case 3 => put(&mut f, contexts + 4, 4, 11),
            case 4 => put(&mut f, contexts + 12, 4, 9),
            case 5 => put(&mut f, registers + 8, 8, 0x200000),
            else => put(&mut f, registers, 8, 0xc000002),
        }
        let mut rejected = false;
        try platform::decode(&f.bytes[..f.strings], &mut p) catch error {
            assert error == platform::Error::Invalid; set rejected = true;
        };
        assert rejected;
    }
}

/// PLIC register selection uses the decoded machine context and rejects ambiguous mappings.
@test unsafe fn plicMapping() throws (testing::TestError) {
    let mut f: Fixture = undefined; construct(&mut f, true);
    let mut p: platform::Platform = undefined;
    try! platform::decode(&f.bytes[..f.strings], &mut p);
    let selected = try! plic::mapping(&p) else panic "missing PLIC";
    assert selected.base == 0xc000000 and selected.count == 31 and selected.context == 1;
    set p.irqContexts[0] = 16;
    let mut rejected = false;
    try plic::mapping(&p) catch error { assert error == abi::Error::InvalidArg; set rejected = true; };
    assert rejected;
    set p.irqContexts[0] = 1;
    set p.devices[2] = p.devices[1]; set p.deviceCount = 3;
    let mut duplicate = false;
    try plic::mapping(&p) catch error { assert error == abi::Error::InvalidArg; set duplicate = true; };
    assert duplicate;
    set p.irqSources = 0;
    assert try! plic::mapping(&p) == nil;
}

/// Resources and their reservations come from the tree rather than fixed addresses.
@test unsafe fn discovery() throws (testing::TestError) {
    let mut f: Fixture = undefined; build(&mut f);
    let mut p: platform::Platform = undefined;
    try! platform::decode(&f.bytes[..f.strings], &mut p);

lib/std/lang/lower.rad +8 -1

    }
    dataSliceHeader(b, dataName, arrInfo.length);
}

/// Lower a constant expression payload into a builder without slot padding.
/// Compute the type layout only when undefined data needs a byte count.
/// Compute the type layout when nil or undefined data needs a byte count.
unsafe fn lowerConstDataPayloadInto(
    self: &mut Lowerer,
    node: *ast::Node,
    ty: resolver::Type,
    dataPrefix: *[u8],

                return;
            }
        }
    }
    match node.value {
        case ast::NodeValue::Nil => {
            let case resolver::Type::Optional(_) = ty else throw LowerError::NilInNonOptional;
            let layout = resolver::getTypeLayout(ty);
            dataBuilderPush(b, il::DataValue {
                item: il::DataItem::Val { typ: il::Type::W8, val: 0 }, count: layout.size,
            });
        },
        case ast::NodeValue::Undef => {
            let layout = resolver::getTypeLayout(ty);
            dataBuilderPush(b, il::DataValue {
                item: il::DataItem::Undef,
                count: layout.size

test/boot/run +2 -2

# Boot the kernel through its production initialization on each supported hart count.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/boot.ras"
cat test/boot/machine.ras kernel/kernel/*.ras > "$work/boot.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
for harts in 1 2 8; do
    # The deterministic emulator shares its tick budget across all harts.
    steps=$((10000000 * harts))
    if ! "$emulator" -machine -max-steps="$steps" -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then

if [ "$status" -ne 2 ] || [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
    cat "$work/log" >&2
    exit 1
fi
printf 'kernel startup: all eight harts reached machine idle without traps\n'
cat test/boot/trap.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/trap.ras"
cat test/boot/trap.ras kernel/kernel/*.ras > "$work/trap.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
status=0
"$emulator" -machine -harts=2 -max-steps=20000000 -run "$work/trap.rv64" > "$work/log" 2>&1 || status=$?
if [ "$status" -ne 2 ] || [ "$(grep -c '^kernel: unexpected trap$' "$work/log")" -ne 1 ] \
    || [ "$(grep -c 'wfi=1' "$work/log")" -ne 2 ]; then

test/dispatch/run +4 -1

# Preempt and resume U-mode and M-mode work through production dispatch.
set -eu
emulator=${RAD_EMULATOR:-emulator}
fixture=${1:-dispatch}
harts=${2:-1}
irq=${3:-0}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp "test/$fixture/spin.rad" "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"

printf '];\n' >> "$work/kernel/dispatchinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
set --
if [ "$irq" -ne 0 ]; then set -- "-irq=$irq" "-irq-at=1"; fi
for count in $harts; do
    "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
    "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 "$@" -run "$work/dispatch.rv64"
    printf '%s: %s-hart execution passed\n' "$fixture" "$count"
done

test/loader/run +1 -2

length=$(wc -c < "$work/loaded.ril")
printf '//! Binary package used by both execution paths.\n/// Complete trusted binary RIL input.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/loadinput.rad"
od -An -v -tu1 "$work/loaded.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/loadinput.rad"
printf '];\n' >> "$work/kernel/loadinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat test/loader/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras \
    kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/loader.ras"
cat test/loader/machine.ras kernel/kernel/*.ras > "$work/loader.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
    -run bin/kernel.build.rv64 -- "$work/std.ril" "$work/kernel.ril" "$work/loader.ras" "$work/loader.rv64"
"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/loader.rv64"
printf 'runtime loader: hosted and loaded entries agree after workspace release\n'

test/smp/kernel/dispatchcheck.rad +125 -1

use kernel::sync;
use kernel::remote;
use kernel::calls;
use kernel::events;
use kernel::timers;
use kernel::interrupts;
use kernel::plic;
use kernel::dispatchinput;

/// Bootstrap resource authority.
unsafe static TABLE: capability::Table = undefined;
/// Release/acquire barrier after fixture publication.

fn kernelGp() -> u64;
/// M-mode entry of the completion checker.
fn completion() -> u64;
/// Write one diagnostic byte to the fixture UART.
fn put(byte: u8);
/// Set the fixture UART interrupt-enable register.
fn uartInterrupt(value: u8);
/// Finish the machine test successfully.
fn finish();

/// Publish fixture state on the last initialized hart and wait on every hart.
export unsafe fn start(last: bool) {

    set USER = cap.object;
    let mut user = try! domains::get(&domains::STORE, USER);
    let self = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE),
    });
    assert boot::PLATFORM.irqSources == 31;
    let irq = try! interrupts::seed(&mut interrupts::STORE, &mut user.memory.table, 7);
    sync::storeRelease(memory(dataAddress(&user, &"spin::interrupt"[..])) as *mut u64, *irq);
    let target = try! registry::exported(&registry::STORE, user.image, &"spin::worker"[..]);
    let case shared::Target::Function(entry) = target else panic "worker entry";
    set COUNTERS = dataAddress(&user, &"spin::count"[..]);
    set MIGRATIONS = dataAddress(&user, &"spin::migrations"[..]);
    let notification = @sliceOf(memory(dataAddress(&user, &"spin::notification"[..])), 8);

        let destination = (hart + notifications - 1) % notifications;
        assert context.hart == nil or context.hart == destination;
        assert sync::loadAcquire(memory(MIGRATIONS + hart as u64 * 8) as *u64) == destination as u64 + 1;
        assert context.pending == nil;
    }
    assert domains::STORE.events.queues[USER.index].tail == notifications;
    assert domains::STORE.events.queues[USER.index].tail == notifications + 1;
    let user = try! domains::get(&domains::STORE, USER);
    assert user.memory.ring.data[0].kind == events::INTERRUPT and user.memory.ring.data[0].code == 7;
    sync::leave(guard);
    for byte in "smp completed transactions:" { put(byte); }
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
        put(32); number(counts[hart]);

            assert acknowledged;
        }
    }
    for byte in "smp request acknowledgements passed\n" { put(byte); }
    if (boot::PLATFORM.harts & 2) <> 0 { idleWakeup(); }
    interruptOwnership();
    interruptStorm();
    let maximum = sync::maximum();
    assert maximum > 0 and maximum < 1000000;
    for byte in "smp metadata instructions: 0x" { put(byte); }
    number(maximum);
    put(10);
    finish();
}

/// Transfer a queued IRQ, rearm after consumption, and mask it before capability drop.
unsafe fn interruptOwnership() {
    let guard = sync::enter();
    let mut user = try! domains::get(&domains::STORE, USER);
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    let device = try! plic::mapping(&boot::PLATFORM) else panic "missing PLIC";
    let enabled = memory(device.base + 0x2000 + device.context as u64 * 0x80) as *u32;
    assert (sync::read32(enabled) & (1 << 7)) == 0;
    assert interrupts::STORE.records[6].state == interrupts::State::Queued;
    let original = interrupts::STORE.records[6].handle;
    let target = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: CHECKER, rights: abi::Rights(0),
    });
    let back = try! capability::install(&mut checker.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(0),
    });
    let moved = try! calls::invoke(USER, 11, &[*original, *target, (abi::READ | abi::TRANSFER) as u64, 0], 0);
    assert interrupts::STORE.records[6].owner == CHECKER;
    let first = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
    assert interrupts::STORE.records[6].state == interrupts::State::Queued;
    assert (sync::read32(enabled) & (1 << 7)) == 0;
    let event = try! events::pop(&mut checker.memory.ring) else panic "missing transferred IRQ";
    assert event.kind == events::INTERRUPT and event.code == 7;
    let before = domains::STORE.events.queues[USER.index].tail;
    let returned = try! calls::invoke(CHECKER, 11, &[moved, *back, (abi::READ | abi::TRANSFER) as u64, 0], 0);
    let second = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
    assert domains::STORE.events.queues[USER.index].tail == before;
    assert (sync::read32(enabled) & (1 << 7)) == 0;
    let consumed = try! events::pop(&mut user.memory.ring) else panic "missing original IRQ";
    assert consumed.kind == events::INTERRUPT and consumed.code == 7;
    let rearmed = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
    assert interrupts::STORE.records[6].state == interrupts::State::Idle;
    assert (sync::read32(enabled) & (1 << 7)) <> 0;
    let dropped = try! calls::invoke(USER, 12, &[returned, 0, 0, 0], 0);
    assert dropped == 0 and interrupts::STORE.slots[6].state == slots::State::Free;
    assert (sync::read32(enabled) & (1 << 7)) == 0;
    sync::leave(guard);
    for byte in "smp IRQ transfer, rearm, and drop passed\n" { put(byte); }
}

/// Keep a level source asserted across full queues and repeated driver progress.
unsafe fn interruptStorm() {
    let guard = sync::enter();
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    let handle = try! interrupts::seed(&mut interrupts::STORE, &mut checker.memory.table, 10);
    let device = try! plic::mapping(&boot::PLATFORM) else panic "missing PLIC";
    let enabled = memory(device.base + 0x2000 + device.context as u64 * 0x80) as *u32;
    sync::leave(guard);
    for i in 0..events::ORDINARY {
        let filling = sync::enter();
        assert try! domains::notify(&mut domains::STORE, CHECKER,
            events::Event { kind: events::WAKEUP, reserved: 0, code: i, value: 0 });
        sync::leave(filling);
    }
    let arming = sync::enter();
    let ready = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
    uartInterrupt(2);
    sync::leave(arming);
    let mut pending = false;
    for attempt in 0..10000 {
        let checking = sync::enter();
        set pending = interrupts::STORE.records[9].state == interrupts::State::Pending;
        if pending {
            assert (sync::read32(enabled) & (1 << 10)) == 0;
            let queue = domains::STORE.events.queues[CHECKER.index];
            assert queue.tail - queue.head == events::ORDINARY;
        }
        sync::leave(checking);
        if pending { break; }
    }
    assert pending;
    for i in 0..events::ORDINARY {
        let draining = sync::enter();
        let mut receiver = try! domains::get(&domains::STORE, CHECKER);
        let event = try! events::pop(&mut receiver.memory.ring) else panic "missing saturation event";
        assert event.kind == events::WAKEUP and event.code == i;
        sync::leave(draining);
    }
    let posting = sync::enter();
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    try! timers::request(&mut timers::STORE, &domains::STORE, CHECKER, 0, clock, 2000000, 0x77);
    let request = try! dispatch::request(0, remote::Action::Reschedule);
    sync::leave(posting);
    let acknowledged = try! dispatch::awaitRequest(0, request);
    assert acknowledged == 0;
    let mut received: u32 = 0;
    let mut timed = false;
    for attempt in 0..1000 {
        let consuming = sync::enter();
        let mut receiver = try! domains::get(&domains::STORE, CHECKER);
        if let event = try! events::pop(&mut receiver.memory.ring) {
            if event.kind == events::INTERRUPT {
                assert event.code == 10;
                set received += 1;
            } else {
                assert event.kind == events::TIMEOUT and event.code == 0x77;
                assert dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) >= clock + 2000000;
                set timed = true;
            }
        }
        let notified = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
        sync::leave(consuming);
        if timed and received >= 16 { break; }
    }
    let stopping = sync::enter();
    uartInterrupt(0);
    let dropped = try! calls::invoke(CHECKER, 12, &[*handle, 0, 0, 0], 0);
    assert dropped == 0 and (sync::read32(enabled) & (1 << 10)) == 0;
    sync::leave(stopping);
    assert timed and received >= 16;
    for byte in "smp IRQ saturation and storm timeout passed\n" { put(byte); }
}

/// Resume an actual U-mode Wait call through an IPI to an idle remote hart.
unsafe fn idleWakeup() {
    let guard = sync::enter();
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    let handle = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,

test/smp/machine.ras +7 -0

.export @kernel::dispatchcheck::memory;
.export @kernel::dispatchcheck::kernelGp;
.export @kernel::dispatchcheck::completion;
.export @kernel::dispatchcheck::finish;
.export @kernel::dispatchcheck::put;
.export @kernel::dispatchcheck::uartInterrupt;
@kernel::dispatchcheck::memory
    ret;
@kernel::dispatchcheck::kernelGp
    mv %a0 %gp;
    ret;

    ebreak;
@kernel::dispatchcheck::put
    li %t0 0x10000000;
    sb %a0 0(%t0);
    ret;
@kernel::dispatchcheck::uartInterrupt
    li %t0 0x10000000;
    fence iorw iorw;
    sb %a0 1(%t0);
    fence iorw iorw;
    ret;
@kernel::dispatchcheck::finish
    li %t0 0x10001000;
    li %t1 0x5555;
    sw %t1 0(%t0);
    ebreak;

test/smp/spin.rad +5 -0

export static windows: [u64; 8] = [0; 8];

/// Capability used to notify the shared domain from each online hart.
export static notification: u64 = 0;

/// Shared-domain capability for the injected physical interrupt source.
export static interrupt: u64 = 0;

/// Observed destination hart plus one after each context changes budget.
export static migrations: [u64; 8] = [0; 8];

/// Wait handshake: 2/3 bracket wakeup; 4 requests a second wait, bracketed by 5/6.
export static wakeState: u64 = 0;

/// Nonzero selects the independently activated idle-wakeup context.
export static idle: u64 = 0;

/// Check hart identity and shared-table allocation through repeated direct calls.
fn run(hart: u64) {
    let source = sys::queryInterrupt(abi::Handle(interrupt));
    assert source.number == 7;
    let original = sys::currentContext();
    assert original.hart == hart;
    let budget = abi::Handle(windows[hart as u32]);
    let before = try! sys::queryBudget(budget);
    let right = try! sys::budgetSplit(budget, before.start + 1000);

test/tests/static.optional.nil.rad added +24 -0

1	+	//! returns: 0
2	+	//! Static nil values occupy their complete scalar, pointer, and aggregate layouts.
3	+
4	+	/// Payload with alignment and padding requirements.
5	+	record Payload: Copy { base: u64, count: u32 }
6	+	/// Adjacent fields must remain intact around a nil aggregate.
7	+	record Container: Copy { first: u64, item: ?Payload, last: u64 }
8	+
9	+	static VALUE: ?Payload = nil;
10	+	static NUMBER: ?u64 = nil;
11	+	static POINTER: ?*u64 = nil;
12	+	static ITEMS: [?Payload; 2] = [nil; 2];
13	+	static BOX: Container = Container { first: 0x1122334455667788, item: nil, last: 0x8877665544332211 };
14	+
15	+	/// Check complete nil layouts and subsequent mutable initialization.
16	+	@default fn main() -> i32 {
17	+	assert VALUE == nil and NUMBER == nil and POINTER == nil;
18	+	assert ITEMS[0] == nil and ITEMS[1] == nil;
19	+	assert BOX.item == nil and BOX.first == 0x1122334455667788 and BOX.last == 0x8877665544332211;
20	+	set VALUE = Payload { base: 42, count: 7 };
21	+	let value = VALUE else panic "missing initialized payload";
22	+	assert value.base == 42 and value.count == 7;
23	+	return 0;
24	+	}

165	180		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
166	181		let page = try pages::get(&pages::STORE, entry.object);
167	182		try reply(&domain, arguments, &[page.base, page.count as u64]);
168	183		return 0;
169	184		},
	185	+	case 46 => {
	186	+	let number = try interrupts::query(&interrupts::STORE, &domain.memory.table, handle);
	187	+	if arguments[2] <> @sizeOf(abi::InterruptInfo) as u64 or arguments[3] <> 0
	188	+	or not buffer(&domain, arguments[1], 4, abi::Rights(abi::WRITE), 4) { throw abi::Error::InvalidArg; }
	189	+	set *memory32(arguments[1]) = number;
	190	+	return 0;
	191	+	},
170	192		case 47 => {
171	193		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
172	194		let target = try domains::get(&domains::STORE, entry.object);
173	195		let mut parent: u64 = 0;
174	196		if let object = target.parent { set parent = abi::id(object); }

196	218		try reply(&domain, arguments, &[abi::id(object), entry, package.codeAddress, package.code.len as u64 * 4, package.memory as u64]);
197	219		return 0;
198	220		},
199	221		case 60 => {
200	222		if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
201		-	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
	223	+	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ), 8) {
202	224		throw abi::Error::InvalidArg;
203	225		}
204	226		let words = @sliceOf(memory(arguments[1]), 4);
205	227		let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
206	228		return abi::id(try domains::contextCreate(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

360	382		case 51 => return try runtimeLoad(&mut domain, loader::Request {
361	383		authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
362	384		}, guard),
363	385		case 60 => {
364	386		if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
365		-	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
	387	+	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ), 8) {
366	388		sync::leave(guard); throw abi::Error::InvalidArg;
367	389		}
368	390		let words = @sliceOf(memory(arguments[1]), 4);
369	391		let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
370	392		let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

153	155		/// Publish one event after capacity and receiver validation.
154	156		fn publish(queue: &mut Queue, ring: &mut Ring, event: Event, subject: abi::Ref) {
155	157		let index = queue.tail & (CAPACITY - 1);
156	158		set ring.data[index] = event;
157	159		set queue.subjects[index] = subject;
	160	+	let mut source: u32 = 0;
	161	+	if event.kind == INTERRUPT { set source = event.code; }
	162	+	set queue.sources[index] = source;
158	163		set queue.tail = next(queue.tail);
159	164		sync::storeRelease32(&mut ring.tail, queue.tail);
160	165		}
161	166
162	167		/// Send an ordinary event, returning false when its reserved occupancy limit is reached.
163	168		export fn send(store: &mut Store, receiver: abi::Ref, ring: &mut Ring, event: Event) -> bool throws (abi::Error) {
164	169		if event.reserved <> 0 or (event.kind <> INTERRUPT and event.kind <> TIMEOUT and event.kind <> WAKEUP) {
165	170		throw abi::Error::InvalidArg;
166	171		}
	172	+	if event.kind == INTERRUPT and event.code == 0 { throw abi::Error::InvalidArg; }
167	173		try refresh(store, receiver, ring);
	174	+	if event.kind == INTERRUPT and interruptPending(store, receiver, event.code) { return true; }
168	175		let queue = &mut store.queues[receiver.index];
169	176		if distance(queue.tail, queue.head) >= ORDINARY { return false; }
170	177		publish(queue, ring, event, abi::Ref { index: 0, generation: 0 });
171	178		return true;
172	179		}
173	180
	181	+	/// Test trusted IRQ metadata after the receiver's consumer progress is refreshed.
	182	+	export fn interruptPending(store: &Store, receiver: abi::Ref, source: u32) -> bool {
	183	+	if source == 0 or not live(store, receiver) { return false; }
	184	+	let queue = &store.queues[receiver.index];
	185	+	for i in 0..distance(queue.tail, queue.head) {
	186	+	let index = ((queue.head & (CAPACITY - 1)) + i) & (CAPACITY - 1);
	187	+	if queue.sources[index] == source { return true; }
	188	+	}
	189	+	return false;
	190	+	}
	191	+
174	192		/// Read one published event and release its slot. Callers serialize consumers.
175	193		export fn pop(ring: &mut Ring) -> ?Event throws (abi::Error) {
176	194		let head = sync::loadAcquire32(&ring.head);
177	195		let tail = sync::loadAcquire32(&ring.tail);
178	196		let count = distance(tail, head);

200	260
201	261		/// Extract native platform resources. Discard the output if decoding fails.
202	262		export unsafe fn decode(bytes: &[u8], platform: &mut Platform) throws (Error) {
203	263		set platform.ramCount = 0; set platform.reservedCount = 0; set platform.deviceCount = 0;
204	264		set platform.harts = 0; set platform.timebase = 0; set platform.stackCount = 0;
	265	+	set platform.irqSources = 0;
	266	+	for hart in 0..limits::HARTS { set platform.irqContexts[hart] = nil; }
	267	+	let mut references: [u32; limits::HARTS] = [0; limits::HARTS];
	268	+	let mut interruptController: ?Node = nil;
205	269		let tree = try fdt::header(bytes) catch err { throw Error::Tree(err); };
206	270		let mut at = tree.reservations.start;
207	271		while at < tree.reservations.end {
208	272		try protect(platform, try extent(bytes, at, 2, 2)); set at += 16;
209	273		}

66	66		lib/std.test.rv64.s \
67	67		lib/std.test.rv64.o
68	68
69	69		# Kernel modules and tests use a separate package and test entry point.
70	70		KERNEL_SRC := kernel/kernel.rad $(shell find kernel/kernel -name '.rad' ! -name 'tests.rad' ! -path '/tests/*' 2>/dev/null)
71		-	KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras
	71	+	KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras kernel/kernel/plic.ras
72	72		KERNEL_TEST_SRC := kernel/kernel/tests.rad $(shell find kernel/kernel/tests -name '*.rad' 2>/dev/null)
73	73		KERNEL := -pkg kernel $(patsubst %,-mod %,$(sort $(KERNEL_SRC)))
74	74		KERNEL_TEST := $(BIN_DIR)/kernel.test.rv64
75	75
76	76		kernel-test: emulator $(KERNEL_TEST)

210	210		@RAD_EMULATOR="$(EMU)" sh test/dispatch/run
211	211
212	212		# Concurrent user contexts and metadata transactions on every online hart.
213	213		.PHONY: smp-test
214	214		smp-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
215		-	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run smp "1 2 8"
	215	+	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run smp "1 2 8" 7

11	11		export mod platform;
12	12		export mod trap;
13	13		export mod capability;
14	14		export mod transactions;
15	15		export mod events;
	16	+	export mod interrupts;
	17	+	export mod plic;
16	18		export mod registry;
17	19		export mod frames;
18	20		export mod backing;
19	21		export mod pages;
20	22		export mod loader;

23	23		base: u64,
24	24		/// Number of contiguous physical frames.
25	25		count: u64,
26	26		}
27	27
	28	+	/// Physical source returned by QueryInterrupt.
	29	+	export record InterruptInfo: Copy {
	30	+	/// PLIC source number; zero does not name an interrupt.
	31	+	number: u32,
	32	+	}
	33	+
28	34		/// Domain identity and lifetime returned by QueryDomain.
29	35		export record DomainInfo: Copy {
30	36		/// Generation-bearing domain identity.
31	37		id: u64,
32	38		/// Historical creator identity.

11	11		use super::domains;
12	12		use super::budgets;
13	13		use super::dispatch;
14	14		use super::timers;
15	15		use super::remote;
	16	+	use super::interrupts;
	17	+	use super::plic;
16	18		use std::arch::rv64::shared::catalog;
17	19
18	20		/// Platform data published by hart zero before secondary initialization.
19	21		export unsafe static PLATFORM: platform::Platform = undefined;
20	22		/// Release/acquire publication flag for PLATFORM.

25	27		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
26	28
27	29		/// Handle timer boundaries and user calls, and stop on unsupported traps.
28	30		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
29	31		let cause = trap::classify(frame.cause);
30		-	if cause == trap::Cause::Timer or cause == trap::Cause::Software {
	32	+	if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
31	33		dispatch::interrupt(frame, hart);
32	34		return;
33	35		}
34	36		if cause == trap::Cause::UserCall and trap::fromUser(frame) {
35	37		dispatch::call(frame, hart);

80	82		try! pages::initialize(&mut pages::STORE, &PLATFORM);
81	83		domains::initialize(&mut domains::STORE);
82	84		budgets::initialize(&mut budgets::STORE);
83	85		timers::initialize(&mut timers::STORE);
84	86		remote::initialize(&mut remote::STORE, PLATFORM.harts);
	87	+	interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
	88	+	try! plic::initialize(&PLATFORM);
85	89		assert entryCount == 2;
86	90		registry::initialize(&mut registry::STORE);
87	91		let bootCatalog = @sliceOf(entries, entryCount);
88	92		try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
89	93		let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";

14	14		use super::loader;
15	15		use super::dispatch;
16	16		use super::timers;
17	17		use super::sync;
18	18		use super::remote;
	19	+	use super::interrupts;
	20	+	use super::plic;
19	21
20	22		/// Resolve the owning hart for an operation that can change CPU authority.
21	23		unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
22	24		match operation {
23	25		case 11, 12, 70, 71, 73 => {},

72	74		}
73	75		if entry.kind == abi::Kind::Budget {
74	76		if not moving { throw abi::Error::Denied; }
75	77		return try budgets::transfer(&mut budgets::STORE, &mut domain.memory.table, &mut destination.memory.table, handle, rights, now);
76	78		}
	79	+	if entry.kind == abi::Kind::Interrupt {
	80	+	if not moving { throw abi::Error::Denied; }
	81	+	return try interrupts::transfer(&mut interrupts::STORE, &mut domain.memory.table,
	82	+	&mut destination.memory.table, handle, rights);
	83	+	}
77	84		try resident(entry);
78	85		let pending = try transactions::reserve(&mut destination.memory.table, owner, handle, entry, rights, moving);
79	86		if moving { return transactions::transfer(&mut domain.memory.table, &mut destination.memory.table, pending); }
80	87		return transactions::grant(&mut destination.memory.table, pending);
81	88		}

85	92		let entry = try transactions::dropping(&domain.memory.table, handle, domain.events);
86	93		let table: &mut capability::Table = &mut domain.memory.table;
87	94		match entry.kind {
88	95		case abi::Kind::Page => try pages::drop(&mut pages::STORE, table, handle),
89	96		case abi::Kind::Budget => try budgets::drop(&mut budgets::STORE, table, handle),
	97	+	case abi::Kind::Interrupt => {
	98	+	let number = try interrupts::query(&interrupts::STORE, table, handle);
	99	+	plic::mask(number);
	100	+	try! interrupts::drop(&mut interrupts::STORE, table, handle);
	101	+	},
90	102		else => {
91	103		try resident(entry);
92	104		let removed = try! capability::invalidate(table, handle);
93	105		},
94	106		}
95	107		}
96	108
97	109		/// Map a checked, aligned caller record address to words.
98	110		fn memory(address: u64) -> *mut u64;
	111	+	/// Map a checked four-byte-aligned result buffer to its source-number word.
	112	+	fn memory32(address: u64) -> *mut u32;
99	113
100	114		/// Test whether one nonwrapping extent contains the complete result range.
101	115		fn contains(base: u64, size: u64, address: u64, length: u64) -> bool {
102	116		return address >= base and address - base <= size and length <= size - (address - base);
103	117		}
104	118
105	119		/// Validate aligned Page or private package-data storage with the given rights.
106	120		/// Package-state tables and kernel metadata cannot hold caller-supplied records.
107		-	unsafe fn buffer(domain: &domains::Domain, address: u64, size: u64, rights: abi::Rights) -> bool {
108		-	if (address & 7) <> 0 or size > 0xffffffffffffffff - address { return false; }
	121	+	unsafe fn buffer(domain: &domains::Domain, address: u64, size: u64, rights: abi::Rights, alignment: u64) -> bool {
	122	+	assert alignment == 4 or alignment == 8;
	123	+	if (address & (alignment - 1)) <> 0 or size > 0xffffffffffffffff - address { return false; }
109	124		if pages::accessible(&pages::STORE, &domain.memory.table, address, size, rights) { return true; }
110	125		for i in 0..limits::PACKAGES {
111	126		if registry::STORE.slots[i].state <> slots::State::Live { continue; }
112	127		let base = domain.graph.table[i];
113	128		if base == 0 { continue; }

118	133		}
119	134
120	135		/// Copy a fixed query record only after validating its exact buffer size.
121	136		unsafe fn reply(domain: &domains::Domain, arguments: &[u64], words: &[u64]) throws (abi::Error) {
122	137		let size = words.len as u64 * 8;
123		-	if arguments[2] <> size or arguments[3] <> 0 or not buffer(domain, arguments[1], size, abi::Rights(abi::WRITE)) {
	138	+	if arguments[2] <> size or arguments[3] <> 0 or not buffer(domain, arguments[1], size, abi::Rights(abi::WRITE), 8) {
124	139		throw abi::Error::InvalidArg;
125	140		}
126	141		let output = @sliceOf(memory(arguments[1]), words.len);
127	142		for i in 0..words.len { set output[i] = words[i]; }
128	143		}

1	1		//! Typed access to a checked user record buffer.
2	2		.text;
3	3		.export @kernel::calls::memory;
	4	+	.export @kernel::calls::memory32;
4	5
5	6		// The caller checked alignment, extent, and the required domain access rights.
6	7		@kernel::calls::memory
	8	+	@kernel::calls::memory32
7	9		ret;

10	10		use super::range;
11	11		use super::calls;
12	12		use super::timers;
13	13		use super::sync;
14	14		use super::remote;
	15	+	use super::plic;
15	16
16	17		/// Dispatch ownership and idle state retained by one physical hart.
17	18		export record State: Copy {
18	19		/// Physical hart whose windows this state selects.
19	20		hart: u32,

54	55		fn stackPointer() -> u64;
55	56		/// Trap anchor of the hart currently executing this continuation.
56	57		fn localAnchor() -> *mut trap::Hart;
57	58		/// Address of the M-mode idle loop.
58	59		fn idleAddress() -> u64;
59		-	/// Enable timer and software interrupt delivery with global interrupts disabled.
	60	+	/// Enable machine interrupt sources with global interrupts disabled.
60	61		fn enable();
61	62
62	63		/// Initialize this hart's idle frame and validated timer before installing dispatch.
63	64		export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {
64	65		let device = try timer(machine, id);

84	85		assert pointer >= stack.start and pointer < stack.end;
85	86		let guard = sync::enter();
86	87		writeSoftware(HARTS[id].timer.software, 0);
87	88		let previous = HARTS[id].state.current;
88	89		save(&mut HARTS[id].state, &mut domains::STORE, frame);
	90	+	if trap::classify(frame.cause) == trap::Cause::External { plic::claim(id); }
89	91		let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
90	92		let mut count: u32 = 0;
91	93		for i in 0..limits::REMOTE_PER_HART {
92	94		let delivery = try! remote::take(&mut remote::STORE, id) else { break; };
93	95		match delivery.request.action {

101	103		set deliveries[count] = delivery;
102	104		set count += 1;
103	105		}
104	106		let clock = now(HARTS[id].timer);
105	107		let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
	108	+	let notified = timeout.harts \| plic::service(clock);
106	109		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
107	110		let mut deadline = choice.deadline;
108	111		if timeout.deadline < deadline { set deadline = timeout.deadline; }
109	112		for target in 0..limits::HARTS {
110		-	if target <> id and (timeout.harts & (1 << target)) <> 0 { signal(target); }
	113	+	if target <> id and (notified & (1 << target)) <> 0 { signal(target); }
111	114		}
112	115		arm(HARTS[id].timer, deadline);
113	116		if previous <> choice.context {
114	117		if let context = previous { signalContext(context, clock); }
115	118		}

60	60		j @dispatchIdle;
61	61
62	62		// Global interrupt enable is restored from the selected frame by mret.
63	63		@kernel::dispatch::enable
64	64		csrr %t0 mie;
65		-	ori %t0 %t0 136;
	65	+	li %t1 2184;
	66	+	or %t0 %t0 %t1;
66	67		csrw mie %t0;
67	68		ret;
68	69
69	70		// The validated CLINT mapping contains a naturally aligned mtime register.
70	71		@kernel::dispatch::read

51	51		head: u32,
52	52		/// Authoritative producer counter.
53	53		tail: u32,
54	54		/// Terminal domain per occupied slot; generation zero denotes an ordinary event.
55	55		subjects: [abi::Ref; CAPACITY],
	56	+	/// Interrupt source per occupied slot, or zero for another event kind.
	57	+	sources: [u32; CAPACITY],
56	58		}
57	59
58	60		/// Lifetime state of a terminal-event record.
59	61		export union State: Copy {
60	62		/// Available for a new domain's terminal reservation.

53	53		devices: [Device; limits::DEVICES],
54	54		/// Number of MMIO devices.
55	55		deviceCount: u32,
56	56		/// Bit mask of online physical hart IDs.
57	57		harts: u32,
	58	+	/// PLIC source count, or zero when no supported controller is present.
	59	+	irqSources: u32,
	60	+	/// Machine-mode PLIC context index for each online hart.
	61	+	irqContexts: [?u32; limits::HARTS],
58	62		/// Timer ticks per second.
59	63		timebase: u32,
60	64		/// Kernel stack ranges, indexed by physical hart ID.
61	65		stacks: [range::Range; limits::HARTS],
62	66		/// Number of stack ranges in the boot contract.

79	83		registers: ?fdt::Span,
80	84		/// Device compatibility strings, if present.
81	85		compatible: ?fdt::Span,
82	86		/// Device type string, if present.
83	87		deviceType: ?fdt::Span,
	88	+	/// Interrupt-controller reference identity, or zero when absent.
	89	+	phandle: u32,
	90	+	/// Cells in one interrupt specifier.
	91	+	interruptCells: u32,
	92	+	/// PLIC source count, or zero when absent.
	93	+	sources: u32,
	94	+	/// PLIC context references and local interrupt numbers.
	95	+	interrupts: ?fdt::Span,
84	96		/// Node status permits use.
85	97		enabled: bool,
86	98		/// This node or its parent is the reserved-memory container.
87	99		reserved: bool,
88	100		}
89	101
	102	+	/// Read a CPU node's bounded physical hart identifier.
	103	+	fn cpu(bytes: &[u8], node: Node) -> u32 throws (Error) {
	104	+	let registers = node.registers else { throw Error::Invalid; };
	105	+	if node.sizeCells <> 0 or node.addressCells < 1 or node.addressCells > 2
	106	+	or registers.end - registers.start <> node.addressCells * 4 { throw Error::Invalid; }
	107	+	let hart = try fdt::integer(bytes, registers.start, node.addressCells * 4) catch err { throw Error::Tree(err); };
	108	+	if hart >= limits::HARTS as u64 { throw Error::Invalid; }
	109	+	return hart as u32;
	110	+	}
	111	+
	112	+	/// Associate a CPU-local interrupt controller with its parent hart.
	113	+	fn controller(bytes: &[u8], node: Node, parent: Node, references: &mut [u32]) throws (Error) {
	114	+	let kind = parent.deviceType else { throw Error::Invalid; };
	115	+	if not string(bytes, kind, &"cpu"[..]) or node.phandle == 0 or node.interruptCells <> 1 { throw Error::Invalid; }
	116	+	let hart = try cpu(bytes, parent);
	117	+	if references[hart] <> 0 { throw Error::Invalid; }
	118	+	for reference in &references[..] { if reference == node.phandle { throw Error::Invalid; } }
	119	+	set references[hart] = node.phandle;
	120	+	}
	121	+
	122	+	/// Resolve PLIC context ordinals after all CPU controller references are known.
	123	+	fn plic(bytes: &[u8], node: Node, references: &[u32], platform: &mut Platform) throws (Error) {
	124	+	if node.sources == 0 or node.sources > limits::INTERRUPTS or node.interruptCells <> 1 { throw Error::Invalid; }
	125	+	let contexts = node.interrupts else { throw Error::Invalid; };
	126	+	let length = contexts.end - contexts.start;
	127	+	if length == 0 or length % 8 <> 0 or length / 8 > limits::HARTS * 2 { throw Error::Invalid; }
	128	+	let registers = node.registers else { throw Error::Invalid; };
	129	+	if registers.end - registers.start <> (node.addressCells + node.sizeCells) * 4 { throw Error::Invalid; }
	130	+	let memory = try extent(bytes, registers.start, node.addressCells, node.sizeCells);
	131	+	let required = 0x200008 + (length as u64 / 8 - 1) * 0x1000;
	132	+	if (memory.start & 3) <> 0 or memory.end - memory.start < required { throw Error::Invalid; }
	133	+	for index in 0..length / 8 {
	134	+	let offset = contexts.start + index * 8;
	135	+	let reference = try fdt::integer(bytes, offset, 4) catch err { throw Error::Tree(err); };
	136	+	let cause = try fdt::integer(bytes, offset + 4, 4) catch err { throw Error::Tree(err); };
	137	+	let mut hart: ?u32 = nil;
	138	+	for id in 0..limits::HARTS { if references[id] <> 0 and references[id] as u64 == reference { set hart = id; } }
	139	+	let id = hart else { throw Error::Invalid; };
	140	+	if cause == 9 or cause == 0xffffffff { continue; }
	141	+	if cause <> 11 { throw Error::Invalid; }
	142	+	if (platform.harts & (1 << id)) == 0 { continue; }
	143	+	if platform.irqContexts[id] <> nil { throw Error::Invalid; }
	144	+	set platform.irqContexts[id] = index;
	145	+	}
	146	+	for hart in 0..limits::HARTS {
	147	+	if (platform.harts & (1 << hart)) <> 0 and platform.irqContexts[hart] == nil { throw Error::Invalid; }
	148	+	}
	149	+	set platform.irqSources = node.sources;
	150	+	}
	151	+
90	152		/// Compare a complete byte span with a name.
91	153		fn equal(bytes: &[u8], span: fdt::Span, value: &[u8]) -> bool {
92	154		if span.end - span.start <> value.len { return false; }
93	155		for i in 0..value.len { if bytes[span.start + i] <> value[i] { return false; } }
94	156		return true;

152	214		fn finish(bytes: &[u8], node: Node, platform: &mut Platform) throws (Error) {
153	215		if not node.enabled { return; }
154	216		let registers = node.registers else { return; };
155	217		if let typeName = node.deviceType {
156	218		if string(bytes, typeName, &"cpu"[..]) {
157		-	if node.sizeCells <> 0 or node.addressCells < 1 or node.addressCells > 2
158		-	or registers.end - registers.start <> node.addressCells * 4 { throw Error::Invalid; }
159		-	let hart = try fdt::integer(bytes, registers.start, node.addressCells * 4) catch err { throw Error::Tree(err); };
160		-	if hart >= limits::HARTS as u64 or (platform.harts & (1 << hart as u32)) <> 0 { throw Error::Invalid; }
161		-	set platform.harts \|= 1 << hart as u32;
	219	+	let hart = try cpu(bytes, node);
	220	+	if (platform.harts & (1 << hart)) <> 0 { throw Error::Invalid; }
	221	+	set platform.harts \|= 1 << hart;
162	222		return;
163	223		}
164	224		}
165	225		let width = (node.addressCells + node.sizeCells) * 4;
166	226		if width == 0 or (registers.end - registers.start) % width <> 0 { throw Error::Invalid; }

223	287		set reserved = parent.reserved;
224	288		set enabled = parent.enabled;
225	289		}
226	290		if state.depth == 2 and equal(bytes, name, &"reserved-memory"[..]) { set reserved = true; }
227	291		set nodes[state.depth - 1] = Node { name, addressCells, sizeCells, childAddress: 2, childSize: 1,
228		-	registers: nil, compatible: nil, deviceType: nil, enabled, reserved };
	292	+	registers: nil, compatible: nil, deviceType: nil, phandle: 0, interruptCells: 0,
	293	+	sources: 0, interrupts: nil, enabled, reserved };
229	294		},
230	295		case fdt::Event::Property { name, value } => {
231	296		if equal(bytes, name, &"#address-cells"[..]) {
232	297		let count = try scalar(bytes, value);
233	298		if count > 2 { throw Error::Invalid; } set nodes[state.depth - 1].childAddress = count;

235	300		let count = try scalar(bytes, value);
236	301		if count > 2 { throw Error::Invalid; } set nodes[state.depth - 1].childSize = count;
237	302		} else if equal(bytes, name, &"reg"[..]) { set nodes[state.depth - 1].registers = value; }
238	303		else if equal(bytes, name, &"compatible"[..]) { set nodes[state.depth - 1].compatible = value; }
239	304		else if equal(bytes, name, &"device_type"[..]) { set nodes[state.depth - 1].deviceType = value; }
	305	+	else if equal(bytes, name, &"phandle"[..]) { set nodes[state.depth - 1].phandle = try scalar(bytes, value); }
	306	+	else if equal(bytes, name, &"#interrupt-cells"[..]) { set nodes[state.depth - 1].interruptCells = try scalar(bytes, value); }
	307	+	else if equal(bytes, name, &"riscv,ndev"[..]) { set nodes[state.depth - 1].sources = try scalar(bytes, value); }
	308	+	else if equal(bytes, name, &"interrupts-extended"[..]) { set nodes[state.depth - 1].interrupts = value; }
240	309		else if equal(bytes, name, &"status"[..]) {
241	310		set nodes[state.depth - 1].enabled = nodes[state.depth - 1].enabled and (string(bytes, value, &"okay"[..]) or string(bytes, value, &"ok"[..]));
242	311		} else if equal(bytes, name, &"ranges"[..]) and value.start <> value.end { throw Error::Invalid; }
243	312		else if state.depth == 2 and equal(bytes, nodes[state.depth - 1].name, &"cpus"[..]) and equal(bytes, name, &"timebase-frequency"[..]) {
244	313		set platform.timebase = try scalar(bytes, value);

251	320		if (stack.start & 15) <> 0 or (stack.end & 15) <> 0 { throw Error::Invalid; }
252	321		set platform.stacks[i] = stack; try protect(platform, stack);
253	322		}
254	323		}
255	324		},
256		-	case fdt::Event::EndNode => try finish(bytes, nodes[state.depth], platform),
	325	+	case fdt::Event::EndNode => {
	326	+	let node = nodes[state.depth];
	327	+	if let compatible = node.compatible {
	328	+	if string(bytes, compatible, &"riscv,cpu-intc"[..]) {
	329	+	if state.depth == 0 { throw Error::Invalid; }
	330	+	try controller(bytes, node, nodes[state.depth - 1], &mut references[..]);
	331	+	}
	332	+	if node.enabled and string(bytes, compatible, &"riscv,plic0"[..]) {
	333	+	if interruptController <> nil { throw Error::Invalid; }
	334	+	set interruptController = node;
	335	+	}
	336	+	}
	337	+	try finish(bytes, node, platform);
	338	+	},
257	339		case fdt::Event::End => break,
258	340		}
259	341		}
260	342		if platform.ramCount == 0 or platform.harts == 0 or platform.timebase == 0 { throw Error::Invalid; }
	343	+	if let node = interruptController { try plic(bytes, node, &references[..], platform); }
261	344		for hart in 0..limits::HARTS {
262	345		if (platform.harts & (1 << hart)) == 0 { continue; }
263	346		if hart >= platform.stackCount or not inRam(platform, platform.stacks[hart]) { throw Error::Invalid; }
264	347		for other in 0..hart {
265	348		if (platform.harts & (1 << other)) <> 0 and overlaps(platform.stacks[hart], platform.stacks[other]) {

122	122		try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
123	123		return info;
124	124		}
125	125		}
126	126
	127	+	/// Read a physical interrupt source. An invalid handle is a program error.
	128	+	export fn queryInterrupt(handle: abi::Handle) -> abi::InterruptInfo {
	129	+	unsafe {
	130	+	let mut info: abi::InterruptInfo = undefined;
	131	+	try! result(ecall(abi::Operation::QueryInterrupt as u32, *handle as i64,
	132	+	(&mut info) as i64, @sizeOf(abi::InterruptInfo) as i64, 0));
	133	+	return info;
	134	+	}
	135	+	}
	136	+
127	137		/// Read domain identity and lifetime. An invalid handle is a program error.
128	138		export fn queryDomain(handle: abi::Handle) -> abi::DomainInfo {
129	139		unsafe {
130	140		let mut info: abi::DomainInfo = undefined;
131	141		try! result(ecall(abi::Operation::QueryDomain as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::DomainInfo) as i64, 0));

11	11		use kernel::registry;
12	12		use kernel::limits;
13	13		use kernel::trap;
14	14		use kernel::budgets;
15	15		use kernel::remote;
	16	+	use kernel::interrupts;
	17	+
	18	+	/// Interrupt queries write exactly four bytes at four-byte-aligned addresses.
	19	+	@test unsafe fn interruptRecord() throws (testing::TestError) {
	20	+	let owner = initialize();
	21	+	interrupts::initialize(&mut interrupts::STORE, 8);
	22	+	let handle = try! interrupts::seed(&mut interrupts::STORE, &mut MEMORY.table, 7);
	23	+	let page = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
	24	+	let base = (&RAM[OFFSET]) as u64;
	25	+	set RAM[OFFSET] = 0xaaaaaaaa55555555;
	26	+	set RAM[OFFSET + 1] = 0x1122334455667788;
	27	+	let result = try! calls::invoke(owner, 46, &[*handle, base + 4, 4, 0], 0);
	28	+	assert result == 0 and @sizeOf(abi::InterruptInfo) == 4;
	29	+	assert RAM[OFFSET] == 0x0000000755555555 and RAM[OFFSET + 1] == 0x1122334455667788;
	30	+	let mut rejected: u32 = 0;
	31	+	for args in &[
	32	+	[handle, base + 2, 4, 0], [handle, base + 4, 8, 0],
	33	+	[handle, base + 4096, 4, 0], [handle, base + 4, 4, 1],
	34	+	] {
	35	+	try calls::invoke(owner, 46, &args[..], 0) catch error {
	36	+	assert error == abi::Error::InvalidArg; set rejected += 1;
	37	+	};
	38	+	}
	39	+	assert rejected == 4 and RAM[OFFSET] == 0x0000000755555555;
	40	+	let boundary = try! calls::invoke(owner, 46, &[*handle, base + 4092, 4, 0], 0);
	41	+	assert boundary == 0 and RAM[OFFSET + 511] == 0x0000000700000000;
	42	+	let decoded = try! abi::decode(abi::Handle(page));
	43	+	set MEMORY.table.entries[decoded.object.index].rights = abi::Rights(abi::READ);
	44	+	try calls::invoke(owner, 46, &[*handle, base + 4, 4, 0], 0) catch error {
	45	+	assert error == abi::Error::InvalidArg; set rejected += 1;
	46	+	};
	47	+	assert rejected == 5;
	48	+	}
16	49
17	50		/// Deferred budget operations recheck authority and expiration at execution time.
18	51		@test unsafe fn deferredBudget() throws (testing::TestError) {
19	52		let owner = initialize();
20	53		let handle = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 1, 10, 100);

7	7
8	8		/// Kernel event metadata workspace.
9	9		unsafe static STORE: events::Store = undefined;
10	10		/// Consumer-visible ring for the receiver.
11	11		unsafe static RING: events::Ring = undefined;
	12	+	/// Separate receiver used to check per-domain IRQ coalescing.
	13	+	unsafe static OTHER: events::Ring = undefined;
12	14
13	15		/// Return the receiver's current domain reference.
14	16		fn receiver() -> abi::Ref { return abi::Ref { index: 0, generation: 1 }; }
15	17
16	18		/// Reset metadata and open one queue.

22	24		/// Build an ordinary wakeup with a recognizable payload.
23	25		fn wake(value: u64) -> events::Event {
24	26		return events::Event { kind: events::WAKEUP, reserved: 0, code: 17, value };
25	27		}
26	28
	29	+	/// Duplicate IRQ notifications retain one queue entry even at ordinary capacity.
	30	+	@test unsafe fn interruptCoalescing() throws (testing::TestError) {
	31	+	initialize();
	32	+	let irq = events::Event { kind: events::INTERRUPT, reserved: 0, code: 5, value: 0 };
	33	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	34	+	set RING.data[0].code = 99;
	35	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	36	+	assert STORE.queues[0].tail == 1;
	37	+	for i in 1..events::ORDINARY { assert try! events::send(&mut STORE, receiver(), &mut RING, wake(i as u64)); }
	38	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	39	+	assert STORE.queues[0].tail == events::ORDINARY;
	40	+	assert not try! events::send(&mut STORE, receiver(), &mut RING,
	41	+	events::Event { kind: events::INTERRUPT, reserved: 0, code: 6, value: 0 });
	42	+	let consumed = try! events::pop(&mut RING) else panic "missing interrupt";
	43	+	assert consumed.code == 99;
	44	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	45	+	assert STORE.queues[0].tail == events::ORDINARY + 1;
	46	+	}
	47	+
	48	+	/// Pending IRQ identity is scoped to the receiver and survives counter wrap.
	49	+	@test unsafe fn interruptReceivers() throws (testing::TestError) {
	50	+	initialize();
	51	+	let other = abi::Ref { index: 1, generation: 1 };
	52	+	try! events::open(&mut STORE, other, &mut OTHER);
	53	+	set STORE.queues[0].head = 0xfffffffe;
	54	+	set STORE.queues[0].tail = 0xfffffffe;
	55	+	set RING.head = 0xfffffffe;
	56	+	set RING.tail = 0xfffffffe;
	57	+	let irq = events::Event { kind: events::INTERRUPT, reserved: 0, code: 5, value: 0 };
	58	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	59	+	assert try! events::send(&mut STORE, receiver(), &mut RING,
	60	+	events::Event { kind: events::INTERRUPT, reserved: 0, code: 6, value: 0 });
	61	+	assert try! events::send(&mut STORE, other, &mut OTHER, irq);
	62	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	63	+	assert STORE.queues[0].tail == 0 and STORE.queues[1].tail == 1;
	64	+	assert events::interruptPending(&STORE, receiver(), 5);
	65	+	assert events::interruptPending(&STORE, other, 5);
	66	+	let first = try! events::pop(&mut RING) else panic "missing wrapped IRQ";
	67	+	assert first.code == 5;
	68	+	try! events::refresh(&mut STORE, receiver(), &RING);
	69	+	assert not events::interruptPending(&STORE, receiver(), 5);
	70	+	assert events::interruptPending(&STORE, receiver(), 6);
	71	+	assert events::interruptPending(&STORE, other, 5);
	72	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	73	+	assert STORE.queues[0].tail == 1;
	74	+	}
	75	+
27	76		/// Fill ordinary capacity while preserving the reserved lifecycle capacity.
28	77		@test unsafe fn saturation() throws (testing::TestError) {
29	78		initialize();
30	79		for i in 0..events::ORDINARY {
31	80		try testing::expect(try! events::send(&mut STORE, receiver(), &mut RING, wake(i as u64)));

1	1		//! Platform resource discovery and reserved-memory exclusion fixtures.
2	2
3	3		use std::testing;
4	4		use kernel::platform;
5	5		use kernel::range;
	6	+	use kernel::plic;
	7	+	use kernel::abi;
6	8
7	9		/// Fixed FDT construction workspace with separate structure and string regions.
8	10		record Fixture: Copy {
9	11		/// Entire blob.
10	12		bytes: [u8; 4096],

12	14		offset: u32,
13	15		/// Next property-name byte.
14	16		strings: u32,
15	17		/// Stack range value offset for malformed-input checks.
16	18		stack: u32,
	19	+	/// PLIC context-list payload for mapping rejection checks.
	20	+	contexts: u32,
	21	+	/// PLIC source-count payload for capacity checks.
	22	+	sources: u32,
	23	+	/// PLIC register extent payload for bounds checks.
	24	+	plic: u32,
17	25		}
18	26
19	27		/// Write a big-endian fixture field.
20	28		fn put(f: &mut Fixture, offset: u32, width: u32, value: u64) {
21	29		for i in 0..width { set f.bytes[offset + i] = (value >> ((width - i - 1) * 8) as u64) as u8; }