radiance · commit

kernel/kernel.rad +2 -0

export mod platform;
export mod trap;
export mod capability;
export mod transactions;
export mod events;
export mod interrupts;
export mod plic;
export mod registry;
export mod frames;
export mod backing;
export mod pages;
export mod loader;

kernel/kernel/abi.rad +6 -0

    base: u64,
    /// Number of contiguous physical frames.
    count: u64,
}

/// Physical source returned by QueryInterrupt.
export record InterruptInfo: Copy {
    /// PLIC source number; zero does not name an interrupt.
    number: u32,
}

/// Domain identity and lifetime returned by QueryDomain.
export record DomainInfo: Copy {
    /// Generation-bearing domain identity.
    id: u64,
    /// Historical creator identity.

kernel/kernel/boot.rad +5 -1

use super::domains;
use super::budgets;
use super::dispatch;
use super::timers;
use super::remote;
use super::interrupts;
use super::plic;
use std::arch::rv64::shared::catalog;

/// Platform data published by hart zero before secondary initialization.
export unsafe static PLATFORM: platform::Platform = undefined;
/// Release/acquire publication flag for PLATFORM.

unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Handle timer boundaries and user calls, and stop on unsupported traps.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
    let cause = trap::classify(frame.cause);
    if cause == trap::Cause::Timer or cause == trap::Cause::Software {
    if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
        dispatch::interrupt(frame, hart);
        return;
    }
    if cause == trap::Cause::UserCall and trap::fromUser(frame) {
        dispatch::call(frame, hart);

        try! pages::initialize(&mut pages::STORE, &PLATFORM);
        domains::initialize(&mut domains::STORE);
        budgets::initialize(&mut budgets::STORE);
        timers::initialize(&mut timers::STORE);
        remote::initialize(&mut remote::STORE, PLATFORM.harts);
        interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
        try! plic::initialize(&PLATFORM);
        assert entryCount == 2;
        registry::initialize(&mut registry::STORE);
        let bootCatalog = @sliceOf(entries, entryCount);
        try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
        let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";

kernel/kernel/calls.rad +31 -5

use super::loader;
use super::dispatch;
use super::timers;
use super::sync;
use super::remote;
use super::interrupts;
use super::plic;

/// Resolve the owning hart for an operation that can change CPU authority.
unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
    match operation {
        case 11, 12, 70, 71, 73 => {

        if not moving {
            throw abi::Error::Denied;
        }
        return try budgets::transfer(&mut budgets::STORE, &mut domain.memory.table, &mut destination.memory.table, handle, rights, now);
    }
    if entry.kind == abi::Kind::Interrupt {
        if not moving {
            throw abi::Error::Denied;
        }
        return try interrupts::transfer(&mut interrupts::STORE, &mut domain.memory.table,
            &mut destination.memory.table, handle, rights);
    }
    try resident(entry);
    let pending = try transactions::reserve(&mut destination.memory.table, owner, handle, entry, rights, moving);
    if moving {
        return transactions::transfer(&mut domain.memory.table, &mut destination.memory.table, pending);
    }

    let entry = try transactions::dropping(&domain.memory.table, handle, domain.events);
    let table: &mut capability::Table = &mut domain.memory.table;
    match entry.kind {
        case abi::Kind::Page => try pages::drop(&mut pages::STORE, table, handle),
        case abi::Kind::Budget => try budgets::drop(&mut budgets::STORE, table, handle),
        case abi::Kind::Interrupt => {
            let number = try interrupts::query(&interrupts::STORE, table, handle);
            plic::mask(number);
            try! interrupts::drop(&mut interrupts::STORE, table, handle);
        },
        else => {
            try resident(entry);
            let removed = try! capability::invalidate(table, handle);
        },
    }
}

/// Map a checked, aligned caller record address to words.
fn memory(address: u64) -> *mut u64;
/// Map a checked four-byte-aligned result buffer to its source-number word.
fn memory32(address: u64) -> *mut u32;

/// Test whether one nonwrapping extent contains the complete result range.
fn contains(base: u64, size: u64, address: u64, length: u64) -> bool {
    return address >= base and address - base <= size and length <= size - (address - base);
}

/// Validate aligned Page or private package-data storage with the given rights.
/// Package-state tables and kernel metadata cannot hold caller-supplied records.
unsafe fn buffer(domain: &domains::Domain, address: u64, size: u64, rights: abi::Rights) -> bool {
    if (address & 7) <> 0 or size > 0xffffffffffffffff - address {
unsafe fn buffer(domain: &domains::Domain, address: u64, size: u64, rights: abi::Rights, alignment: u64) -> bool {
    assert alignment == 4 or alignment == 8;
    if (address & (alignment - 1)) <> 0 or size > 0xffffffffffffffff - address {
        return false;
    }
    if pages::accessible(&pages::STORE, &domain.memory.table, address, size, rights) {
        return true;
    }

}

/// Copy a fixed query record only after validating its exact buffer size.
unsafe fn reply(domain: &domains::Domain, arguments: &[u64], words: &[u64]) throws (abi::Error) {
    let size = words.len as u64 * 8;
    if arguments[2] <> size or arguments[3] <> 0 or not buffer(domain, arguments[1], size, abi::Rights(abi::WRITE)) {
    if arguments[2] <> size or arguments[3] <> 0 or not buffer(domain, arguments[1], size, abi::Rights(abi::WRITE), 8) {
        throw abi::Error::InvalidArg;
    }
    let output = @sliceOf(memory(arguments[1]), words.len);
    for i in 0..words.len {
        set output[i] = words[i];

            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
            let page = try pages::get(&pages::STORE, entry.object);
            try reply(&domain, arguments, &[page.base, page.count as u64]);
            return 0;
        },
        case 46 => {
            let number = try interrupts::query(&interrupts::STORE, &domain.memory.table, handle);
            if arguments[2] <> @sizeOf(abi::InterruptInfo) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], 4, abi::Rights(abi::WRITE), 4) {
                    throw abi::Error::InvalidArg;
                }
            set *memory32(arguments[1]) = number;
            return 0;
        },
        case 47 => {
            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
            let target = try domains::get(&domains::STORE, entry.object);
            let mut parent: u64 = 0;
            if let object = target.parent {

            try reply(&domain, arguments, &[abi::id(object), entry, package.codeAddress, package.code.len as u64 * 4, package.memory as u64]);
            return 0;
        },
        case 60 => {
            if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ), 8) {
                throw abi::Error::InvalidArg;
            }
            let words = @sliceOf(memory(arguments[1]), 4);
            let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
            return abi::id(try domains::contextCreate(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

        case 51 => return try runtimeLoad(&mut domain, loader::Request {
            authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
        }, guard),
        case 60 => {
            if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ), 8) {
                sync::leave(guard); throw abi::Error::InvalidArg;
            }
            let words = @sliceOf(memory(arguments[1]), 4);
            let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
            let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

kernel/kernel/calls.ras +2 -0

//! Typed access to a checked user record buffer.
.text;
.export @kernel::calls::memory;
.export @kernel::calls::memory32;

// The caller checked alignment, extent, and the required domain access rights.
@kernel::calls::memory
@kernel::calls::memory32
    ret;

kernel/kernel/dispatch.rad +7 -2

use super::range;
use super::calls;
use super::timers;
use super::sync;
use super::remote;
use super::plic;

/// Dispatch ownership and idle state retained by one physical hart.
export record State: Copy {
    /// Physical hart whose windows this state selects.
    hart: u32,

fn stackPointer() -> u64;
/// Trap anchor of the hart currently executing this continuation.
fn localAnchor() -> *mut trap::Hart;
/// Address of the M-mode idle loop.
fn idleAddress() -> u64;
/// Enable timer and software interrupt delivery with global interrupts disabled.
/// Enable machine interrupt sources with global interrupts disabled.
fn enable();

/// Initialize this hart's idle frame and validated timer before installing dispatch.
export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {
    let device = try timer(machine, id);

    assert pointer >= stack.start and pointer < stack.end;
    let guard = sync::enter();
    writeSoftware(HARTS[id].timer.software, 0);
    let previous = HARTS[id].state.current;
    save(&mut HARTS[id].state, &mut domains::STORE, frame);
    if trap::classify(frame.cause) == trap::Cause::External {
        plic::claim(id);
    }
    let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
    let mut count: u32 = 0;
    for i in 0..limits::REMOTE_PER_HART {
        let delivery = try! remote::take(&mut remote::STORE, id) else {
            break;

        set deliveries[count] = delivery;
        set count += 1;
    }
    let clock = now(HARTS[id].timer);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
    let notified = timeout.harts | plic::service(clock);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    let mut deadline = choice.deadline;
    if timeout.deadline < deadline {
        set deadline = timeout.deadline;
    }
    for target in 0..limits::HARTS {
        if target <> id and (timeout.harts & (1 << target)) <> 0 {
        if target <> id and (notified & (1 << target)) <> 0 {
            signal(target);
        }
    }
    arm(HARTS[id].timer, deadline);
    if previous <> choice.context {

kernel/kernel/dispatch.ras +2 -1

    j @dispatchIdle;

// Global interrupt enable is restored from the selected frame by mret.
@kernel::dispatch::enable
    csrr %t0 mie;
    ori %t0 %t0 136;
    li %t1 2184;
    or %t0 %t0 %t1;
    csrw mie %t0;
    ret;

// The validated CLINT mapping contains a naturally aligned mtime register.
@kernel::dispatch::read

kernel/kernel/events.rad +28 -0

    head: u32,
    /// Authoritative producer counter.
    tail: u32,
    /// Terminal domain per occupied slot; generation zero denotes an ordinary event.
    subjects: [abi::Ref; CAPACITY],
    /// Interrupt source per occupied slot, or zero for another event kind.
    sources: [u32; CAPACITY],
}

/// Lifetime state of a terminal-event record.
export union State: Copy {
    /// Available for a new domain's terminal reservation.

/// Publish one event after capacity and receiver validation.
fn publish(queue: &mut Queue, ring: &mut Ring, event: Event, subject: abi::Ref) {
    let index = queue.tail & (CAPACITY - 1);
    set ring.data[index] = event;
    set queue.subjects[index] = subject;
    let mut source: u32 = 0;
    if event.kind == INTERRUPT {
        set source = event.code;
    }
    set queue.sources[index] = source;
    set queue.tail = next(queue.tail);
    sync::storeRelease32(&mut ring.tail, queue.tail);
}

/// Send an ordinary event, returning false when its reserved occupancy limit is reached.
export fn send(store: &mut Store, receiver: abi::Ref, ring: &mut Ring, event: Event) -> bool throws (abi::Error) {
    if event.reserved <> 0 or (event.kind <> INTERRUPT and event.kind <> TIMEOUT and event.kind <> WAKEUP) {
        throw abi::Error::InvalidArg;
    }
    if event.kind == INTERRUPT and event.code == 0 {
        throw abi::Error::InvalidArg;
    }
    try refresh(store, receiver, ring);
    if event.kind == INTERRUPT and interruptPending(store, receiver, event.code) {
        return true;
    }
    let queue = &mut store.queues[receiver.index];
    if distance(queue.tail, queue.head) >= ORDINARY {
        return false;
    }
    publish(queue, ring, event, abi::Ref { index: 0, generation: 0 });
    return true;
}

/// Test trusted IRQ metadata after the receiver's consumer progress is refreshed.
export fn interruptPending(store: &Store, receiver: abi::Ref, source: u32) -> bool {
    if source == 0 or not live(store, receiver) {
        return false;
    }
    let queue = &store.queues[receiver.index];
    for i in 0..distance(queue.tail, queue.head) {
        let index = ((queue.head & (CAPACITY - 1)) + i) & (CAPACITY - 1);
        if queue.sources[index] == source {
            return true;
        }
    }
    return false;
}

/// Read one published event and release its slot. Callers serialize consumers.
export fn pop(ring: &mut Ring) -> ?Event throws (abi::Error) {
    let head = sync::loadAcquire32(&ring.head);
    let tail = sync::loadAcquire32(&ring.tail);
    let count = distance(tail, head);

kernel/kernel/interrupts.rad added +161 -0

1	+	//! Exclusive IRQ ownership and retained notification state.
2	+	use super::abi;
3	+	use super::limits;
4	+	use super::slots;
5	+	use super::capability;
6	+	use super::transactions;
7	+	use super::domains;
8	+	use super::events;
9	+
10	+	/// Notification state while hardware delivery is serialized.
11	+	export union State: Copy {
12	+	/// No retained notification prevents the source from being enabled.
13	+	Idle,
14	+	/// The claimed source needs an event in its current receiver.
15	+	Pending,
16	+	/// A queued event must be consumed before the source is enabled.
17	+	Queued,
18	+	}
19	+
20	+	/// One physical source; its table index is the PLIC number minus one.
21	+	export record Interrupt: Copy {
22	+	/// Domain holding the sole interrupt capability.
23	+	owner: abi::Ref,
24	+	/// Sole live capability in the owner's table.
25	+	handle: abi::Handle,
26	+	/// Retained notification and rearming state.
27	+	state: State,
28	+	}
29	+
30	+	/// Fixed source identities and their exclusive receivers.
31	+	export record Store: Copy {
32	+	/// Number of physical sources reported by the platform.
33	+	count: u32,
34	+	/// Source object generations, indexed by PLIC number minus one.
35	+	slots: [slots::Slot; limits::INTERRUPTS],
36	+	/// Published ownership and notification state.
37	+	records: [Interrupt; limits::INTERRUPTS],
38	+	}
39	+
40	+	/// Global interrupt ownership serialized by the metadata lock.
41	+	export unsafe static STORE: Store = undefined;
42	+
43	+	/// Initialize the bounded physical-source profile before issuing capabilities.
44	+	export fn initialize(store: &mut Store, count: u32) {
45	+	assert count <= limits::INTERRUPTS;
46	+	set store.count = count;
47	+	slots::initialize(&mut store.slots[..]);
48	+	}
49	+
50	+	/// Resolve a source object after checking its generation.
51	+	export fn get(store: &Store, object: abi::Ref) -> Interrupt throws (abi::Error) {
52	+	if object.index >= store.count or not slots::matches(&store.slots[..], object, slots::State::Live) {
53	+	throw abi::Error::BadHandle;
54	+	}
55	+	return store.records[object.index];
56	+	}
57	+
58	+	/// Resolve the sole capability owner with the required rights.
59	+	fn owned(store: &Store, table: &capability::Table, handle: abi::Handle, rights: u16) -> abi::Ref throws (abi::Error) {
60	+	let entry = try capability::lookup(table, handle, abi::Kind::Interrupt, abi::Rights(rights));
61	+	let source = try get(store, entry.object);
62	+	if source.owner <> table.owner or source.handle <> handle {
63	+	throw abi::Error::BadHandle;
64	+	}
65	+	return entry.object;
66	+	}
67	+
68	+	/// Issue one source during trusted resource initialization.
69	+	export fn seed(store: &mut Store, table: &mut capability::Table, number: u32) -> abi::Handle throws (abi::Error) {
70	+	if number == 0 or number > store.count {
71	+	throw abi::Error::InvalidArg;
72	+	}
73	+	let index = number - 1;
74	+	if store.slots[index].state <> slots::State::Free {
75	+	throw abi::Error::Busy;
76	+	}
77	+	let pending = try slots::reserve(&mut table.slots[..]);
78	+	let object = abi::Ref { index, generation: store.slots[index].generation };
79	+	let handle = try! abi::handle(abi::Kind::Interrupt, slots::reference(&pending));
80	+	set store.slots[index].state = slots::State::Reserved;
81	+	set store.records[index] = Interrupt { owner: table.owner, handle, state: State::Idle };
82	+	let live = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
83	+	return capability::publish(table, pending, capability::Entry {
84	+	kind: abi::Kind::Interrupt, object: live, rights: abi::Rights(abi::READ \| abi::TRANSFER),
85	+	});
86	+	}
87	+
88	+	/// Read the source number through its sole capability.
89	+	export fn query(store: &Store, table: &capability::Table, handle: abi::Handle) -> u32 throws (abi::Error) {
90	+	return (try owned(store, table, handle, 0)).index + 1;
91	+	}
92	+
93	+	/// Move exclusive ownership after reserving the receiver's capability slot.
94	+	export fn transfer(store: &mut Store, source: &mut capability::Table, target: &mut capability::Table,
95	+	handle: abi::Handle, rights: u64) -> abi::Handle throws (abi::Error)
96	+	{
97	+	let object = try owned(store, source, handle, abi::TRANSFER);
98	+	let entry = try! capability::get(source, handle);
99	+	let pending = try transactions::reserve(target, source.owner, handle, entry, rights, true);
100	+	let result = transactions::transfer(source, target, pending);
101	+	set store.records[object.index].owner = target.owner;
102	+	set store.records[object.index].handle = result;
103	+	if store.records[object.index].state == State::Queued {
104	+	set store.records[object.index].state = State::Pending;
105	+	}
106	+	return result;
107	+	}
108	+
109	+	/// Invalidate a source capability after its hardware route has been masked.
110	+	export fn drop(store: &mut Store, table: &mut capability::Table, handle: abi::Handle) throws (abi::Error) {
111	+	let object = try owned(store, table, handle, 0);
112	+	let removed = try! capability::invalidate(table, handle);
113	+	try! slots::release(&mut store.slots[..], object);
114	+	}
115	+
116	+	/// Retain a claimed source without scanning domain capability tables.
117	+	export fn claim(store: &mut Store, number: u32) -> bool {
118	+	if number == 0 or number > store.count {
119	+	return false;
120	+	}
121	+	let index = number - 1;
122	+	if store.slots[index].state <> slots::State::Live {
123	+	return false;
124	+	}
125	+	if store.records[index].state == State::Idle {
126	+	set store.records[index].state = State::Pending;
127	+	}
128	+	return true;
129	+	}
130	+
131	+	/// Deliver retained work or acknowledge consumption; true permits hardware rearming.
132	+	export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, number: u32) -> bool throws (abi::Error) {
133	+	if number == 0 or number > store.count {
134	+	throw abi::Error::InvalidArg;
135	+	}
136	+	let index = number - 1;
137	+	if store.slots[index].state <> slots::State::Live {
138	+	return false;
139	+	}
140	+	let source = store.records[index];
141	+	let mut domain = try domains::get(receivers, source.owner) catch {
142	+	return false;
143	+	};
144	+	if domain.state == domains::Lifecycle::Dead {
145	+	return false;
146	+	}
147	+	try events::refresh(&mut receivers.events, source.owner, &domain.memory.ring);
148	+	if source.state == State::Queued {
149	+	if events::interruptPending(&receivers.events, source.owner, number) {
150	+	return false;
151	+	}
152	+	set store.records[index].state = State::Idle;
153	+	} else if source.state == State::Pending {
154	+	if try domains::notify(receivers, source.owner,
155	+	events::Event { kind: events::INTERRUPT, reserved: 0, code: number, value: 0 }) {
156	+	set store.records[index].state = State::Queued;
157	+	}
158	+	return false;
159	+	}
160	+	return true;
161	+	}

kernel/kernel/platform.rad +156 -11

    devices: [Device; limits::DEVICES],
    /// Number of MMIO devices.
    deviceCount: u32,
    /// Bit mask of online physical hart IDs.
    harts: u32,
    /// PLIC source count, or zero when no supported controller is present.
    irqSources: u32,
    /// Machine-mode PLIC context index for each online hart.
    irqContexts: [?u32; limits::HARTS],
    /// Timer ticks per second.
    timebase: u32,
    /// Kernel stack ranges, indexed by physical hart ID.
    stacks: [range::Range; limits::HARTS],
    /// Number of stack ranges in the boot contract.

    registers: ?fdt::Span,
    /// Device compatibility strings, if present.
    compatible: ?fdt::Span,
    /// Device type string, if present.
    deviceType: ?fdt::Span,
    /// Interrupt-controller reference identity, or zero when absent.
    phandle: u32,
    /// Cells in one interrupt specifier.
    interruptCells: u32,
    /// PLIC source count, or zero when absent.
    sources: u32,
    /// PLIC context references and local interrupt numbers.
    interrupts: ?fdt::Span,
    /// Node status permits use.
    enabled: bool,
    /// This node or its parent is the reserved-memory container.
    reserved: bool,
}

/// Read a CPU node's bounded physical hart identifier.
fn cpu(bytes: &[u8], node: Node) -> u32 throws (Error) {
    let registers = node.registers else {
        throw Error::Invalid;
    };
    if node.sizeCells <> 0 or node.addressCells < 1 or node.addressCells > 2
        or registers.end - registers.start <> node.addressCells * 4 {
            throw Error::Invalid;
        }
    let hart = try fdt::integer(bytes, registers.start, node.addressCells * 4) catch err {
        throw Error::Tree(err);
    };
    if hart >= limits::HARTS as u64 {
        throw Error::Invalid;
    }
    return hart as u32;
}

/// Associate a CPU-local interrupt controller with its parent hart.
fn controller(bytes: &[u8], node: Node, parent: Node, references: &mut [u32]) throws (Error) {
    let kind = parent.deviceType else {
        throw Error::Invalid;
    };
    if not string(bytes, kind, &"cpu"[..]) or node.phandle == 0 or node.interruptCells <> 1 {
        throw Error::Invalid;
    }
    let hart = try cpu(bytes, parent);
    if references[hart] <> 0 {
        throw Error::Invalid;
    }
    for reference in &references[..] {
        if reference == node.phandle {
            throw Error::Invalid;
        }
    }
    set references[hart] = node.phandle;
}

/// Resolve PLIC context ordinals after all CPU controller references are known.
fn plic(bytes: &[u8], node: Node, references: &[u32], platform: &mut Platform) throws (Error) {
    if node.sources == 0 or node.sources > limits::INTERRUPTS or node.interruptCells <> 1 {
        throw Error::Invalid;
    }
    let contexts = node.interrupts else {
        throw Error::Invalid;
    };
    let length = contexts.end - contexts.start;
    if length == 0 or length % 8 <> 0 or length / 8 > limits::HARTS * 2 {
        throw Error::Invalid;
    }
    let registers = node.registers else {
        throw Error::Invalid;
    };
    if registers.end - registers.start <> (node.addressCells + node.sizeCells) * 4 {
        throw Error::Invalid;
    }
    let memory = try extent(bytes, registers.start, node.addressCells, node.sizeCells);
    let required = 0x200008 + (length as u64 / 8 - 1) * 0x1000;
    if (memory.start & 3) <> 0 or memory.end - memory.start < required {
        throw Error::Invalid;
    }
    for index in 0..length / 8 {
        let offset = contexts.start + index * 8;
        let reference = try fdt::integer(bytes, offset, 4) catch err {
            throw Error::Tree(err);
        };
        let cause = try fdt::integer(bytes, offset + 4, 4) catch err {
            throw Error::Tree(err);
        };
        let mut hart: ?u32 = nil;
        for id in 0..limits::HARTS {
            if references[id] <> 0 and references[id] as u64 == reference {
                set hart = id;
            }
        }
        let id = hart else {
            throw Error::Invalid;
        };
        if cause == 9 or cause == 0xffffffff {
            continue;
        }
        if cause <> 11 {
            throw Error::Invalid;
        }
        if (platform.harts & (1 << id)) == 0 {
            continue;
        }
        if platform.irqContexts[id] <> nil {
            throw Error::Invalid;
        }
        set platform.irqContexts[id] = index;
    }
    for hart in 0..limits::HARTS {
        if (platform.harts & (1 << hart)) <> 0 and platform.irqContexts[hart] == nil {
            throw Error::Invalid;
        }
    }
    set platform.irqSources = node.sources;
}

/// Compare a complete byte span with a name.
fn equal(bytes: &[u8], span: fdt::Span, value: &[u8]) -> bool {
    if span.end - span.start <> value.len {
        return false;
    }

    let registers = node.registers else {
        return;
    };
    if let typeName = node.deviceType {
        if string(bytes, typeName, &"cpu"[..]) {
            if node.sizeCells <> 0 or node.addressCells < 1 or node.addressCells > 2
                or registers.end - registers.start <> node.addressCells * 4 {
                    throw Error::Invalid;
                }
            let hart = try fdt::integer(bytes, registers.start, node.addressCells * 4) catch err {
                throw Error::Tree(err);
            };
            if hart >= limits::HARTS as u64 or (platform.harts & (1 << hart as u32)) <> 0 {
            let hart = try cpu(bytes, node);
            if (platform.harts & (1 << hart)) <> 0 {
                throw Error::Invalid;
            }
            set platform.harts |= 1 << hart as u32;
            set platform.harts |= 1 << hart;
            return;
        }
    }
    let width = (node.addressCells + node.sizeCells) * 4;
    if width == 0 or (registers.end - registers.start) % width <> 0 {


/// Extract native platform resources. Discard the output if decoding fails.
export unsafe fn decode(bytes: &[u8], platform: &mut Platform) throws (Error) {
    set platform.ramCount = 0; set platform.reservedCount = 0; set platform.deviceCount = 0;
    set platform.harts = 0; set platform.timebase = 0; set platform.stackCount = 0;
    set platform.irqSources = 0;
    for hart in 0..limits::HARTS {
        set platform.irqContexts[hart] = nil;
    }
    let mut references: [u32; limits::HARTS] = [0; limits::HARTS];
    let mut interruptController: ?Node = nil;
    let tree = try fdt::header(bytes) catch err {
        throw Error::Tree(err);
    };
    let mut at = tree.reservations.start;
    while at < tree.reservations.end {

                }
                if state.depth == 2 and equal(bytes, name, &"reserved-memory"[..]) {
                    set reserved = true;
                }
                set nodes[state.depth - 1] = Node { name, addressCells, sizeCells, childAddress: 2, childSize: 1,
                    registers: nil, compatible: nil, deviceType: nil, enabled, reserved };
                    registers: nil, compatible: nil, deviceType: nil, phandle: 0, interruptCells: 0,
                    sources: 0, interrupts: nil, enabled, reserved };
            },
            case fdt::Event::Property { name, value } => {
                if equal(bytes, name, &"#address-cells"[..]) {
                    let count = try scalar(bytes, value);
                    if count > 2 {

                    set nodes[state.depth - 1].compatible = value;
                }
                else if equal(bytes, name, &"device_type"[..]) {
                    set nodes[state.depth - 1].deviceType = value;
                }
                else if equal(bytes, name, &"phandle"[..]) {
                    set nodes[state.depth - 1].phandle = try scalar(bytes, value);
                }
                else if equal(bytes, name, &"#interrupt-cells"[..]) {
                    set nodes[state.depth - 1].interruptCells = try scalar(bytes, value);
                }
                else if equal(bytes, name, &"riscv,ndev"[..]) {
                    set nodes[state.depth - 1].sources = try scalar(bytes, value);
                }
                else if equal(bytes, name, &"interrupts-extended"[..]) {
                    set nodes[state.depth - 1].interrupts = value;
                }
                else if equal(bytes, name, &"status"[..]) {
                    set nodes[state.depth - 1].enabled = nodes[state.depth - 1].enabled and (string(bytes, value, &"okay"[..]) or string(bytes, value, &"ok"[..]));
                } else if equal(bytes, name, &"ranges"[..]) and value.start <> value.end {
                    throw Error::Invalid;
                }

                        }
                        set platform.stacks[i] = stack; try protect(platform, stack);
                    }
                }
            },
            case fdt::Event::EndNode => try finish(bytes, nodes[state.depth], platform),
            case fdt::Event::EndNode => {
                let node = nodes[state.depth];
                if let compatible = node.compatible {
                    if string(bytes, compatible, &"riscv,cpu-intc"[..]) {
                        if state.depth == 0 {
                            throw Error::Invalid;
                        }
                        try controller(bytes, node, nodes[state.depth - 1], &mut references[..]);
                    }
                    if node.enabled and string(bytes, compatible, &"riscv,plic0"[..]) {
                        if interruptController <> nil {
                            throw Error::Invalid;
                        }
                        set interruptController = node;
                    }
                }
                try finish(bytes, node, platform);
            },
            case fdt::Event::End => break,
        }
    }
    if platform.ramCount == 0 or platform.harts == 0 or platform.timebase == 0 {
        throw Error::Invalid;
    }
    if let node = interruptController {
        try plic(bytes, node, &references[..], platform);
    }
    for hart in 0..limits::HARTS {
        if (platform.harts & (1 << hart)) == 0 {
            continue;
        }
        if hart >= platform.stackCount or not inRam(platform, platform.stacks[hart]) {

kernel/kernel/plic.rad added +141 -0

1	+	//! PLIC register access and bounded external-interrupt delivery.
2	+	use super::abi;
3	+	use super::limits;
4	+	use super::platform;
5	+	use super::sync;
6	+	use super::slots;
7	+	use super::interrupts;
8	+	use super::domains;
9	+	use super::budgets;
10	+
11	+	/// Validated registers for the kernel's machine interrupt receiver.
12	+	export record Mapping: Copy {
13	+	/// Physical controller base.
14	+	base: u64,
15	+	/// Supported nonzero source count.
16	+	count: u32,
17	+	/// PLIC machine context assigned to physical hart zero.
18	+	context: u32,
19	+	}
20	+
21	+	/// Controller state published before secondary harts start.
22	+	static CONTROLLER: ?Mapping = nil;
23	+
24	+	/// Map a validated aligned MMIO address to one device word.
25	+	fn pointer(address: u64) -> *unsafe mut u32;
26	+
27	+	/// Select the controller and validate its routed context's register extent.
28	+	export fn mapping(machine: &platform::Platform) -> ?Mapping throws (abi::Error) {
29	+	if machine.irqSources == 0 {
30	+	return nil;
31	+	}
32	+	if machine.irqSources > limits::INTERRUPTS or (machine.harts & 1) == 0 {
33	+	throw abi::Error::InvalidArg;
34	+	}
35	+	let context = machine.irqContexts[0] else {
36	+	throw abi::Error::InvalidArg;
37	+	};
38	+	if context >= limits::HARTS * 2 {
39	+	throw abi::Error::InvalidArg;
40	+	}
41	+	let mut result: ?Mapping = nil;
42	+	for i in 0..machine.deviceCount {
43	+	let device = machine.devices[i];
44	+	if device.kind <> platform::Kind::Plic {
45	+	continue;
46	+	}
47	+	let required = 0x200008 + context as u64 * 0x1000;
48	+	if result <> nil or (device.memory.start & 3) <> 0 or device.memory.end < device.memory.start
49	+	or device.memory.end - device.memory.start < required {
50	+	throw abi::Error::InvalidArg;
51	+	}
52	+	set result = Mapping { base: device.memory.start, count: machine.irqSources, context };
53	+	}
54	+	if result == nil {
55	+	throw abi::Error::InvalidArg;
56	+	}
57	+	return result;
58	+	}
59	+
60	+	/// Mask all machine contexts and initialize one priority class before dispatch.
61	+	export unsafe fn initialize(machine: &platform::Platform) throws (abi::Error) {
62	+	set CONTROLLER = try mapping(machine);
63	+	let device = CONTROLLER else {
64	+	return;
65	+	};
66	+	for hart in 0..limits::HARTS {
67	+	let context = machine.irqContexts[hart] else {
68	+	continue;
69	+	};
70	+	for word in 0..(device.count + 32) / 32 {
71	+	sync::write32(pointer(device.base + 0x2000 + context as u64 * 0x80 + word as u64 * 4), 0);
72	+	}
73	+	sync::write32(pointer(device.base + 0x200000 + context as u64 * 0x1000), 0);
74	+	}
75	+	for number in 1..device.count + 1 {
76	+	sync::write32(pointer(device.base + number as u64 * 4), 1);
77	+	}
78	+	}
79	+
80	+	/// Change one source enable bit while the metadata lock serializes the word.
81	+	unsafe fn enable(device: Mapping, number: u32, enabled: bool) {
82	+	assert number > 0 and number <= device.count;
83	+	let address = pointer(device.base + 0x2000 + device.context as u64 * 0x80 + (number / 32) as u64 * 4);
84	+	let before = sync::read32(address as *unsafe u32);
85	+	let bit = 1 << (number % 32);
86	+	let mut after = before & ~bit;
87	+	if enabled {
88	+	set after \|= bit;
89	+	}
90	+	if after <> before {
91	+	sync::write32(address, after);
92	+	}
93	+	}
94	+
95	+	/// Mask a source before its exclusive capability is invalidated.
96	+	export unsafe fn mask(number: u32) {
97	+	let device = CONTROLLER else {
98	+	return;
99	+	};
100	+	enable(device, number, false);
101	+	}
102	+
103	+	/// Claim one source, retain its work, complete it while enabled, then mask it.
104	+	export unsafe fn claim(hart: u32) {
105	+	let device = CONTROLLER else {
106	+	return;
107	+	};
108	+	assert hart == 0;
109	+	let address = pointer(device.base + 0x200004 + device.context as u64 * 0x1000);
110	+	let number = sync::read32(address as *unsafe u32);
111	+	if number == 0 {
112	+	return;
113	+	}
114	+	assert number <= device.count;
115	+	let retained = interrupts::claim(&mut interrupts::STORE, number);
116	+	sync::write32(address, number);
117	+	enable(device, number, false);
118	+	}
119	+
120	+	/// Deliver retained notifications, rearm consumed sources, and report runnable harts.
121	+	export unsafe fn service(now: u64) -> u32 {
122	+	let device = CONTROLLER else {
123	+	return 0;
124	+	};
125	+	let mut harts: u32 = 0;
126	+	for number in 1..device.count + 1 {
127	+	let index = number - 1;
128	+	let mut pending = false;
129	+	if interrupts::STORE.slots[index].state == slots::State::Live {
130	+	set pending = interrupts::STORE.records[index].state == interrupts::State::Pending;
131	+	}
132	+	let enabled = try interrupts::service(&mut interrupts::STORE, &mut domains::STORE, number) catch {
133	+	false
134	+	};
135	+	enable(device, number, enabled);
136	+	if pending and interrupts::STORE.records[index].state == interrupts::State::Queued {
137	+	set harts \|= budgets::wakeTargets(&budgets::STORE, &domains::STORE, interrupts::STORE.records[index].owner, now);
138	+	}
139	+	}
140	+	return harts;
141	+	}

kernel/kernel/plic.ras added +7 -0

1	+	//! Typed access to validated, naturally aligned PLIC registers.
2	+	.text;
3	+	.export @kernel::plic::pointer;
4	+
5	+	// Register extents and alignment are checked before device access.
6	+	@kernel::plic::pointer
7	+	ret;

kernel/kernel/sys.rad +10 -0

        try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
        return info;
    }
}

/// Read a physical interrupt source. An invalid handle is a program error.
export fn queryInterrupt(handle: abi::Handle) -> abi::InterruptInfo {
    unsafe {
        let mut info: abi::InterruptInfo = undefined;
        try! result(ecall(abi::Operation::QueryInterrupt as u32, *handle as i64,
            (&mut info) as i64, @sizeOf(abi::InterruptInfo) as i64, 0));
        return info;
    }
}

/// Read domain identity and lifetime. An invalid handle is a program error.
export fn queryDomain(handle: abi::Handle) -> abi::DomainInfo {
    unsafe {
        let mut info: abi::DomainInfo = undefined;
        try! result(ecall(abi::Operation::QueryDomain as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::DomainInfo) as i64, 0));

kernel/kernel/tests.rad +1 -0

export mod frames;
export mod backing;
export mod pages;
export mod transactions;
export mod events;
export mod interrupts;
export mod registry;
export mod loader;
export mod instances;
export mod domains;
export mod budgets;

kernel/kernel/tests/calls.rad +33 -0

use kernel::registry;
use kernel::limits;
use kernel::trap;
use kernel::budgets;
use kernel::remote;
use kernel::interrupts;

/// Interrupt queries write exactly four bytes at four-byte-aligned addresses.
@test unsafe fn interruptRecord() throws (testing::TestError) {
    let owner = initialize();
    interrupts::initialize(&mut interrupts::STORE, 8);
    let handle = try! interrupts::seed(&mut interrupts::STORE, &mut MEMORY.table, 7);
    let page = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
    let base = (&RAM[OFFSET]) as u64;
    set RAM[OFFSET] = 0xaaaaaaaa55555555;
    set RAM[OFFSET + 1] = 0x1122334455667788;
    let result = try! calls::invoke(owner, 46, &[*handle, base + 4, 4, 0], 0);
    assert result == 0 and @sizeOf(abi::InterruptInfo) == 4;
    assert RAM[OFFSET] == 0x0000000755555555 and RAM[OFFSET + 1] == 0x1122334455667788;
    let mut rejected: u32 = 0;
    for args in &[
        [*handle, base + 2, 4, 0], [*handle, base + 4, 8, 0],
        [*handle, base + 4096, 4, 0], [*handle, base + 4, 4, 1],
    ] {
        try calls::invoke(owner, 46, &args[..], 0) catch error {
            assert error == abi::Error::InvalidArg; set rejected += 1;
        };
    }
    assert rejected == 4 and RAM[OFFSET] == 0x0000000755555555;
    let boundary = try! calls::invoke(owner, 46, &[*handle, base + 4092, 4, 0], 0);
    assert boundary == 0 and RAM[OFFSET + 511] == 0x0000000700000000;
    let decoded = try! abi::decode(abi::Handle(page));
    set MEMORY.table.entries[decoded.object.index].rights = abi::Rights(abi::READ);
    try calls::invoke(owner, 46, &[*handle, base + 4, 4, 0], 0) catch error {
        assert error == abi::Error::InvalidArg; set rejected += 1;
    };
    assert rejected == 5;
}

/// Deferred budget operations recheck authority and expiration at execution time.
@test unsafe fn deferredBudget() throws (testing::TestError) {
    let owner = initialize();
    let handle = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 1, 10, 100);

kernel/kernel/tests/events.rad +51 -0


/// Kernel event metadata workspace.
unsafe static STORE: events::Store = undefined;
/// Consumer-visible ring for the receiver.
unsafe static RING: events::Ring = undefined;
/// Separate receiver used to check per-domain IRQ coalescing.
unsafe static OTHER: events::Ring = undefined;

/// Return the receiver's current domain reference.
fn receiver() -> abi::Ref {
    return abi::Ref { index: 0, generation: 1 };
}

/// Build an ordinary wakeup with a recognizable payload.
fn wake(value: u64) -> events::Event {
    return events::Event { kind: events::WAKEUP, reserved: 0, code: 17, value };
}

/// Duplicate IRQ notifications retain one queue entry even at ordinary capacity.
@test unsafe fn interruptCoalescing() throws (testing::TestError) {
    initialize();
    let irq = events::Event { kind: events::INTERRUPT, reserved: 0, code: 5, value: 0 };
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    set RING.data[0].code = 99;
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == 1;
    for i in 1..events::ORDINARY {
        assert try! events::send(&mut STORE, receiver(), &mut RING, wake(i as u64));
    }
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == events::ORDINARY;
    assert not try! events::send(&mut STORE, receiver(), &mut RING,
        events::Event { kind: events::INTERRUPT, reserved: 0, code: 6, value: 0 });
    let consumed = try! events::pop(&mut RING) else panic "missing interrupt";
    assert consumed.code == 99;
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == events::ORDINARY + 1;
}

/// Pending IRQ identity is scoped to the receiver and survives counter wrap.
@test unsafe fn interruptReceivers() throws (testing::TestError) {
    initialize();
    let other = abi::Ref { index: 1, generation: 1 };
    try! events::open(&mut STORE, other, &mut OTHER);
    set STORE.queues[0].head = 0xfffffffe;
    set STORE.queues[0].tail = 0xfffffffe;
    set RING.head = 0xfffffffe;
    set RING.tail = 0xfffffffe;
    let irq = events::Event { kind: events::INTERRUPT, reserved: 0, code: 5, value: 0 };
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert try! events::send(&mut STORE, receiver(), &mut RING,
        events::Event { kind: events::INTERRUPT, reserved: 0, code: 6, value: 0 });
    assert try! events::send(&mut STORE, other, &mut OTHER, irq);
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == 0 and STORE.queues[1].tail == 1;
    assert events::interruptPending(&STORE, receiver(), 5);
    assert events::interruptPending(&STORE, other, 5);
    let first = try! events::pop(&mut RING) else panic "missing wrapped IRQ";
    assert first.code == 5;
    try! events::refresh(&mut STORE, receiver(), &RING);
    assert not events::interruptPending(&STORE, receiver(), 5);
    assert events::interruptPending(&STORE, receiver(), 6);
    assert events::interruptPending(&STORE, other, 5);
    assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
    assert STORE.queues[0].tail == 1;
}

/// Fill ordinary capacity while preserving the reserved lifecycle capacity.
@test unsafe fn saturation() throws (testing::TestError) {
    initialize();
    for i in 0..events::ORDINARY {
        try testing::expect(try! events::send(&mut STORE, receiver(), &mut RING, wake(i as u64)));

kernel/kernel/tests/interrupts.rad added +120 -0

1	+	//! Exclusive source ownership, deferred notification, and rearming.
2	+	use std::testing;
3	+	use kernel::abi;
4	+	use kernel::slots;
5	+	use kernel::capability;
6	+	use kernel::domains;
7	+	use kernel::events;
8	+	use kernel::interrupts;
9	+	use kernel::limits;
10	+
11	+	/// Physical source metadata under test.
12	+	unsafe static STORE: interrupts::Store = undefined;
13	+	/// Receiver lifetimes and trusted event metadata.
14	+	unsafe static DOMAINS: domains::Store = undefined;
15	+	/// Source domain capability table and ring.
16	+	unsafe static SOURCE: domains::Memory = undefined;
17	+	/// Target domain capability table and ring.
18	+	unsafe static TARGET: domains::Memory = undefined;
19	+
20	+	/// Initialize two live receivers and eight physical sources.
21	+	unsafe fn initialize() {
22	+	interrupts::initialize(&mut STORE, 8);
23	+	domains::initialize(&mut DOMAINS);
24	+	open(&mut SOURCE);
25	+	open(&mut TARGET);
26	+	}
27	+
28	+	/// Publish one receiver backed by statically retained test memory.
29	+	unsafe fn open(memory: &mut domains::Memory) {
30	+	let pending = try! slots::reserve(&mut DOMAINS.slots[..]);
31	+	let owner = try! slots::commit(&mut DOMAINS.slots[..], pending);
32	+	capability::initialize(&mut memory.table, owner);
33	+	set DOMAINS.records[owner.index].state = domains::Lifecycle::Active;
34	+	set DOMAINS.records[owner.index].memory = (&mut memory) as unsafe mut domains::Memory;
35	+	try! events::open(&mut DOMAINS.events, owner, &mut memory.ring);
36	+	}
37	+
38	+	/// Transfer preserves queued events and moves pending service to the new owner.
39	+	@test unsafe fn ownership() throws (testing::TestError) {
40	+	initialize();
41	+	let first = try! interrupts::seed(&mut STORE, &mut SOURCE.table, 5);
42	+	assert try! interrupts::query(&STORE, &SOURCE.table, first) == 5;
43	+	assert interrupts::claim(&mut STORE, 5);
44	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 5);
45	+	assert STORE.records[4].state == interrupts::State::Queued;
46	+	let second = try! interrupts::transfer(&mut STORE, &mut SOURCE.table, &mut TARGET.table,
47	+	first, (abi::READ \| abi::TRANSFER) as u64);
48	+	assert STORE.records[4].state == interrupts::State::Pending;
49	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 5);
50	+	assert DOMAINS.events.queues[0].tail == 1 and DOMAINS.events.queues[1].tail == 1;
51	+	let returned = try! interrupts::transfer(&mut STORE, &mut TARGET.table, &mut SOURCE.table,
52	+	second, (abi::READ \| abi::TRANSFER) as u64);
53	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 5);
54	+	assert DOMAINS.events.queues[0].tail == 1;
55	+	let consumed = try! events::pop(&mut SOURCE.ring) else panic "missing source event";
56	+	assert consumed.kind == events::INTERRUPT and consumed.code == 5;
57	+	assert try! interrupts::service(&mut STORE, &mut DOMAINS, 5);
58	+	assert STORE.records[4].state == interrupts::State::Idle;
59	+	let old = try! capability::get(&SOURCE.table, returned);
60	+	try! interrupts::drop(&mut STORE, &mut SOURCE.table, returned);
61	+	assert not interrupts::claim(&mut STORE, 5);
62	+	let replacement = try! interrupts::seed(&mut STORE, &mut SOURCE.table, 5);
63	+	let fresh = try! capability::get(&SOURCE.table, replacement);
64	+	assert fresh.object.index == old.object.index and fresh.object.generation <> old.object.generation;
65	+	}
66	+
67	+	/// A full event queue retains the source until capacity and consumer progress permit rearming.
68	+	@test unsafe fn saturation() throws (testing::TestError) {
69	+	initialize();
70	+	let handle = try! interrupts::seed(&mut STORE, &mut SOURCE.table, 3);
71	+	let owner = SOURCE.table.owner;
72	+	for i in 0..events::ORDINARY {
73	+	assert try! events::send(&mut DOMAINS.events, owner, &mut SOURCE.ring,
74	+	events::Event { kind: events::WAKEUP, reserved: 0, code: i, value: 0 });
75	+	}
76	+	assert interrupts::claim(&mut STORE, 3);
77	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
78	+	assert STORE.records[2].state == interrupts::State::Pending;
79	+	assert interrupts::claim(&mut STORE, 3);
80	+	let freed = try! events::pop(&mut SOURCE.ring) else panic "full ring";
81	+	assert freed.kind == events::WAKEUP;
82	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
83	+	assert STORE.records[2].state == interrupts::State::Queued;
84	+	for i in 1..events::ORDINARY {
85	+	let event = try! events::pop(&mut SOURCE.ring) else panic "retained wakeup";
86	+	assert event.kind == events::WAKEUP;
87	+	}
88	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
89	+	let event = try! events::pop(&mut SOURCE.ring) else panic "retained interrupt";
90	+	assert event.code == 3 and event.kind == events::INTERRUPT;
91	+	assert try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
92	+	set DOMAINS.records[owner.index].state = domains::Lifecycle::Dead;
93	+	assert interrupts::claim(&mut STORE, 3);
94	+	assert not try! interrupts::service(&mut STORE, &mut DOMAINS, 3);
95	+	}
96	+
97	+	/// Target exhaustion leaves the source handle and retained notification unchanged.
98	+	@test unsafe fn atomicTransfer() throws (testing::TestError) {
99	+	initialize();
100	+	let handle = try! interrupts::seed(&mut STORE, &mut SOURCE.table, 7);
101	+	assert interrupts::claim(&mut STORE, 7);
102	+	for i in 0..limits::HANDLES {
103	+	let filler = try! capability::install(&mut TARGET.table, capability::Entry {
104	+	kind: abi::Kind::Domain, object: SOURCE.table.owner, rights: abi::Rights(0),
105	+	});
106	+	}
107	+	let mut full = false;
108	+	try interrupts::transfer(&mut STORE, &mut SOURCE.table, &mut TARGET.table, handle, abi::READ as u64) catch error {
109	+	assert error == abi::Error::InvalidArg; set full = true;
110	+	};
111	+	assert full;
112	+	assert STORE.records[6].owner == SOURCE.table.owner and STORE.records[6].handle == handle;
113	+	assert STORE.records[6].state == interrupts::State::Pending;
114	+	assert try! interrupts::query(&STORE, &SOURCE.table, handle) == 7;
115	+	let mut invalid = false;
116	+	try interrupts::seed(&mut STORE, &mut SOURCE.table, 0) catch error {
117	+	assert error == abi::Error::InvalidArg; set invalid = true;
118	+	};
119	+	assert invalid and not interrupts::claim(&mut STORE, 0) and not interrupts::claim(&mut STORE, 9);
120	+	}

kernel/kernel/tests/platform.rad +96 -2

//! Platform resource discovery and reserved-memory exclusion fixtures.

use std::testing;
use kernel::platform;
use kernel::range;
use kernel::plic;
use kernel::abi;

/// Fixed FDT construction workspace with separate structure and string regions.
record Fixture: Copy {
    /// Entire blob.
    bytes: [u8; 4096],

    offset: u32,
    /// Next property-name byte.
    strings: u32,
    /// Stack range value offset for malformed-input checks.
    stack: u32,
    /// PLIC context-list payload for mapping rejection checks.
    contexts: u32,
    /// PLIC source-count payload for capacity checks.
    sources: u32,
    /// PLIC register extent payload for bounds checks.
    plic: u32,
}

/// Write a big-endian fixture field.
fn put(f: &mut Fixture, offset: u32, width: u32, value: u64) {
    for i in 0..width {

    return offset;
}

/// Construct the supported one-hart boot contract.
fn build(f: &mut Fixture) {
    construct(f, false);
}

/// Construct a boot contract with optional supervisor and machine PLIC contexts.
fn construct(f: &mut Fixture, interrupts: bool) {
    for i in 0..f.bytes.len {
        set f.bytes[i] = 0;
    }
    set f.offset = 56; set f.strings = 3072;
    begin(f, &""[..]);

    let ram = region(f, &"reg"[..], 0x80000000, 0x10000); word(f, 2);
    begin(f, &"cpus"[..]);
    scalar(f, &"#address-cells"[..], 1); scalar(f, &"#size-cells"[..], 0);
    scalar(f, &"timebase-frequency"[..], 10000000);
    begin(f, &"cpu@0"[..]); string(f, &"device_type"[..], &"cpu"[..]);
    scalar(f, &"reg"[..], 0); word(f, 2); word(f, 2);
    scalar(f, &"reg"[..], 0);
    if interrupts {
        begin(f, &"interrupt-controller"[..]);
        string(f, &"compatible"[..], &"riscv,cpu-intc"[..]);
        scalar(f, &"phandle"[..], 17); scalar(f, &"#interrupt-cells"[..], 1);
        let marker = property(f, &"interrupt-controller"[..], 0);
        word(f, 2);
    }
    word(f, 2); word(f, 2);
    begin(f, &"reserved-memory"[..]);
    scalar(f, &"#address-cells"[..], 2); scalar(f, &"#size-cells"[..], 2);
    let ranges = property(f, &"ranges"[..], 0);
    begin(f, &"image@80002000"[..]);
    let image = region(f, &"reg"[..], 0x80002000, 0x3000); word(f, 2); word(f, 2);
    begin(f, &"soc"[..]);
    scalar(f, &"#address-cells"[..], 2); scalar(f, &"#size-cells"[..], 2);
    let bus = property(f, &"ranges"[..], 0);
    begin(f, &"uart@10000000"[..]);
    string(f, &"compatible"[..], &"ns16550a"[..]);
    let uart = region(f, &"reg"[..], 0x10000000, 0x100); word(f, 2); word(f, 2);
    let uart = region(f, &"reg"[..], 0x10000000, 0x100); word(f, 2);
    if interrupts {
        begin(f, &"plic@c000000"[..]);
        string(f, &"compatible"[..], &"riscv,plic0"[..]);
        set f.plic = region(f, &"reg"[..], 0xc000000, 0x4000000);
        scalar(f, &"#interrupt-cells"[..], 1);
        let sources = property(f, &"riscv,ndev"[..], 4);
        set f.sources = sources; put(f, sources, 4, 31);
        let contexts = property(f, &"interrupts-extended"[..], 16);
        set f.contexts = contexts;
        put(f, contexts, 4, 17); put(f, contexts + 4, 4, 9);
        put(f, contexts + 8, 4, 17); put(f, contexts + 12, 4, 11);
        word(f, 2);
    }
    word(f, 2);
    word(f, 2); word(f, 9);
    let total = f.strings; let size = f.offset - 56;
    put(f, 0, 4, 0xd00dfeed); put(f, 4, 4, total as u64);
    put(f, 8, 4, 56); put(f, 12, 4, 3072); put(f, 16, 4, 40);
    put(f, 20, 4, 17); put(f, 24, 4, 16);
    put(f, 32, 4, (total - 3072) as u64); put(f, 36, 4, size as u64);
}

/// PLIC machine contexts follow reference-list order rather than physical hart ID.
@test unsafe fn interruptContexts() throws (testing::TestError) {
    let mut f: Fixture = undefined; construct(&mut f, true);
    let mut p: platform::Platform = undefined;
    try! platform::decode(&f.bytes[..f.strings], &mut p);
    assert p.irqSources == 31 and p.irqContexts[0] == 1;
    assert p.deviceCount == 2 and p.devices[1].kind == platform::Kind::Plic;
    let initialContexts = f.contexts;
    put(&mut f, initialContexts + 4, 4, 0xffffffff);
    try! platform::decode(&f.bytes[..f.strings], &mut p);
    assert p.irqContexts[0] == 1;
    for change in 0..7 {
        construct(&mut f, true);
        let sources = f.sources;
        let contexts = f.contexts;
        let registers = f.plic;
        match change {
            case 0 => put(&mut f, sources, 4, 0),
            case 1 => put(&mut f, sources, 4, 129),
            case 2 => put(&mut f, contexts + 8, 4, 99),
            case 3 => put(&mut f, contexts + 4, 4, 11),
            case 4 => put(&mut f, contexts + 12, 4, 9),
            case 5 => put(&mut f, registers + 8, 8, 0x200000),
            else => put(&mut f, registers, 8, 0xc000002),
        }
        let mut rejected = false;
        try platform::decode(&f.bytes[..f.strings], &mut p) catch error {
            assert error == platform::Error::Invalid; set rejected = true;
        };
        assert rejected;
    }
}

/// PLIC register selection uses the decoded machine context and rejects ambiguous mappings.
@test unsafe fn plicMapping() throws (testing::TestError) {
    let mut f: Fixture = undefined; construct(&mut f, true);
    let mut p: platform::Platform = undefined;
    try! platform::decode(&f.bytes[..f.strings], &mut p);
    let selected = try! plic::mapping(&p) else panic "missing PLIC";
    assert selected.base == 0xc000000 and selected.count == 31 and selected.context == 1;
    set p.irqContexts[0] = 16;
    let mut rejected = false;
    try plic::mapping(&p) catch error {
        assert error == abi::Error::InvalidArg;
        set rejected = true;
    };
    assert rejected;
    set p.irqContexts[0] = 1;
    set p.devices[2] = p.devices[1]; set p.deviceCount = 3;
    let mut duplicate = false;
    try plic::mapping(&p) catch error {
        assert error == abi::Error::InvalidArg;
        set duplicate = true;
    };
    assert duplicate;
    set p.irqSources = 0;
    assert try! plic::mapping(&p) == nil;
}

/// Resources and their reservations come from the tree rather than fixed addresses.
@test unsafe fn discovery() throws (testing::TestError) {
    let mut f: Fixture = undefined; build(&mut f);
    let mut p: platform::Platform = undefined;
    try! platform::decode(&f.bytes[..f.strings], &mut p);

test/boot/run +2 -2

# Boot the kernel through its production initialization on each supported hart count.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/boot.ras"
cat test/boot/machine.ras kernel/kernel/*.ras > "$work/boot.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
for harts in 1 2 8; do
    # The deterministic emulator shares its tick budget across all harts.
    steps=$((10000000 * harts))
    if ! "$emulator" -machine -max-steps="$steps" -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then

if [ "$status" -ne 2 ] || [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
    cat "$work/log" >&2
    exit 1
fi
printf 'kernel startup: all eight harts reached machine idle without traps\n'
cat test/boot/trap.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/trap.ras"
cat test/boot/trap.ras kernel/kernel/*.ras > "$work/trap.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
status=0
"$emulator" -machine -harts=2 -max-steps=20000000 -run "$work/trap.rv64" > "$work/log" 2>&1 || status=$?
if [ "$status" -ne 2 ] || [ "$(grep -c '^kernel: unexpected trap$' "$work/log")" -ne 1 ] \
    || [ "$(grep -c 'wfi=1' "$work/log")" -ne 2 ]; then

test/dispatch/run +4 -1

# Preempt and resume U-mode and M-mode work through production dispatch.
set -eu
emulator=${RAD_EMULATOR:-emulator}
fixture=${1:-dispatch}
harts=${2:-1}
irq=${3:-0}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp "test/$fixture/spin.rad" "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"

printf '];\n' >> "$work/kernel/dispatchinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
set --
if [ "$irq" -ne 0 ]; then set -- "-irq=$irq" "-irq-at=1"; fi
for count in $harts; do
    "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
    "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 "$@" -run "$work/dispatch.rv64"
    printf '%s: %s-hart execution passed\n' "$fixture" "$count"
done

test/loader/run +1 -2

length=$(wc -c < "$work/loaded.ril")
printf '//! Binary package used by both execution paths.\n/// Complete trusted binary RIL input.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/loadinput.rad"
od -An -v -tu1 "$work/loaded.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/loadinput.rad"
printf '];\n' >> "$work/kernel/loadinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat test/loader/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras \
    kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/loader.ras"
cat test/loader/machine.ras kernel/kernel/*.ras > "$work/loader.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
    -run bin/kernel.build.rv64 -- "$work/std.ril" "$work/kernel.ril" "$work/loader.ras" "$work/loader.rv64"
"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/loader.rv64"
printf 'runtime loader: hosted and loaded entries agree after workspace release\n'

test/smp/kernel/dispatchcheck.rad +133 -1

use kernel::sync;
use kernel::remote;
use kernel::calls;
use kernel::events;
use kernel::timers;
use kernel::interrupts;
use kernel::plic;
use kernel::dispatchinput;

/// Bootstrap resource authority.
unsafe static TABLE: capability::Table = undefined;
/// Release/acquire barrier after fixture publication.

fn kernelGp() -> u64;
/// M-mode entry of the completion checker.
fn completion() -> u64;
/// Write one diagnostic byte to the fixture UART.
fn put(byte: u8);
/// Set the fixture UART interrupt-enable register.
fn uartInterrupt(value: u8);
/// Finish the machine test successfully.
fn finish();

/// Publish fixture state on the last initialized hart and wait on every hart.
export unsafe fn start(last: bool) {

    set USER = cap.object;
    let mut user = try! domains::get(&domains::STORE, USER);
    let self = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE),
    });
    assert boot::PLATFORM.irqSources == 31;
    let irq = try! interrupts::seed(&mut interrupts::STORE, &mut user.memory.table, 7);
    sync::storeRelease(memory(dataAddress(&user, &"spin::interrupt"[..])) as *mut u64, *irq);
    let target = try! registry::exported(&registry::STORE, user.image, &"spin::worker"[..]);
    let case shared::Target::Function(entry) = target else panic "worker entry";
    set COUNTERS = dataAddress(&user, &"spin::count"[..]);
    set MIGRATIONS = dataAddress(&user, &"spin::migrations"[..]);
    let notification = @sliceOf(memory(dataAddress(&user, &"spin::notification"[..])), 8);

        let destination = (hart + notifications - 1) % notifications;
        assert context.hart == nil or context.hart == destination;
        assert sync::loadAcquire(memory(MIGRATIONS + hart as u64 * 8) as *u64) == destination as u64 + 1;
        assert context.pending == nil;
    }
    assert domains::STORE.events.queues[USER.index].tail == notifications;
    assert domains::STORE.events.queues[USER.index].tail == notifications + 1;
    let user = try! domains::get(&domains::STORE, USER);
    assert user.memory.ring.data[0].kind == events::INTERRUPT and user.memory.ring.data[0].code == 7;
    sync::leave(guard);
    for byte in "smp completed transactions:" {
        put(byte);
    }
    for hart in 0..8 {

        put(byte);
    }
    if (boot::PLATFORM.harts & 2) <> 0 {
        idleWakeup();
    }
    interruptOwnership();
    interruptStorm();
    let maximum = sync::maximum();
    assert maximum > 0 and maximum < 1000000;
    for byte in "smp metadata instructions: 0x" {
        put(byte);
    }
    number(maximum);
    put(10);
    finish();
}

/// Transfer a queued IRQ, rearm after consumption, and mask it before capability drop.
unsafe fn interruptOwnership() {
    let guard = sync::enter();
    let mut user = try! domains::get(&domains::STORE, USER);
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    let device = try! plic::mapping(&boot::PLATFORM) else panic "missing PLIC";
    let enabled = memory(device.base + 0x2000 + device.context as u64 * 0x80) as *u32;
    assert (sync::read32(enabled) & (1 << 7)) == 0;
    assert interrupts::STORE.records[6].state == interrupts::State::Queued;
    let original = interrupts::STORE.records[6].handle;
    let target = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: CHECKER, rights: abi::Rights(0),
    });
    let back = try! capability::install(&mut checker.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(0),
    });
    let moved = try! calls::invoke(USER, 11, &[*original, *target, (abi::READ | abi::TRANSFER) as u64, 0], 0);
    assert interrupts::STORE.records[6].owner == CHECKER;
    let first = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
    assert interrupts::STORE.records[6].state == interrupts::State::Queued;
    assert (sync::read32(enabled) & (1 << 7)) == 0;
    let event = try! events::pop(&mut checker.memory.ring) else panic "missing transferred IRQ";
    assert event.kind == events::INTERRUPT and event.code == 7;
    let before = domains::STORE.events.queues[USER.index].tail;
    let returned = try! calls::invoke(CHECKER, 11, &[moved, *back, (abi::READ | abi::TRANSFER) as u64, 0], 0);
    let second = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
    assert domains::STORE.events.queues[USER.index].tail == before;
    assert (sync::read32(enabled) & (1 << 7)) == 0;
    let consumed = try! events::pop(&mut user.memory.ring) else panic "missing original IRQ";
    assert consumed.kind == events::INTERRUPT and consumed.code == 7;
    let rearmed = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
    assert interrupts::STORE.records[6].state == interrupts::State::Idle;
    assert (sync::read32(enabled) & (1 << 7)) <> 0;
    let dropped = try! calls::invoke(USER, 12, &[returned, 0, 0, 0], 0);
    assert dropped == 0 and interrupts::STORE.slots[6].state == slots::State::Free;
    assert (sync::read32(enabled) & (1 << 7)) == 0;
    sync::leave(guard);
    for byte in "smp IRQ transfer, rearm, and drop passed\n" {
        put(byte);
    }
}

/// Keep a level source asserted across full queues and repeated driver progress.
unsafe fn interruptStorm() {
    let guard = sync::enter();
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    let handle = try! interrupts::seed(&mut interrupts::STORE, &mut checker.memory.table, 10);
    let device = try! plic::mapping(&boot::PLATFORM) else panic "missing PLIC";
    let enabled = memory(device.base + 0x2000 + device.context as u64 * 0x80) as *u32;
    sync::leave(guard);
    for i in 0..events::ORDINARY {
        let filling = sync::enter();
        assert try! domains::notify(&mut domains::STORE, CHECKER,
            events::Event { kind: events::WAKEUP, reserved: 0, code: i, value: 0 });
        sync::leave(filling);
    }
    let arming = sync::enter();
    let ready = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
    uartInterrupt(2);
    sync::leave(arming);
    let mut pending = false;
    for attempt in 0..10000 {
        let checking = sync::enter();
        set pending = interrupts::STORE.records[9].state == interrupts::State::Pending;
        if pending {
            assert (sync::read32(enabled) & (1 << 10)) == 0;
            let queue = domains::STORE.events.queues[CHECKER.index];
            assert queue.tail - queue.head == events::ORDINARY;
        }
        sync::leave(checking);
        if pending {
            break;
        }
    }
    assert pending;
    for i in 0..events::ORDINARY {
        let draining = sync::enter();
        let mut receiver = try! domains::get(&domains::STORE, CHECKER);
        let event = try! events::pop(&mut receiver.memory.ring) else panic "missing saturation event";
        assert event.kind == events::WAKEUP and event.code == i;
        sync::leave(draining);
    }
    let posting = sync::enter();
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    try! timers::request(&mut timers::STORE, &domains::STORE, CHECKER, 0, clock, 2000000, 0x77);
    let request = try! dispatch::request(0, remote::Action::Reschedule);
    sync::leave(posting);
    let acknowledged = try! dispatch::awaitRequest(0, request);
    assert acknowledged == 0;
    let mut received: u32 = 0;
    let mut timed = false;
    for attempt in 0..1000 {
        let consuming = sync::enter();
        let mut receiver = try! domains::get(&domains::STORE, CHECKER);
        if let event = try! events::pop(&mut receiver.memory.ring) {
            if event.kind == events::INTERRUPT {
                assert event.code == 10;
                set received += 1;
            } else {
                assert event.kind == events::TIMEOUT and event.code == 0x77;
                assert dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) >= clock + 2000000;
                set timed = true;
            }
        }
        let notified = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));
        sync::leave(consuming);
        if timed and received >= 16 {
            break;
        }
    }
    let stopping = sync::enter();
    uartInterrupt(0);
    let dropped = try! calls::invoke(CHECKER, 12, &[*handle, 0, 0, 0], 0);
    assert dropped == 0 and (sync::read32(enabled) & (1 << 10)) == 0;
    sync::leave(stopping);
    assert timed and received >= 16;
    for byte in "smp IRQ saturation and storm timeout passed\n" {
        put(byte);
    }
}

/// Resume an actual U-mode Wait call through an IPI to an idle remote hart.
unsafe fn idleWakeup() {
    let guard = sync::enter();
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    let handle = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,

test/smp/machine.ras +7 -0

.export @kernel::dispatchcheck::memory;
.export @kernel::dispatchcheck::kernelGp;
.export @kernel::dispatchcheck::completion;
.export @kernel::dispatchcheck::finish;
.export @kernel::dispatchcheck::put;
.export @kernel::dispatchcheck::uartInterrupt;
@kernel::dispatchcheck::memory
    ret;
@kernel::dispatchcheck::kernelGp
    mv %a0 %gp;
    ret;

    ebreak;
@kernel::dispatchcheck::put
    li %t0 0x10000000;
    sb %a0 0(%t0);
    ret;
@kernel::dispatchcheck::uartInterrupt
    li %t0 0x10000000;
    fence iorw iorw;
    sb %a0 1(%t0);
    fence iorw iorw;
    ret;
@kernel::dispatchcheck::finish
    li %t0 0x10001000;
    li %t1 0x5555;
    sw %t1 0(%t0);
    ebreak;

test/smp/spin.rad +5 -0

export static windows: [u64; 8] = [0; 8];

/// Capability used to notify the shared domain from each online hart.
export static notification: u64 = 0;

/// Shared-domain capability for the injected physical interrupt source.
export static interrupt: u64 = 0;

/// Observed destination hart plus one after each context changes budget.
export static migrations: [u64; 8] = [0; 8];

/// Wait handshake: 2/3 bracket wakeup; 4 requests a second wait, bracketed by 5/6.
export static wakeState: u64 = 0;

/// Nonzero selects the independently activated idle-wakeup context.
export static idle: u64 = 0;

/// Check hart identity and shared-table allocation through repeated direct calls.
fn run(hart: u64) {
    let source = sys::queryInterrupt(abi::Handle(interrupt));
    assert source.number == 7;
    let original = sys::currentContext();
    assert original.hart == hart;
    let budget = abi::Handle(windows[hart as u32]);
    let before = try! sys::queryBudget(budget);
    let right = try! sys::budgetSplit(budget, before.start + 1000);

211	228		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
212	229		let page = try pages::get(&pages::STORE, entry.object);
213	230		try reply(&domain, arguments, &[page.base, page.count as u64]);
214	231		return 0;
215	232		},
	233	+	case 46 => {
	234	+	let number = try interrupts::query(&interrupts::STORE, &domain.memory.table, handle);
	235	+	if arguments[2] <> @sizeOf(abi::InterruptInfo) as u64 or arguments[3] <> 0
	236	+	or not buffer(&domain, arguments[1], 4, abi::Rights(abi::WRITE), 4) {
	237	+	throw abi::Error::InvalidArg;
	238	+	}
	239	+	set *memory32(arguments[1]) = number;
	240	+	return 0;
	241	+	},
216	242		case 47 => {
217	243		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
218	244		let target = try domains::get(&domains::STORE, entry.object);
219	245		let mut parent: u64 = 0;
220	246		if let object = target.parent {

248	274		try reply(&domain, arguments, &[abi::id(object), entry, package.codeAddress, package.code.len as u64 * 4, package.memory as u64]);
249	275		return 0;
250	276		},
251	277		case 60 => {
252	278		if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
253		-	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
	279	+	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ), 8) {
254	280		throw abi::Error::InvalidArg;
255	281		}
256	282		let words = @sliceOf(memory(arguments[1]), 4);
257	283		let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
258	284		return abi::id(try domains::contextCreate(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

451	477		case 51 => return try runtimeLoad(&mut domain, loader::Request {
452	478		authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
453	479		}, guard),
454	480		case 60 => {
455	481		if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
456		-	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
	482	+	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ), 8) {
457	483		sync::leave(guard); throw abi::Error::InvalidArg;
458	484		}
459	485		let words = @sliceOf(memory(arguments[1]), 4);
460	486		let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
461	487		let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

163	165		/// Publish one event after capacity and receiver validation.
164	166		fn publish(queue: &mut Queue, ring: &mut Ring, event: Event, subject: abi::Ref) {
165	167		let index = queue.tail & (CAPACITY - 1);
166	168		set ring.data[index] = event;
167	169		set queue.subjects[index] = subject;
	170	+	let mut source: u32 = 0;
	171	+	if event.kind == INTERRUPT {
	172	+	set source = event.code;
	173	+	}
	174	+	set queue.sources[index] = source;
168	175		set queue.tail = next(queue.tail);
169	176		sync::storeRelease32(&mut ring.tail, queue.tail);
170	177		}
171	178
172	179		/// Send an ordinary event, returning false when its reserved occupancy limit is reached.
173	180		export fn send(store: &mut Store, receiver: abi::Ref, ring: &mut Ring, event: Event) -> bool throws (abi::Error) {
174	181		if event.reserved <> 0 or (event.kind <> INTERRUPT and event.kind <> TIMEOUT and event.kind <> WAKEUP) {
175	182		throw abi::Error::InvalidArg;
176	183		}
	184	+	if event.kind == INTERRUPT and event.code == 0 {
	185	+	throw abi::Error::InvalidArg;
	186	+	}
177	187		try refresh(store, receiver, ring);
	188	+	if event.kind == INTERRUPT and interruptPending(store, receiver, event.code) {
	189	+	return true;
	190	+	}
178	191		let queue = &mut store.queues[receiver.index];
179	192		if distance(queue.tail, queue.head) >= ORDINARY {
180	193		return false;
181	194		}
182	195		publish(queue, ring, event, abi::Ref { index: 0, generation: 0 });
183	196		return true;
184	197		}
185	198
	199	+	/// Test trusted IRQ metadata after the receiver's consumer progress is refreshed.
	200	+	export fn interruptPending(store: &Store, receiver: abi::Ref, source: u32) -> bool {
	201	+	if source == 0 or not live(store, receiver) {
	202	+	return false;
	203	+	}
	204	+	let queue = &store.queues[receiver.index];
	205	+	for i in 0..distance(queue.tail, queue.head) {
	206	+	let index = ((queue.head & (CAPACITY - 1)) + i) & (CAPACITY - 1);
	207	+	if queue.sources[index] == source {
	208	+	return true;
	209	+	}
	210	+	}
	211	+	return false;
	212	+	}
	213	+
186	214		/// Read one published event and release its slot. Callers serialize consumers.
187	215		export fn pop(ring: &mut Ring) -> ?Event throws (abi::Error) {
188	216		let head = sync::loadAcquire32(&ring.head);
189	217		let tail = sync::loadAcquire32(&ring.tail);
190	218		let count = distance(tail, head);

192	304		let registers = node.registers else {
193	305		return;
194	306		};
195	307		if let typeName = node.deviceType {
196	308		if string(bytes, typeName, &"cpu"[..]) {
197		-	if node.sizeCells <> 0 or node.addressCells < 1 or node.addressCells > 2
198		-	or registers.end - registers.start <> node.addressCells * 4 {
199		-	throw Error::Invalid;
200		-	}
201		-	let hart = try fdt::integer(bytes, registers.start, node.addressCells * 4) catch err {
202		-	throw Error::Tree(err);
203		-	};
204		-	if hart >= limits::HARTS as u64 or (platform.harts & (1 << hart as u32)) <> 0 {
	309	+	let hart = try cpu(bytes, node);
	310	+	if (platform.harts & (1 << hart)) <> 0 {
205	311		throw Error::Invalid;
206	312		}
207		-	set platform.harts \|= 1 << hart as u32;
	313	+	set platform.harts \|= 1 << hart;
208	314		return;
209	315		}
210	316		}
211	317		let width = (node.addressCells + node.sizeCells) * 4;
212	318		if width == 0 or (registers.end - registers.start) % width <> 0 {

11	11		export mod platform;
12	12		export mod trap;
13	13		export mod capability;
14	14		export mod transactions;
15	15		export mod events;
	16	+	export mod interrupts;
	17	+	export mod plic;
16	18		export mod registry;
17	19		export mod frames;
18	20		export mod backing;
19	21		export mod pages;
20	22		export mod loader;

23	23		base: u64,
24	24		/// Number of contiguous physical frames.
25	25		count: u64,
26	26		}
27	27
	28	+	/// Physical source returned by QueryInterrupt.
	29	+	export record InterruptInfo: Copy {
	30	+	/// PLIC source number; zero does not name an interrupt.
	31	+	number: u32,
	32	+	}
	33	+
28	34		/// Domain identity and lifetime returned by QueryDomain.
29	35		export record DomainInfo: Copy {
30	36		/// Generation-bearing domain identity.
31	37		id: u64,
32	38		/// Historical creator identity.

11	11		use super::domains;
12	12		use super::budgets;
13	13		use super::dispatch;
14	14		use super::timers;
15	15		use super::remote;
	16	+	use super::interrupts;
	17	+	use super::plic;
16	18		use std::arch::rv64::shared::catalog;
17	19
18	20		/// Platform data published by hart zero before secondary initialization.
19	21		export unsafe static PLATFORM: platform::Platform = undefined;
20	22		/// Release/acquire publication flag for PLATFORM.

25	27		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
26	28
27	29		/// Handle timer boundaries and user calls, and stop on unsupported traps.
28	30		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
29	31		let cause = trap::classify(frame.cause);
30		-	if cause == trap::Cause::Timer or cause == trap::Cause::Software {
	32	+	if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
31	33		dispatch::interrupt(frame, hart);
32	34		return;
33	35		}
34	36		if cause == trap::Cause::UserCall and trap::fromUser(frame) {
35	37		dispatch::call(frame, hart);

86	88		try! pages::initialize(&mut pages::STORE, &PLATFORM);
87	89		domains::initialize(&mut domains::STORE);
88	90		budgets::initialize(&mut budgets::STORE);
89	91		timers::initialize(&mut timers::STORE);
90	92		remote::initialize(&mut remote::STORE, PLATFORM.harts);
	93	+	interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
	94	+	try! plic::initialize(&PLATFORM);
91	95		assert entryCount == 2;
92	96		registry::initialize(&mut registry::STORE);
93	97		let bootCatalog = @sliceOf(entries, entryCount);
94	98		try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
95	99		let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";

14	14		use super::loader;
15	15		use super::dispatch;
16	16		use super::timers;
17	17		use super::sync;
18	18		use super::remote;
	19	+	use super::interrupts;
	20	+	use super::plic;
19	21
20	22		/// Resolve the owning hart for an operation that can change CPU authority.
21	23		unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
22	24		match operation {
23	25		case 11, 12, 70, 71, 73 => {

95	97		if not moving {
96	98		throw abi::Error::Denied;
97	99		}
98	100		return try budgets::transfer(&mut budgets::STORE, &mut domain.memory.table, &mut destination.memory.table, handle, rights, now);
99	101		}
	102	+	if entry.kind == abi::Kind::Interrupt {
	103	+	if not moving {
	104	+	throw abi::Error::Denied;
	105	+	}
	106	+	return try interrupts::transfer(&mut interrupts::STORE, &mut domain.memory.table,
	107	+	&mut destination.memory.table, handle, rights);
	108	+	}
100	109		try resident(entry);
101	110		let pending = try transactions::reserve(&mut destination.memory.table, owner, handle, entry, rights, moving);
102	111		if moving {
103	112		return transactions::transfer(&mut domain.memory.table, &mut destination.memory.table, pending);
104	113		}

110	119		let entry = try transactions::dropping(&domain.memory.table, handle, domain.events);
111	120		let table: &mut capability::Table = &mut domain.memory.table;
112	121		match entry.kind {
113	122		case abi::Kind::Page => try pages::drop(&mut pages::STORE, table, handle),
114	123		case abi::Kind::Budget => try budgets::drop(&mut budgets::STORE, table, handle),
	124	+	case abi::Kind::Interrupt => {
	125	+	let number = try interrupts::query(&interrupts::STORE, table, handle);
	126	+	plic::mask(number);
	127	+	try! interrupts::drop(&mut interrupts::STORE, table, handle);
	128	+	},
115	129		else => {
116	130		try resident(entry);
117	131		let removed = try! capability::invalidate(table, handle);
118	132		},
119	133		}
120	134		}
121	135
122	136		/// Map a checked, aligned caller record address to words.
123	137		fn memory(address: u64) -> *mut u64;
	138	+	/// Map a checked four-byte-aligned result buffer to its source-number word.
	139	+	fn memory32(address: u64) -> *mut u32;
124	140
125	141		/// Test whether one nonwrapping extent contains the complete result range.
126	142		fn contains(base: u64, size: u64, address: u64, length: u64) -> bool {
127	143		return address >= base and address - base <= size and length <= size - (address - base);
128	144		}
129	145
130	146		/// Validate aligned Page or private package-data storage with the given rights.
131	147		/// Package-state tables and kernel metadata cannot hold caller-supplied records.
132		-	unsafe fn buffer(domain: &domains::Domain, address: u64, size: u64, rights: abi::Rights) -> bool {
133		-	if (address & 7) <> 0 or size > 0xffffffffffffffff - address {
	148	+	unsafe fn buffer(domain: &domains::Domain, address: u64, size: u64, rights: abi::Rights, alignment: u64) -> bool {
	149	+	assert alignment == 4 or alignment == 8;
	150	+	if (address & (alignment - 1)) <> 0 or size > 0xffffffffffffffff - address {
134	151		return false;
135	152		}
136	153		if pages::accessible(&pages::STORE, &domain.memory.table, address, size, rights) {
137	154		return true;
138	155		}

153	170		}
154	171
155	172		/// Copy a fixed query record only after validating its exact buffer size.
156	173		unsafe fn reply(domain: &domains::Domain, arguments: &[u64], words: &[u64]) throws (abi::Error) {
157	174		let size = words.len as u64 * 8;
158		-	if arguments[2] <> size or arguments[3] <> 0 or not buffer(domain, arguments[1], size, abi::Rights(abi::WRITE)) {
	175	+	if arguments[2] <> size or arguments[3] <> 0 or not buffer(domain, arguments[1], size, abi::Rights(abi::WRITE), 8) {
159	176		throw abi::Error::InvalidArg;
160	177		}
161	178		let output = @sliceOf(memory(arguments[1]), words.len);
162	179		for i in 0..words.len {
163	180		set output[i] = words[i];

1	1		//! Typed access to a checked user record buffer.
2	2		.text;
3	3		.export @kernel::calls::memory;
	4	+	.export @kernel::calls::memory32;
4	5
5	6		// The caller checked alignment, extent, and the required domain access rights.
6	7		@kernel::calls::memory
	8	+	@kernel::calls::memory32
7	9		ret;

10	10		use super::range;
11	11		use super::calls;
12	12		use super::timers;
13	13		use super::sync;
14	14		use super::remote;
	15	+	use super::plic;
15	16
16	17		/// Dispatch ownership and idle state retained by one physical hart.
17	18		export record State: Copy {
18	19		/// Physical hart whose windows this state selects.
19	20		hart: u32,

54	55		fn stackPointer() -> u64;
55	56		/// Trap anchor of the hart currently executing this continuation.
56	57		fn localAnchor() -> *mut trap::Hart;
57	58		/// Address of the M-mode idle loop.
58	59		fn idleAddress() -> u64;
59		-	/// Enable timer and software interrupt delivery with global interrupts disabled.
	60	+	/// Enable machine interrupt sources with global interrupts disabled.
60	61		fn enable();
61	62
62	63		/// Initialize this hart's idle frame and validated timer before installing dispatch.
63	64		export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {
64	65		let device = try timer(machine, id);

84	85		assert pointer >= stack.start and pointer < stack.end;
85	86		let guard = sync::enter();
86	87		writeSoftware(HARTS[id].timer.software, 0);
87	88		let previous = HARTS[id].state.current;
88	89		save(&mut HARTS[id].state, &mut domains::STORE, frame);
	90	+	if trap::classify(frame.cause) == trap::Cause::External {
	91	+	plic::claim(id);
	92	+	}
89	93		let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
90	94		let mut count: u32 = 0;
91	95		for i in 0..limits::REMOTE_PER_HART {
92	96		let delivery = try! remote::take(&mut remote::STORE, id) else {
93	97		break;

104	108		set deliveries[count] = delivery;
105	109		set count += 1;
106	110		}
107	111		let clock = now(HARTS[id].timer);
108	112		let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
	113	+	let notified = timeout.harts \| plic::service(clock);
109	114		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
110	115		let mut deadline = choice.deadline;
111	116		if timeout.deadline < deadline {
112	117		set deadline = timeout.deadline;
113	118		}
114	119		for target in 0..limits::HARTS {
115		-	if target <> id and (timeout.harts & (1 << target)) <> 0 {
	120	+	if target <> id and (notified & (1 << target)) <> 0 {
116	121		signal(target);
117	122		}
118	123		}
119	124		arm(HARTS[id].timer, deadline);
120	125		if previous <> choice.context {

60	60		j @dispatchIdle;
61	61
62	62		// Global interrupt enable is restored from the selected frame by mret.
63	63		@kernel::dispatch::enable
64	64		csrr %t0 mie;
65		-	ori %t0 %t0 136;
	65	+	li %t1 2184;
	66	+	or %t0 %t0 %t1;
66	67		csrw mie %t0;
67	68		ret;
68	69
69	70		// The validated CLINT mapping contains a naturally aligned mtime register.
70	71		@kernel::dispatch::read

51	51		head: u32,
52	52		/// Authoritative producer counter.
53	53		tail: u32,
54	54		/// Terminal domain per occupied slot; generation zero denotes an ordinary event.
55	55		subjects: [abi::Ref; CAPACITY],
	56	+	/// Interrupt source per occupied slot, or zero for another event kind.
	57	+	sources: [u32; CAPACITY],
56	58		}
57	59
58	60		/// Lifetime state of a terminal-event record.
59	61		export union State: Copy {
60	62		/// Available for a new domain's terminal reservation.

53	53		devices: [Device; limits::DEVICES],
54	54		/// Number of MMIO devices.
55	55		deviceCount: u32,
56	56		/// Bit mask of online physical hart IDs.
57	57		harts: u32,
	58	+	/// PLIC source count, or zero when no supported controller is present.
	59	+	irqSources: u32,
	60	+	/// Machine-mode PLIC context index for each online hart.
	61	+	irqContexts: [?u32; limits::HARTS],
58	62		/// Timer ticks per second.
59	63		timebase: u32,
60	64		/// Kernel stack ranges, indexed by physical hart ID.
61	65		stacks: [range::Range; limits::HARTS],
62	66		/// Number of stack ranges in the boot contract.

79	83		registers: ?fdt::Span,
80	84		/// Device compatibility strings, if present.
81	85		compatible: ?fdt::Span,
82	86		/// Device type string, if present.
83	87		deviceType: ?fdt::Span,
	88	+	/// Interrupt-controller reference identity, or zero when absent.
	89	+	phandle: u32,
	90	+	/// Cells in one interrupt specifier.
	91	+	interruptCells: u32,
	92	+	/// PLIC source count, or zero when absent.
	93	+	sources: u32,
	94	+	/// PLIC context references and local interrupt numbers.
	95	+	interrupts: ?fdt::Span,
84	96		/// Node status permits use.
85	97		enabled: bool,
86	98		/// This node or its parent is the reserved-memory container.
87	99		reserved: bool,
88	100		}
89	101
	102	+	/// Read a CPU node's bounded physical hart identifier.
	103	+	fn cpu(bytes: &[u8], node: Node) -> u32 throws (Error) {
	104	+	let registers = node.registers else {
	105	+	throw Error::Invalid;
	106	+	};
	107	+	if node.sizeCells <> 0 or node.addressCells < 1 or node.addressCells > 2
	108	+	or registers.end - registers.start <> node.addressCells * 4 {
	109	+	throw Error::Invalid;
	110	+	}
	111	+	let hart = try fdt::integer(bytes, registers.start, node.addressCells * 4) catch err {
	112	+	throw Error::Tree(err);
	113	+	};
	114	+	if hart >= limits::HARTS as u64 {
	115	+	throw Error::Invalid;
	116	+	}
	117	+	return hart as u32;
	118	+	}
	119	+
	120	+	/// Associate a CPU-local interrupt controller with its parent hart.
	121	+	fn controller(bytes: &[u8], node: Node, parent: Node, references: &mut [u32]) throws (Error) {
	122	+	let kind = parent.deviceType else {
	123	+	throw Error::Invalid;
	124	+	};
	125	+	if not string(bytes, kind, &"cpu"[..]) or node.phandle == 0 or node.interruptCells <> 1 {
	126	+	throw Error::Invalid;
	127	+	}
	128	+	let hart = try cpu(bytes, parent);
	129	+	if references[hart] <> 0 {
	130	+	throw Error::Invalid;
	131	+	}
	132	+	for reference in &references[..] {
	133	+	if reference == node.phandle {
	134	+	throw Error::Invalid;
	135	+	}
	136	+	}
	137	+	set references[hart] = node.phandle;
	138	+	}
	139	+
	140	+	/// Resolve PLIC context ordinals after all CPU controller references are known.
	141	+	fn plic(bytes: &[u8], node: Node, references: &[u32], platform: &mut Platform) throws (Error) {
	142	+	if node.sources == 0 or node.sources > limits::INTERRUPTS or node.interruptCells <> 1 {
	143	+	throw Error::Invalid;
	144	+	}
	145	+	let contexts = node.interrupts else {
	146	+	throw Error::Invalid;
	147	+	};
	148	+	let length = contexts.end - contexts.start;
	149	+	if length == 0 or length % 8 <> 0 or length / 8 > limits::HARTS * 2 {
	150	+	throw Error::Invalid;
	151	+	}
	152	+	let registers = node.registers else {
	153	+	throw Error::Invalid;
	154	+	};
	155	+	if registers.end - registers.start <> (node.addressCells + node.sizeCells) * 4 {
	156	+	throw Error::Invalid;
	157	+	}
	158	+	let memory = try extent(bytes, registers.start, node.addressCells, node.sizeCells);
	159	+	let required = 0x200008 + (length as u64 / 8 - 1) * 0x1000;
	160	+	if (memory.start & 3) <> 0 or memory.end - memory.start < required {
	161	+	throw Error::Invalid;
	162	+	}
	163	+	for index in 0..length / 8 {
	164	+	let offset = contexts.start + index * 8;
	165	+	let reference = try fdt::integer(bytes, offset, 4) catch err {
	166	+	throw Error::Tree(err);
	167	+	};
	168	+	let cause = try fdt::integer(bytes, offset + 4, 4) catch err {
	169	+	throw Error::Tree(err);
	170	+	};
	171	+	let mut hart: ?u32 = nil;
	172	+	for id in 0..limits::HARTS {
	173	+	if references[id] <> 0 and references[id] as u64 == reference {
	174	+	set hart = id;
	175	+	}
	176	+	}
	177	+	let id = hart else {
	178	+	throw Error::Invalid;
	179	+	};
	180	+	if cause == 9 or cause == 0xffffffff {
	181	+	continue;
	182	+	}
	183	+	if cause <> 11 {
	184	+	throw Error::Invalid;
	185	+	}
	186	+	if (platform.harts & (1 << id)) == 0 {
	187	+	continue;
	188	+	}
	189	+	if platform.irqContexts[id] <> nil {
	190	+	throw Error::Invalid;
	191	+	}
	192	+	set platform.irqContexts[id] = index;
	193	+	}
	194	+	for hart in 0..limits::HARTS {
	195	+	if (platform.harts & (1 << hart)) <> 0 and platform.irqContexts[hart] == nil {
	196	+	throw Error::Invalid;
	197	+	}
	198	+	}
	199	+	set platform.irqSources = node.sources;
	200	+	}
	201	+
90	202		/// Compare a complete byte span with a name.
91	203		fn equal(bytes: &[u8], span: fdt::Span, value: &[u8]) -> bool {
92	204		if span.end - span.start <> value.len {
93	205		return false;
94	206		}

266	372
267	373		/// Extract native platform resources. Discard the output if decoding fails.
268	374		export unsafe fn decode(bytes: &[u8], platform: &mut Platform) throws (Error) {
269	375		set platform.ramCount = 0; set platform.reservedCount = 0; set platform.deviceCount = 0;
270	376		set platform.harts = 0; set platform.timebase = 0; set platform.stackCount = 0;
	377	+	set platform.irqSources = 0;
	378	+	for hart in 0..limits::HARTS {
	379	+	set platform.irqContexts[hart] = nil;
	380	+	}
	381	+	let mut references: [u32; limits::HARTS] = [0; limits::HARTS];
	382	+	let mut interruptController: ?Node = nil;
271	383		let tree = try fdt::header(bytes) catch err {
272	384		throw Error::Tree(err);
273	385		};
274	386		let mut at = tree.reservations.start;
275	387		while at < tree.reservations.end {

295	407		}
296	408		if state.depth == 2 and equal(bytes, name, &"reserved-memory"[..]) {
297	409		set reserved = true;
298	410		}
299	411		set nodes[state.depth - 1] = Node { name, addressCells, sizeCells, childAddress: 2, childSize: 1,
300		-	registers: nil, compatible: nil, deviceType: nil, enabled, reserved };
	412	+	registers: nil, compatible: nil, deviceType: nil, phandle: 0, interruptCells: 0,
	413	+	sources: 0, interrupts: nil, enabled, reserved };
301	414		},
302	415		case fdt::Event::Property { name, value } => {
303	416		if equal(bytes, name, &"#address-cells"[..]) {
304	417		let count = try scalar(bytes, value);
305	418		if count > 2 {

319	432		set nodes[state.depth - 1].compatible = value;
320	433		}
321	434		else if equal(bytes, name, &"device_type"[..]) {
322	435		set nodes[state.depth - 1].deviceType = value;
323	436		}
	437	+	else if equal(bytes, name, &"phandle"[..]) {
	438	+	set nodes[state.depth - 1].phandle = try scalar(bytes, value);
	439	+	}
	440	+	else if equal(bytes, name, &"#interrupt-cells"[..]) {
	441	+	set nodes[state.depth - 1].interruptCells = try scalar(bytes, value);
	442	+	}
	443	+	else if equal(bytes, name, &"riscv,ndev"[..]) {
	444	+	set nodes[state.depth - 1].sources = try scalar(bytes, value);
	445	+	}
	446	+	else if equal(bytes, name, &"interrupts-extended"[..]) {
	447	+	set nodes[state.depth - 1].interrupts = value;
	448	+	}
324	449		else if equal(bytes, name, &"status"[..]) {
325	450		set nodes[state.depth - 1].enabled = nodes[state.depth - 1].enabled and (string(bytes, value, &"okay"[..]) or string(bytes, value, &"ok"[..]));
326	451		} else if equal(bytes, name, &"ranges"[..]) and value.start <> value.end {
327	452		throw Error::Invalid;
328	453		}

341	466		}
342	467		set platform.stacks[i] = stack; try protect(platform, stack);
343	468		}
344	469		}
345	470		},
346		-	case fdt::Event::EndNode => try finish(bytes, nodes[state.depth], platform),
	471	+	case fdt::Event::EndNode => {
	472	+	let node = nodes[state.depth];
	473	+	if let compatible = node.compatible {
	474	+	if string(bytes, compatible, &"riscv,cpu-intc"[..]) {
	475	+	if state.depth == 0 {
	476	+	throw Error::Invalid;
	477	+	}
	478	+	try controller(bytes, node, nodes[state.depth - 1], &mut references[..]);
	479	+	}
	480	+	if node.enabled and string(bytes, compatible, &"riscv,plic0"[..]) {
	481	+	if interruptController <> nil {
	482	+	throw Error::Invalid;
	483	+	}
	484	+	set interruptController = node;
	485	+	}
	486	+	}
	487	+	try finish(bytes, node, platform);
	488	+	},
347	489		case fdt::Event::End => break,
348	490		}
349	491		}
350	492		if platform.ramCount == 0 or platform.harts == 0 or platform.timebase == 0 {
351	493		throw Error::Invalid;
352	494		}
	495	+	if let node = interruptController {
	496	+	try plic(bytes, node, &references[..], platform);
	497	+	}
353	498		for hart in 0..limits::HARTS {
354	499		if (platform.harts & (1 << hart)) == 0 {
355	500		continue;
356	501		}
357	502		if hart >= platform.stackCount or not inRam(platform, platform.stacks[hart]) {

124	124		try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
125	125		return info;
126	126		}
127	127		}
128	128
	129	+	/// Read a physical interrupt source. An invalid handle is a program error.
	130	+	export fn queryInterrupt(handle: abi::Handle) -> abi::InterruptInfo {
	131	+	unsafe {
	132	+	let mut info: abi::InterruptInfo = undefined;
	133	+	try! result(ecall(abi::Operation::QueryInterrupt as u32, *handle as i64,
	134	+	(&mut info) as i64, @sizeOf(abi::InterruptInfo) as i64, 0));
	135	+	return info;
	136	+	}
	137	+	}
	138	+
129	139		/// Read domain identity and lifetime. An invalid handle is a program error.
130	140		export fn queryDomain(handle: abi::Handle) -> abi::DomainInfo {
131	141		unsafe {
132	142		let mut info: abi::DomainInfo = undefined;
133	143		try! result(ecall(abi::Operation::QueryDomain as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::DomainInfo) as i64, 0));

11	11		use kernel::registry;
12	12		use kernel::limits;
13	13		use kernel::trap;
14	14		use kernel::budgets;
15	15		use kernel::remote;
	16	+	use kernel::interrupts;
	17	+
	18	+	/// Interrupt queries write exactly four bytes at four-byte-aligned addresses.
	19	+	@test unsafe fn interruptRecord() throws (testing::TestError) {
	20	+	let owner = initialize();
	21	+	interrupts::initialize(&mut interrupts::STORE, 8);
	22	+	let handle = try! interrupts::seed(&mut interrupts::STORE, &mut MEMORY.table, 7);
	23	+	let page = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
	24	+	let base = (&RAM[OFFSET]) as u64;
	25	+	set RAM[OFFSET] = 0xaaaaaaaa55555555;
	26	+	set RAM[OFFSET + 1] = 0x1122334455667788;
	27	+	let result = try! calls::invoke(owner, 46, &[*handle, base + 4, 4, 0], 0);
	28	+	assert result == 0 and @sizeOf(abi::InterruptInfo) == 4;
	29	+	assert RAM[OFFSET] == 0x0000000755555555 and RAM[OFFSET + 1] == 0x1122334455667788;
	30	+	let mut rejected: u32 = 0;
	31	+	for args in &[
	32	+	[handle, base + 2, 4, 0], [handle, base + 4, 8, 0],
	33	+	[handle, base + 4096, 4, 0], [handle, base + 4, 4, 1],
	34	+	] {
	35	+	try calls::invoke(owner, 46, &args[..], 0) catch error {
	36	+	assert error == abi::Error::InvalidArg; set rejected += 1;
	37	+	};
	38	+	}
	39	+	assert rejected == 4 and RAM[OFFSET] == 0x0000000755555555;
	40	+	let boundary = try! calls::invoke(owner, 46, &[*handle, base + 4092, 4, 0], 0);
	41	+	assert boundary == 0 and RAM[OFFSET + 511] == 0x0000000700000000;
	42	+	let decoded = try! abi::decode(abi::Handle(page));
	43	+	set MEMORY.table.entries[decoded.object.index].rights = abi::Rights(abi::READ);
	44	+	try calls::invoke(owner, 46, &[*handle, base + 4, 4, 0], 0) catch error {
	45	+	assert error == abi::Error::InvalidArg; set rejected += 1;
	46	+	};
	47	+	assert rejected == 5;
	48	+	}
16	49
17	50		/// Deferred budget operations recheck authority and expiration at execution time.
18	51		@test unsafe fn deferredBudget() throws (testing::TestError) {
19	52		let owner = initialize();
20	53		let handle = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 1, 10, 100);

7	7
8	8		/// Kernel event metadata workspace.
9	9		unsafe static STORE: events::Store = undefined;
10	10		/// Consumer-visible ring for the receiver.
11	11		unsafe static RING: events::Ring = undefined;
	12	+	/// Separate receiver used to check per-domain IRQ coalescing.
	13	+	unsafe static OTHER: events::Ring = undefined;
12	14
13	15		/// Return the receiver's current domain reference.
14	16		fn receiver() -> abi::Ref {
15	17		return abi::Ref { index: 0, generation: 1 };
16	18		}

24	26		/// Build an ordinary wakeup with a recognizable payload.
25	27		fn wake(value: u64) -> events::Event {
26	28		return events::Event { kind: events::WAKEUP, reserved: 0, code: 17, value };
27	29		}
28	30
	31	+	/// Duplicate IRQ notifications retain one queue entry even at ordinary capacity.
	32	+	@test unsafe fn interruptCoalescing() throws (testing::TestError) {
	33	+	initialize();
	34	+	let irq = events::Event { kind: events::INTERRUPT, reserved: 0, code: 5, value: 0 };
	35	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	36	+	set RING.data[0].code = 99;
	37	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	38	+	assert STORE.queues[0].tail == 1;
	39	+	for i in 1..events::ORDINARY {
	40	+	assert try! events::send(&mut STORE, receiver(), &mut RING, wake(i as u64));
	41	+	}
	42	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	43	+	assert STORE.queues[0].tail == events::ORDINARY;
	44	+	assert not try! events::send(&mut STORE, receiver(), &mut RING,
	45	+	events::Event { kind: events::INTERRUPT, reserved: 0, code: 6, value: 0 });
	46	+	let consumed = try! events::pop(&mut RING) else panic "missing interrupt";
	47	+	assert consumed.code == 99;
	48	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	49	+	assert STORE.queues[0].tail == events::ORDINARY + 1;
	50	+	}
	51	+
	52	+	/// Pending IRQ identity is scoped to the receiver and survives counter wrap.
	53	+	@test unsafe fn interruptReceivers() throws (testing::TestError) {
	54	+	initialize();
	55	+	let other = abi::Ref { index: 1, generation: 1 };
	56	+	try! events::open(&mut STORE, other, &mut OTHER);
	57	+	set STORE.queues[0].head = 0xfffffffe;
	58	+	set STORE.queues[0].tail = 0xfffffffe;
	59	+	set RING.head = 0xfffffffe;
	60	+	set RING.tail = 0xfffffffe;
	61	+	let irq = events::Event { kind: events::INTERRUPT, reserved: 0, code: 5, value: 0 };
	62	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	63	+	assert try! events::send(&mut STORE, receiver(), &mut RING,
	64	+	events::Event { kind: events::INTERRUPT, reserved: 0, code: 6, value: 0 });
	65	+	assert try! events::send(&mut STORE, other, &mut OTHER, irq);
	66	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	67	+	assert STORE.queues[0].tail == 0 and STORE.queues[1].tail == 1;
	68	+	assert events::interruptPending(&STORE, receiver(), 5);
	69	+	assert events::interruptPending(&STORE, other, 5);
	70	+	let first = try! events::pop(&mut RING) else panic "missing wrapped IRQ";
	71	+	assert first.code == 5;
	72	+	try! events::refresh(&mut STORE, receiver(), &RING);
	73	+	assert not events::interruptPending(&STORE, receiver(), 5);
	74	+	assert events::interruptPending(&STORE, receiver(), 6);
	75	+	assert events::interruptPending(&STORE, other, 5);
	76	+	assert try! events::send(&mut STORE, receiver(), &mut RING, irq);
	77	+	assert STORE.queues[0].tail == 1;
	78	+	}
	79	+
29	80		/// Fill ordinary capacity while preserving the reserved lifecycle capacity.
30	81		@test unsafe fn saturation() throws (testing::TestError) {
31	82		initialize();
32	83		for i in 0..events::ORDINARY {
33	84		try testing::expect(try! events::send(&mut STORE, receiver(), &mut RING, wake(i as u64)));

1	1		//! Platform resource discovery and reserved-memory exclusion fixtures.
2	2
3	3		use std::testing;
4	4		use kernel::platform;
5	5		use kernel::range;
	6	+	use kernel::plic;
	7	+	use kernel::abi;
6	8
7	9		/// Fixed FDT construction workspace with separate structure and string regions.
8	10		record Fixture: Copy {
9	11		/// Entire blob.
10	12		bytes: [u8; 4096],

12	14		offset: u32,
13	15		/// Next property-name byte.
14	16		strings: u32,
15	17		/// Stack range value offset for malformed-input checks.
16	18		stack: u32,
	19	+	/// PLIC context-list payload for mapping rejection checks.
	20	+	contexts: u32,
	21	+	/// PLIC source-count payload for capacity checks.
	22	+	sources: u32,
	23	+	/// PLIC register extent payload for bounds checks.
	24	+	plic: u32,
17	25		}
18	26
19	27		/// Write a big-endian fixture field.
20	28		fn put(f: &mut Fixture, offset: u32, width: u32, value: u64) {
21	29		for i in 0..width {

69	77		return offset;
70	78		}
71	79
72	80		/// Construct the supported one-hart boot contract.
73	81		fn build(f: &mut Fixture) {
	82	+	construct(f, false);
	83	+	}
	84	+
	85	+	/// Construct a boot contract with optional supervisor and machine PLIC contexts.
	86	+	fn construct(f: &mut Fixture, interrupts: bool) {
74	87		for i in 0..f.bytes.len {
75	88		set f.bytes[i] = 0;
76	89		}
77	90		set f.offset = 56; set f.strings = 3072;
78	91		begin(f, &""[..]);

85	98		let ram = region(f, &"reg"[..], 0x80000000, 0x10000); word(f, 2);
86	99		begin(f, &"cpus"[..]);
87	100		scalar(f, &"#address-cells"[..], 1); scalar(f, &"#size-cells"[..], 0);
88	101		scalar(f, &"timebase-frequency"[..], 10000000);
89	102		begin(f, &"cpu@0"[..]); string(f, &"device_type"[..], &"cpu"[..]);
90		-	scalar(f, &"reg"[..], 0); word(f, 2); word(f, 2);
	103	+	scalar(f, &"reg"[..], 0);
	104	+	if interrupts {
	105	+	begin(f, &"interrupt-controller"[..]);
	106	+	string(f, &"compatible"[..], &"riscv,cpu-intc"[..]);
	107	+	scalar(f, &"phandle"[..], 17); scalar(f, &"#interrupt-cells"[..], 1);
	108	+	let marker = property(f, &"interrupt-controller"[..], 0);
	109	+	word(f, 2);
	110	+	}
	111	+	word(f, 2); word(f, 2);
91	112		begin(f, &"reserved-memory"[..]);
92	113		scalar(f, &"#address-cells"[..], 2); scalar(f, &"#size-cells"[..], 2);
93	114		let ranges = property(f, &"ranges"[..], 0);
94	115		begin(f, &"image@80002000"[..]);
95	116		let image = region(f, &"reg"[..], 0x80002000, 0x3000); word(f, 2); word(f, 2);
96	117		begin(f, &"soc"[..]);
97	118		scalar(f, &"#address-cells"[..], 2); scalar(f, &"#size-cells"[..], 2);
98	119		let bus = property(f, &"ranges"[..], 0);
99	120		begin(f, &"uart@10000000"[..]);
100	121		string(f, &"compatible"[..], &"ns16550a"[..]);
101		-	let uart = region(f, &"reg"[..], 0x10000000, 0x100); word(f, 2); word(f, 2);
	122	+	let uart = region(f, &"reg"[..], 0x10000000, 0x100); word(f, 2);
	123	+	if interrupts {
	124	+	begin(f, &"plic@c000000"[..]);
	125	+	string(f, &"compatible"[..], &"riscv,plic0"[..]);
	126	+	set f.plic = region(f, &"reg"[..], 0xc000000, 0x4000000);
	127	+	scalar(f, &"#interrupt-cells"[..], 1);
	128	+	let sources = property(f, &"riscv,ndev"[..], 4);
	129	+	set f.sources = sources; put(f, sources, 4, 31);
	130	+	let contexts = property(f, &"interrupts-extended"[..], 16);
	131	+	set f.contexts = contexts;
	132	+	put(f, contexts, 4, 17); put(f, contexts + 4, 4, 9);
	133	+	put(f, contexts + 8, 4, 17); put(f, contexts + 12, 4, 11);
	134	+	word(f, 2);
	135	+	}
	136	+	word(f, 2);
102	137		word(f, 2); word(f, 9);
103	138		let total = f.strings; let size = f.offset - 56;
104	139		put(f, 0, 4, 0xd00dfeed); put(f, 4, 4, total as u64);
105	140		put(f, 8, 4, 56); put(f, 12, 4, 3072); put(f, 16, 4, 40);
106	141		put(f, 20, 4, 17); put(f, 24, 4, 16);
107	142		put(f, 32, 4, (total - 3072) as u64); put(f, 36, 4, size as u64);
108	143		}
109	144
	145	+	/// PLIC machine contexts follow reference-list order rather than physical hart ID.
	146	+	@test unsafe fn interruptContexts() throws (testing::TestError) {
	147	+	let mut f: Fixture = undefined; construct(&mut f, true);
	148	+	let mut p: platform::Platform = undefined;
	149	+	try! platform::decode(&f.bytes[..f.strings], &mut p);
	150	+	assert p.irqSources == 31 and p.irqContexts[0] == 1;
	151	+	assert p.deviceCount == 2 and p.devices[1].kind == platform::Kind::Plic;
	152	+	let initialContexts = f.contexts;
	153	+	put(&mut f, initialContexts + 4, 4, 0xffffffff);
	154	+	try! platform::decode(&f.bytes[..f.strings], &mut p);
	155	+	assert p.irqContexts[0] == 1;
	156	+	for change in 0..7 {
	157	+	construct(&mut f, true);
	158	+	let sources = f.sources;
	159	+	let contexts = f.contexts;
	160	+	let registers = f.plic;
	161	+	match change {
	162	+	case 0 => put(&mut f, sources, 4, 0),
	163	+	case 1 => put(&mut f, sources, 4, 129),
	164	+	case 2 => put(&mut f, contexts + 8, 4, 99),
	165	+	case 3 => put(&mut f, contexts + 4, 4, 11),
	166	+	case 4 => put(&mut f, contexts + 12, 4, 9),
	167	+	case 5 => put(&mut f, registers + 8, 8, 0x200000),
	168	+	else => put(&mut f, registers, 8, 0xc000002),
	169	+	}
	170	+	let mut rejected = false;
	171	+	try platform::decode(&f.bytes[..f.strings], &mut p) catch error {
	172	+	assert error == platform::Error::Invalid; set rejected = true;
	173	+	};
	174	+	assert rejected;
	175	+	}
	176	+	}
	177	+
	178	+	/// PLIC register selection uses the decoded machine context and rejects ambiguous mappings.
	179	+	@test unsafe fn plicMapping() throws (testing::TestError) {
	180	+	let mut f: Fixture = undefined; construct(&mut f, true);
	181	+	let mut p: platform::Platform = undefined;
	182	+	try! platform::decode(&f.bytes[..f.strings], &mut p);
	183	+	let selected = try! plic::mapping(&p) else panic "missing PLIC";
	184	+	assert selected.base == 0xc000000 and selected.count == 31 and selected.context == 1;
	185	+	set p.irqContexts[0] = 16;
	186	+	let mut rejected = false;
	187	+	try plic::mapping(&p) catch error {
	188	+	assert error == abi::Error::InvalidArg;
	189	+	set rejected = true;
	190	+	};
	191	+	assert rejected;
	192	+	set p.irqContexts[0] = 1;
	193	+	set p.devices[2] = p.devices[1]; set p.deviceCount = 3;
	194	+	let mut duplicate = false;
	195	+	try plic::mapping(&p) catch error {
	196	+	assert error == abi::Error::InvalidArg;
	197	+	set duplicate = true;
	198	+	};
	199	+	assert duplicate;
	200	+	set p.irqSources = 0;
	201	+	assert try! plic::mapping(&p) == nil;
	202	+	}
	203	+
110	204		/// Resources and their reservations come from the tree rather than fixed addresses.
111	205		@test unsafe fn discovery() throws (testing::TestError) {
112	206		let mut f: Fixture = undefined; build(&mut f);
113	207		let mut p: platform::Platform = undefined;
114	208		try! platform::decode(&f.bytes[..f.strings], &mut p);