radiance · commit

kernel: Allocate and split physical memory

b13595ab029f512923e049284a5a32056419e7d21e2b5f50385e128c84babdcd

Assisted-by: Codex:gpt-6

Alexis Sellier committed 14 days ago 1 parent 0d03396d

kernel/kernel.rad +5 -2

export mod limits;
export mod slots;
export mod range;
export mod sync;
export mod platform;
@test export mod tests;
export mod trap;
export mod boot;
export mod capability;
export mod frames;
export mod backing;
export mod pages;
export mod boot;
@test export mod tests;

kernel/kernel/backing.rad added +147 -0

1	+	//! Allocation lifetimes shared by split pages and exposed domains.
2	+
3	+	use std::lang::gen::bitset;
4	+	use super::abi;
5	+	use super::limits;
6	+	use super::slots;
7	+	use super::frames;
8	+
9	+	/// Words required for one exposure bit per domain slot.
10	+	constant EXPOSURE_WORDS: u32 = (limits::DOMAINS + 31) / 32;
11	+
12	+	/// A physical allocation retained independently of its current page objects.
13	+	export record Allocation: Copy {
14	+	/// Complete physical frame ownership, unchanged by page splits.
15	+	run: frames::Run,
16	+	/// Number of page objects with live handles.
17	+	pages: u32,
18	+	/// Number of live domains with lifetime exposure.
19	+	exposed: u32,
20	+	/// Domain-index exposure bitmap. Domain exit clears bits before slot reuse.
21	+	exposures: [u32; EXPOSURE_WORDS],
22	+	}
23	+
24	+	/// Fixed allocation records and generation-checked exposure-domain membership.
25	+	export record Store: Copy {
26	+	/// Physical frames owned by backing allocations.
27	+	pool: frames::Pool,
28	+	/// Backing-record lifetimes.
29	+	slots: [slots::Slot; limits::ALLOCATIONS],
30	+	/// Payloads for live backing records.
31	+	records: [Allocation; limits::ALLOCATIONS],
32	+	/// Live domain generation, or zero after all of that domain's exposures end.
33	+	domains: [u32; limits::DOMAINS],
34	+	}
35	+
36	+	/// Initialize fresh backing and exposure-domain tables.
37	+	export fn initialize(store: &mut Store) {
38	+	set store.pool.count = 0;
39	+	slots::initialize(&mut store.slots[..]);
40	+	for i in 0..limits::DOMAINS {
41	+	set store.domains[i] = 0;
42	+	}
43	+	}
44	+
45	+	/// Register a domain before it can receive allocation exposure.
46	+	export fn registerDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {
47	+	if domain.index >= limits::DOMAINS or domain.generation == 0 {
48	+	throw abi::Error::InvalidArg;
49	+	}
50	+	if store.domains[domain.index] <> 0 {
51	+	throw abi::Error::Busy;
52	+	}
53	+	set store.domains[domain.index] = domain.generation;
54	+	}
55	+
56	+	/// Test exposure-domain generation without reading beyond the fixed table.
57	+	export fn domainLive(store: &Store, domain: abi::Ref) -> bool {
58	+	return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation;
59	+	}
60	+
61	+	/// Publish reserved metadata with one page object and its live origin exposure.
62	+	/// The caller owns the committed run and has validated the origin domain.
63	+	export unsafe fn publish(store: &mut Store, reservation: slots::Reservation, run: frames::Run, origin: abi::Ref) -> abi::Ref {
64	+	assert domainLive(store, origin);
65	+	let object = slots::reference(&reservation);
66	+	assert slots::matches(&store.slots[..], object, slots::State::Reserved);
67	+	set store.records[object.index].run = run;
68	+	set store.records[object.index].pages = 1;
69	+	set store.records[object.index].exposed = 1;
70	+	let mut bits = bitset::init(&mut store.records[object.index].exposures[..]);
71	+	bitset::put(&mut bits, origin.index);
72	+	return try! slots::commit(&mut store.slots[..], reservation);
73	+	}
74	+
75	+	/// Require a live generation-bearing backing reference.
76	+	fn require(store: &Store, object: abi::Ref) throws (abi::Error) {
77	+	if not slots::matches(&store.slots[..], object, slots::State::Live) {
78	+	throw abi::Error::BadHandle;
79	+	}
80	+	}
81	+
82	+	/// Add one split page to the allocation's lifetime count.
83	+	export fn retainPage(store: &mut Store, object: abi::Ref) throws (abi::Error) {
84	+	try require(store, object);
85	+	if store.records[object.index].pages == 0 {
86	+	throw abi::Error::BadHandle;
87	+	}
88	+	if store.records[object.index].pages == 0xffffffff {
89	+	throw abi::Error::Exhausted;
90	+	}
91	+	set store.records[object.index].pages += 1;
92	+	}
93	+
94	+	/// Record access for a domain for the rest of that domain generation's lifetime.
95	+	export unsafe fn expose(store: &mut Store, object: abi::Ref, domain: abi::Ref) throws (abi::Error) {
96	+	try require(store, object);
97	+	if store.records[object.index].pages == 0 or not domainLive(store, domain) {
98	+	throw abi::Error::BadHandle;
99	+	}
100	+	let mut bits = bitset::new(&mut store.records[object.index].exposures[..]);
101	+	if bitset::contains(&bits, domain.index) {
102	+	return;
103	+	}
104	+	bitset::put(&mut bits, domain.index);
105	+	set store.records[object.index].exposed += 1;
106	+	}
107	+
108	+	/// Reclaim only when no page object and no domain exposure remains.
109	+	fn reclaim(store: &mut Store, object: abi::Ref) {
110	+	if store.records[object.index].pages <> 0 or store.records[object.index].exposed <> 0 {
111	+	return;
112	+	}
113	+	let run = store.records[object.index].run;
114	+	try! frames::release(&mut store.pool, run);
115	+	try! slots::release(&mut store.slots[..], object);
116	+	}
117	+
118	+	/// Release one page object after its final handle disappears.
119	+	export fn releasePage(store: &mut Store, object: abi::Ref) throws (abi::Error) {
120	+	try require(store, object);
121	+	if store.records[object.index].pages == 0 {
122	+	throw abi::Error::BadHandle;
123	+	}
124	+	set store.records[object.index].pages -= 1;
125	+	reclaim(store, object);
126	+	}
127	+
128	+	/// End all exposure for one domain before its slot can be registered again.
129	+	/// Callers serialize this bounded scan with allocation and capability changes.
130	+	export unsafe fn endDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {
131	+	if not domainLive(store, domain) {
132	+	throw abi::Error::BadHandle;
133	+	}
134	+	for i in 0..limits::ALLOCATIONS {
135	+	if store.slots[i].state <> slots::State::Live {
136	+	continue;
137	+	}
138	+	let mut bits = bitset::new(&mut store.records[i].exposures[..]);
139	+	if not bitset::contains(&bits, domain.index) {
140	+	continue;
141	+	}
142	+	bitset::clear(&mut bits, domain.index);
143	+	set store.records[i].exposed -= 1;
144	+	reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation });
145	+	}
146	+	set store.domains[domain.index] = 0;
147	+	}

kernel/kernel/boot.rad +2 -0

use super::fdt;
use super::range;
use super::limits;
use super::sync;
use super::trap;
use super::pages;

/// Platform data published by hart zero before secondary initialization.
export unsafe static PLATFORM: platform::Platform = undefined;
/// Release/acquire publication flag for PLATFORM.
static READY: u64 = 0;

        let blob = @sliceOf(tree, size);
        try! platform::decode(&blob[..], &mut PLATFORM);
        let treeRange = range::new(treeAddress, size as u64) else panic "FDT range";
        assert platform::inRam(&PLATFORM, treeRange);
        try! platform::protect(&mut PLATFORM, treeRange);
        try! pages::initialize(&mut pages::STORE, &PLATFORM);
        print("kernel: platform ready\n");
        sync::storeRelease(&mut READY, 1);
    } else {
        while sync::loadAcquire(&READY) == 0 {
        }

kernel/kernel/boot.ras +2 -1

    csrw mstatus %zero;
    csrw mscratch %sp;
    mv %a2 %sp;
    addi %sp %sp -16;
    sd %ra 0(%sp);
    call @kernel::boot::enter;
    la %t0 @kernel::boot::enter;
    jalr %ra %t0 0;
    ld %ra 0(%sp);
    addi %sp %sp 16;
    ret;

@kernel::boot::read8

kernel/kernel/capability.rad +7 -0

    if entry.kind == abi::Kind::Empty or entry.object.generation == 0 {
        throw abi::Error::InvalidArg;
    }
    let rights = try abi::rights(*entry.rights as u64);
    let reservation = try slots::reserve(&mut table.slots[..]);
    return publish(table, reservation, entry);
}

/// Publish valid authority into a reservation from this table.
/// The caller has validated metadata and acquired the object's lifetime reference.
export fn publish(table: &mut Table, reservation: slots::Reservation, entry: Entry) -> abi::Handle {
    let object = slots::reference(&reservation);
    assert slots::matches(&table.slots[..], object, slots::State::Reserved);
    set table.entries[object.index] = entry;
    let live = try! slots::commit(&mut table.slots[..], reservation);
    return try! abi::handle(entry.kind, live);
}


kernel/kernel/frames.rad added +167 -0

1	+	//! Bounded physical-frame reservations. Callers serialize pool mutations.
2	+
3	+	use super::abi;
4	+	use super::limits;
5	+	use super::platform;
6	+	use super::range;
7	+
8	+	/// A contiguous allocation represented by indices in the physical-frame map.
9	+	export record Run: Copy {
10	+	/// First frame-map index.
11	+	first: u32,
12	+	/// Number of physically adjacent frames.
13	+	count: u32,
14	+	}
15	+
16	+	/// Reserved frames that must be committed or cancelled in the same pool.
17	+	export union Reservation: Once {
18	+	/// Frames excluded from subsequent allocation scans.
19	+	Held(Run),
20	+	}
21	+
22	+	/// Physical addresses and occupancy of allocatable frames, sorted by address.
23	+	export record Pool: Copy {
24	+	/// Physical base address of each eligible frame.
25	+	addresses: [u64; limits::FRAMES],
26	+	/// True when the frame is available for reservation.
27	+	free: [bool; limits::FRAMES],
28	+	/// Number of valid frame-map entries.
29	+	count: u32,
30	+	}
31	+
32	+	/// Insert one range into an address-sorted prefix of fixed storage.
33	+	fn insert(ranges: &mut [range::Range], count: u32, value: range::Range) {
34	+	let mut at = count;
35	+	while at > 0 and ranges[at - 1].start > value.start {
36	+	set ranges[at] = ranges[at - 1]; set at -= 1;
37	+	}
38	+	set ranges[at] = value;
39	+	}
40	+
41	+	/// Build a fresh frame map from complete RAM pages with no reserved byte.
42	+	/// The pool has zero valid entries if initialization fails.
43	+	export unsafe fn initialize(pool: &mut Pool, machine: &platform::Platform) throws (abi::Error) {
44	+	set pool.count = 0;
45	+	if machine.ramCount > platform::RAM_BANKS or machine.reservedCount > platform::RESERVATIONS {
46	+	throw abi::Error::InvalidArg;
47	+	}
48	+	let mut reserved: [range::Range; platform::RESERVATIONS] = undefined;
49	+	for i in 0..machine.reservedCount {
50	+	let value = machine.reserved[i];
51	+	if value.start >= value.end {
52	+	throw abi::Error::InvalidArg;
53	+	}
54	+	insert(&mut reserved[..], i, value);
55	+	}
56	+	let mut banks: [range::Range; platform::RAM_BANKS] = undefined;
57	+	for i in 0..machine.ramCount {
58	+	let bank = machine.ram[i];
59	+	if bank.start >= bank.end {
60	+	throw abi::Error::InvalidArg;
61	+	}
62	+	insert(&mut banks[..], i, bank);
63	+	}
64	+	let mut count: u32 = 0;
65	+	let mut next: u32 = 0;
66	+	for i in 0..machine.ramCount {
67	+	let bank = banks[i];
68	+	if i > 0 and banks[i - 1].end > bank.start {
69	+	throw abi::Error::InvalidArg;
70	+	}
71	+	let padding = (limits::FRAME_SIZE - (bank.start & (limits::FRAME_SIZE - 1))) & (limits::FRAME_SIZE - 1);
72	+	if padding > bank.end - bank.start {
73	+	continue;
74	+	}
75	+	let mut address = bank.start + padding;
76	+	let end = bank.end & ~(limits::FRAME_SIZE - 1);
77	+	while address < end {
78	+	while next < machine.reservedCount and reserved[next].end <= address {
79	+	set next += 1;
80	+	}
81	+	if next < machine.reservedCount and reserved[next].start < address + limits::FRAME_SIZE {
82	+	if reserved[next].end >= end {
83	+	break;
84	+	}
85	+	set address = (reserved[next].end + limits::FRAME_SIZE - 1) & ~(limits::FRAME_SIZE - 1);
86	+	continue;
87	+	}
88	+	if count == limits::FRAMES {
89	+	throw abi::Error::Exhausted;
90	+	}
91	+	set pool.addresses[count] = address;
92	+	set pool.free[count] = true;
93	+	set count += 1;
94	+	set address += limits::FRAME_SIZE;
95	+	}
96	+	}
97	+	set pool.count = count;
98	+	}
99	+
100	+	/// Reserve the first physically contiguous free run with a bounded scan.
101	+	export fn reserve(pool: &mut Pool, count: u32) -> Reservation throws (abi::Error) {
102	+	if count == 0 {
103	+	throw abi::Error::InvalidArg;
104	+	}
105	+	let mut first: u32 = 0;
106	+	let mut found: u32 = 0;
107	+	let mut previous: u64 = 0;
108	+	for i in 0..pool.count {
109	+	if not pool.free[i] {
110	+	set found = 0;
111	+	continue;
112	+	}
113	+	if found == 0 or pool.addresses[i] <> previous + limits::FRAME_SIZE {
114	+	set first = i;
115	+	set found = 0;
116	+	}
117	+	set found += 1;
118	+	set previous = pool.addresses[i];
119	+	if found == count {
120	+	for j in first..first + count {
121	+	set pool.free[j] = false;
122	+	}
123	+	return Reservation::Held(Run { first, count });
124	+	}
125	+	}
126	+	throw abi::Error::OutOfMemory;
127	+	}
128	+
129	+	/// Consume a reservation into a backing allocation's frame ownership.
130	+	export fn commit(reservation: Reservation) -> Run {
131	+	match reservation { case Reservation::Held(run) => return run, }
132	+	}
133	+
134	+	/// Resolve a complete physical run and reject map gaps or invalid indices.
135	+	export fn extent(pool: &Pool, run: Run) -> range::Range throws (abi::Error) {
136	+	if run.count == 0 or run.first >= pool.count or run.count > pool.count - run.first {
137	+	throw abi::Error::BadHandle;
138	+	}
139	+	let start = pool.addresses[run.first];
140	+	let length = run.count as u64 * limits::FRAME_SIZE;
141	+	let result = range::new(start, length) else {
142	+	throw abi::Error::BadHandle;
143	+	};
144	+	if pool.addresses[run.first + run.count - 1] <> result.end - limits::FRAME_SIZE {
145	+	throw abi::Error::BadHandle;
146	+	}
147	+	return result;
148	+	}
149	+
150	+	/// Return a backing allocation's frames after all handles and exposures end.
151	+	/// Only the owner of a committed run may release it.
152	+	export fn release(pool: &mut Pool, run: Run) throws (abi::Error) {
153	+	let memory = try extent(pool, run);
154	+	for i in run.first..run.first + run.count {
155	+	if pool.free[i] {
156	+	throw abi::Error::BadHandle;
157	+	}
158	+	}
159	+	for i in run.first..run.first + run.count {
160	+	set pool.free[i] = true;
161	+	}
162	+	}
163	+
164	+	/// Cancel unpublished frame ownership and restore its capacity.
165	+	export fn cancel(pool: &mut Pool, reservation: Reservation) throws (abi::Error) {
166	+	match reservation { case Reservation::Held(run) => try release(pool, run), }
167	+	}

kernel/kernel/pages.rad added +150 -0

1	+	//! Physical page objects, allocation, and exclusive splitting.
2	+
3	+	use super::abi;
4	+	use super::limits;
5	+	use super::slots;
6	+	use super::frames;
7	+	use super::backing;
8	+	use super::capability;
9	+	use super::platform;
10	+
11	+	/// Page capability rights created by physical allocation.
12	+	export constant DEFAULT_RIGHTS: u16 = abi::READ \| abi::WRITE \| abi::GRANT \| abi::TRANSFER;
13	+
14	+	/// One page-aligned segment of a retained backing allocation.
15	+	export record Page: Copy {
16	+	/// Common backing record, including the complete allocation's exposures.
17	+	backing: abi::Ref,
18	+	/// First physical byte of this segment.
19	+	base: u64,
20	+	/// Number of contiguous frames in this segment.
21	+	count: u32,
22	+	/// Domain that created the original allocation.
23	+	origin: abi::Ref,
24	+	/// Number of live capabilities that name this page object.
25	+	handles: u32,
26	+	}
27	+
28	+	/// Fixed physical-memory metadata. Callers serialize shared mutations.
29	+	export record Store: Copy {
30	+	/// Page-object slot lifetimes.
31	+	slots: [slots::Slot; limits::PAGES],
32	+	/// Live page-object payloads.
33	+	records: [Page; limits::PAGES],
34	+	/// Backing allocations and domain exposures.
35	+	backings: backing::Store,
36	+	}
37	+
38	+	/// Physical-memory state shared by the kernel.
39	+	export unsafe static STORE: Store = undefined;
40	+
41	+	/// Zero complete, mapped physical frames. The base must be page-aligned.
42	+	fn zero(base: u64, count: u32);
43	+
44	+	/// Initialize physical-memory metadata before creating domains or pages.
45	+	export unsafe fn initialize(store: &mut Store, machine: &platform::Platform) throws (abi::Error) {
46	+	backing::initialize(&mut store.backings);
47	+	try frames::initialize(&mut store.backings.pool, machine);
48	+	slots::initialize(&mut store.slots[..]);
49	+	}
50	+
51	+	/// Read a live page object through its internal generation-bearing reference.
52	+	export fn get(store: &Store, object: abi::Ref) -> Page throws (abi::Error) {
53	+	if not slots::matches(&store.slots[..], object, slots::State::Live) {
54	+	throw abi::Error::BadHandle;
55	+	}
56	+	return store.records[object.index];
57	+	}
58	+
59	+	/// Allocate and zero frames before publishing a page and its capability.
60	+	/// Allocation authority must name the calling domain. Failure cancels reservations.
61	+	export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
62	+	-> abi::Handle throws (abi::Error)
63	+	{
64	+	if count == 0 {
65	+	throw abi::Error::InvalidArg;
66	+	}
67	+	if count > limits::FRAMES as u64 {
68	+	throw abi::Error::OutOfMemory;
69	+	}
70	+	let owner = table.owner;
71	+	let permission = try capability::authority(table, authority, abi::Rights(abi::ALLOCATE));
72	+	if permission.object <> owner {
73	+	throw abi::Error::Denied;
74	+	}
75	+	if not backing::domainLive(&store.backings, owner) {
76	+	throw abi::Error::BadHandle;
77	+	}
78	+	let handleSlot = try slots::reserve(&mut table.slots[..]);
79	+	let pageSlot = try slots::reserve(&mut store.slots[..]) catch err {
80	+	try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
81	+	};
82	+	let backingSlot = try slots::reserve(&mut store.backings.slots[..]) catch err {
83	+	try! slots::cancel(&mut store.slots[..], pageSlot);
84	+	try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
85	+	};
86	+	let memory = try frames::reserve(&mut store.backings.pool, count as u32) catch err {
87	+	try! slots::cancel(&mut store.backings.slots[..], backingSlot);
88	+	try! slots::cancel(&mut store.slots[..], pageSlot);
89	+	try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
90	+	};
91	+	let run = frames::commit(memory);
92	+	let extent = try! frames::extent(&store.backings.pool, run);
93	+	zero(extent.start, run.count);
94	+	let allocation = backing::publish(&mut store.backings, backingSlot, run, owner);
95	+	let object = slots::reference(&pageSlot);
96	+	set store.records[object.index] = Page { backing: allocation, base: extent.start, count: run.count, origin: owner, handles: 1 };
97	+	let page = try! slots::commit(&mut store.slots[..], pageSlot);
98	+	return capability::publish(table, handleSlot, capability::Entry {
99	+	kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
100	+	});
101	+	}
102	+
103	+	/// Split an exclusive page into two segments with the same backing and rights.
104	+	export fn split(store: &mut Store, table: &mut capability::Table, handle: abi::Handle, leftCount: u64)
105	+	-> abi::Handle throws (abi::Error)
106	+	{
107	+	let entry = try capability::lookup(table, handle, abi::Kind::Page, abi::Rights(0));
108	+	let source = try get(store, entry.object);
109	+	if leftCount == 0 or leftCount >= source.count as u64 {
110	+	throw abi::Error::InvalidArg;
111	+	}
112	+	if source.handles <> 1 {
113	+	throw abi::Error::Busy;
114	+	}
115	+	let handleSlot = try slots::reserve(&mut table.slots[..]);
116	+	let pageSlot = try slots::reserve(&mut store.slots[..]) catch err {
117	+	try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
118	+	};
119	+	let object = slots::reference(&pageSlot);
120	+	try! backing::retainPage(&mut store.backings, source.backing);
121	+	set store.records[object.index] = Page {
122	+	backing: source.backing, base: source.base + leftCount * limits::FRAME_SIZE,
123	+	count: source.count - leftCount as u32, origin: source.origin, handles: 1,
124	+	};
125	+	set store.records[entry.object.index].count = leftCount as u32;
126	+	let page = try! slots::commit(&mut store.slots[..], pageSlot);
127	+	return capability::publish(table, handleSlot, capability::Entry { kind: abi::Kind::Page, object: page, rights: entry.rights });
128	+	}
129	+
130	+	/// Add a capability reference and lifetime exposure before publishing that capability.
131	+	export unsafe fn retain(store: &mut Store, object: abi::Ref, receiver: abi::Ref) throws (abi::Error) {
132	+	let page = try get(store, object);
133	+	if page.handles == 0xffffffff {
134	+	throw abi::Error::Exhausted;
135	+	}
136	+	try backing::expose(&mut store.backings, page.backing, receiver);
137	+	set store.records[object.index].handles += 1;
138	+	}
139	+
140	+	/// Release a capability reference. The backing retains all lifetime exposures.
141	+	export fn release(store: &mut Store, object: abi::Ref) throws (abi::Error) {
142	+	let page = try get(store, object);
143	+	assert page.handles > 0;
144	+	set store.records[object.index].handles -= 1;
145	+	if page.handles > 1 {
146	+	return;
147	+	}
148	+	try! slots::release(&mut store.slots[..], object);
149	+	try! backing::releasePage(&mut store.backings, page.backing);
150	+	}

kernel/kernel/pages.ras added +15 -0

1	+	//! Zero physical frames before capability publication.
2	+	.text;
3	+	.export @kernel::pages::zero;
4	+
5	+	// Physical frame bases and lengths are multiples of 4096 bytes.
6	+	@kernel::pages::zero
7	+	beqz %a1 @done;
8	+	slli %a1 %a1 12;
9	+	add %t0 %a0 %a1;
10	+	@clear
11	+	sd %zero 0(%a0);
12	+	addi %a0 %a0 8;
13	+	bne %a0 %t0 @clear;
14	+	@done
15	+	ret;

kernel/kernel/tests.rad +3 -0

export mod slots;
export mod fdt;
export mod platform;
export mod trap;
export mod capability;
export mod frames;
export mod backing;
export mod pages;

kernel/kernel/tests/backing.rad added +100 -0

1	+	//! Backing allocations outlive page objects until every exposure ends.
2	+
3	+	use std::testing;
4	+	use kernel::abi;
5	+	use kernel::frames;
6	+	use kernel::backing;
7	+	use kernel::slots;
8	+
9	+	/// Fixed backing-store workspace.
10	+	unsafe static STORE: backing::Store = undefined;
11	+
12	+	/// Initialize two domains and four synthetic physical frames.
13	+	unsafe fn initialize() {
14	+	backing::initialize(&mut STORE);
15	+	try! backing::registerDomain(&mut STORE, abi::Ref { index: 0, generation: 1 });
16	+	try! backing::registerDomain(&mut STORE, abi::Ref { index: 1, generation: 1 });
17	+	set STORE.pool.count = 4;
18	+	for i in 0..4 {
19	+	set STORE.pool.addresses[i] = 0x80000000 + i as u64 * 4096;
20	+	set STORE.pool.free[i] = true;
21	+	}
22	+	}
23	+
24	+	/// Publish one backing record with its initial page and origin exposure.
25	+	unsafe fn allocate() -> abi::Ref {
26	+	let slot = try! slots::reserve(&mut STORE.slots[..]);
27	+	let reservation = try! frames::reserve(&mut STORE.pool, 4);
28	+	let run = frames::commit(reservation);
29	+	return backing::publish(&mut STORE, slot, run, abi::Ref { index: 0, generation: 1 });
30	+	}
31	+
32	+	/// Dropping all page handles preserves capacity while an exposed domain lives.
33	+	@test unsafe fn exposures() throws (testing::TestError) {
34	+	initialize();
35	+	let object = allocate();
36	+	let receiver = abi::Ref { index: 1, generation: 1 };
37	+	try! backing::expose(&mut STORE, object, receiver);
38	+	try! backing::expose(&mut STORE, object, receiver);
39	+	try testing::expect(STORE.records[object.index].exposed == 2);
40	+	try! backing::releasePage(&mut STORE, object);
41	+	try! backing::endDomain(&mut STORE, abi::Ref { index: 0, generation: 1 });
42	+	try testing::expect(not STORE.pool.free[0] and slots::matches(&STORE.slots[..], object, slots::State::Live));
43	+	try! backing::endDomain(&mut STORE, receiver);
44	+	for i in 0..4 {
45	+	try testing::expect(STORE.pool.free[i]);
46	+	}
47	+	try testing::expect(not slots::matches(&STORE.slots[..], object, slots::State::Live));
48	+	}
49	+
50	+	/// Splits keep their common backing alive until every page object is gone.
51	+	@test unsafe fn splitLifetime() throws (testing::TestError) {
52	+	initialize();
53	+	let object = allocate();
54	+	try! backing::retainPage(&mut STORE, object);
55	+	try! backing::endDomain(&mut STORE, abi::Ref { index: 0, generation: 1 });
56	+	try! backing::releasePage(&mut STORE, object);
57	+	try testing::expect(not STORE.pool.free[0] and STORE.records[object.index].pages == 1);
58	+	try! backing::releasePage(&mut STORE, object);
59	+	try testing::expect(STORE.pool.free[0]);
60	+	let mut stale = false;
61	+	try backing::releasePage(&mut STORE, object) catch err {
62	+	try testing::expect(err == abi::Error::BadHandle); set stale = true;
63	+	};
64	+	try testing::expect(stale);
65	+	}
66	+
67	+	/// An ended domain generation cannot gain exposures through a reused domain slot.
68	+	@test unsafe fn domainGeneration() throws (testing::TestError) {
69	+	initialize();
70	+	let object = allocate();
71	+	let old = abi::Ref { index: 1, generation: 1 };
72	+	try! backing::expose(&mut STORE, object, old);
73	+	try! backing::endDomain(&mut STORE, old);
74	+	try! backing::registerDomain(&mut STORE, abi::Ref { index: 1, generation: 2 });
75	+	let mut stale = false;
76	+	try backing::expose(&mut STORE, object, old) catch err {
77	+	try testing::expect(err == abi::Error::BadHandle); set stale = true;
78	+	};
79	+	try testing::expect(stale and STORE.records[object.index].exposed == 1);
80	+	try! backing::releasePage(&mut STORE, object);
81	+	try! backing::endDomain(&mut STORE, abi::Ref { index: 0, generation: 1 });
82	+	try testing::expect(STORE.pool.free[0]);
83	+	}
84	+
85	+	/// Retained backing metadata cannot create new page authority after the last page ends.
86	+	@test unsafe fn noResurrection() throws (testing::TestError) {
87	+	initialize();
88	+	let object = allocate();
89	+	try! backing::releasePage(&mut STORE, object);
90	+	let mut failures: u32 = 0;
91	+	try backing::retainPage(&mut STORE, object) catch err {
92	+	try testing::expect(err == abi::Error::BadHandle); set failures += 1;
93	+	};
94	+	try backing::expose(&mut STORE, object, abi::Ref { index: 1, generation: 1 }) catch err {
95	+	try testing::expect(err == abi::Error::BadHandle); set failures += 1;
96	+	};
97	+	try testing::expect(failures == 2 and STORE.records[object.index].exposed == 1);
98	+	try! backing::endDomain(&mut STORE, abi::Ref { index: 0, generation: 1 });
99	+	try testing::expect(STORE.pool.free[0]);
100	+	}

kernel/kernel/tests/frames.rad added +173 -0

1	+	//! Contiguous frame reservation and reserved-memory exclusion.
2	+
3	+	use std::testing;
4	+	use kernel::abi;
5	+	use kernel::frames;
6	+	use kernel::platform;
7	+	use kernel::range;
8	+
9	+	/// Frame-map workspace is static because its size exceeds a small kernel stack.
10	+	unsafe static POOL: frames::Pool = undefined;
11	+
12	+	/// Build eight available frames around a reserved two-frame hole.
13	+	unsafe fn initialize() {
14	+	let mut machine: platform::Platform = undefined;
15	+	set machine.ramCount = 1;
16	+	set machine.ram[0] = range::Range { start: 0x80000000, end: 0x8000a000 };
17	+	set machine.reservedCount = 1;
18	+	set machine.reserved[0] = range::Range { start: 0x80003000, end: 0x80005000 };
19	+	try! frames::initialize(&mut POOL, &machine);
20	+	}
21	+
22	+	/// Adjacent physical frames form runs; a reserved hole cannot be crossed.
23	+	@test unsafe fn fragmentation() throws (testing::TestError) {
24	+	initialize();
25	+	try testing::expect(POOL.count == 8);
26	+	let left = try! frames::reserve(&mut POOL, 3);
27	+	let leftRun = frames::commit(left);
28	+	let right = try! frames::reserve(&mut POOL, 5);
29	+	let rightRun = frames::commit(right);
30	+	let first = try! frames::extent(&POOL, leftRun);
31	+	let second = try! frames::extent(&POOL, rightRun);
32	+	try testing::expect(first.start == 0x80000000 and first.end == 0x80003000);
33	+	try testing::expect(second.start == 0x80005000 and second.end == 0x8000a000);
34	+	try! frames::release(&mut POOL, leftRun);
35	+	try! frames::release(&mut POOL, rightRun);
36	+	let mut failed = false;
37	+	try reserveAndCancel(6) catch err {
38	+	try testing::expect(err == abi::Error::OutOfMemory); set failed = true;
39	+	};
40	+	try testing::expect(failed);
41	+	}
42	+
43	+	/// Consume a successful probe reservation before returning to its caller.
44	+	unsafe fn reserveAndCancel(count: u32) throws (abi::Error) {
45	+	let token = try frames::reserve(&mut POOL, count);
46	+	try frames::cancel(&mut POOL, token);
47	+	}
48	+
49	+	/// Full exhaustion and cancellation preserve the pool's available capacity.
50	+	@test unsafe fn capacity() throws (testing::TestError) {
51	+	initialize();
52	+	let mut allocated: [frames::Run; 8] = undefined;
53	+	for i in 0..8 {
54	+	let token = try! frames::reserve(&mut POOL, 1);
55	+	set allocated[i] = frames::commit(token);
56	+	}
57	+	let mut failed = false;
58	+	try reserveAndCancel(1) catch err {
59	+	try testing::expect(err == abi::Error::OutOfMemory); set failed = true;
60	+	};
61	+	try testing::expect(failed);
62	+	for i in 0..8 {
63	+	try! frames::release(&mut POOL, allocated[i]);
64	+	}
65	+	let pending = try! frames::reserve(&mut POOL, 5);
66	+	try! frames::cancel(&mut POOL, pending);
67	+	let again = try! frames::reserve(&mut POOL, 5);
68	+	try! frames::cancel(&mut POOL, again);
69	+	let mut empty = false;
70	+	try reserveAndCancel(0) catch err {
71	+	try testing::expect(err == abi::Error::InvalidArg); set empty = true;
72	+	};
73	+	try testing::expect(empty);
74	+	}
75	+
76	+	/// A release validates every frame before mutating any occupancy state.
77	+	@test unsafe fn invalidRelease() throws (testing::TestError) {
78	+	initialize();
79	+	let token = try! frames::reserve(&mut POOL, 2);
80	+	let run = frames::commit(token);
81	+	let mut failed = false;
82	+	try frames::release(&mut POOL, frames::Run { first: run.first, count: 3 }) catch err {
83	+	try testing::expect(err == abi::Error::BadHandle); set failed = true;
84	+	};
85	+	try testing::expect(failed);
86	+	try! frames::release(&mut POOL, run);
87	+	let mut duplicate = false;
88	+	try frames::release(&mut POOL, run) catch err {
89	+	try testing::expect(err == abi::Error::BadHandle); set duplicate = true;
90	+	};
91	+	try testing::expect(duplicate);
92	+	}
93	+
94	+	/// Only complete, unreserved physical pages can enter the frame pool.
95	+	@test unsafe fn partialPages() throws (testing::TestError) {
96	+	let mut machine: platform::Platform = undefined;
97	+	set machine.ramCount = 1;
98	+	set machine.ram[0] = range::Range { start: 0x80000001, end: 0x80006001 };
99	+	set machine.reservedCount = 1;
100	+	set machine.reserved[0] = range::Range { start: 0x80002fff, end: 0x80003001 };
101	+	try! frames::initialize(&mut POOL, &machine);
102	+	try testing::expect(POOL.count == 3);
103	+	let token = try! frames::reserve(&mut POOL, 2);
104	+	let run = frames::commit(token);
105	+	let span = try! frames::extent(&POOL, run);
106	+	try testing::expect(span.start == 0x80004000 and span.end == 0x80006000);
107	+	}
108	+
109	+	/// Physically adjacent banks form a run regardless of device-tree order.
110	+	@test unsafe fn bankOrder() throws (testing::TestError) {
111	+	let mut machine: platform::Platform = undefined;
112	+	set machine.ramCount = 2; set machine.reservedCount = 0;
113	+	set machine.ram[0] = range::Range { start: 0x80002000, end: 0x80004000 };
114	+	set machine.ram[1] = range::Range { start: 0x80000000, end: 0x80002000 };
115	+	try! frames::initialize(&mut POOL, &machine);
116	+	let token = try! frames::reserve(&mut POOL, 4);
117	+	let run = frames::commit(token);
118	+	let memory = try! frames::extent(&POOL, run);
119	+	try testing::expect(memory.start == 0x80000000 and memory.end == 0x80004000);
120	+	set machine.ram[1].end = 0x80003000;
121	+	let mut failed = false;
122	+	try frames::initialize(&mut POOL, &machine) catch err {
123	+	try testing::expect(err == abi::Error::InvalidArg); set failed = true;
124	+	};
125	+	try testing::expect(failed and POOL.count == 0);
126	+	}
127	+
128	+	/// The full 65,536-frame profile fits; one extra eligible frame fails cleanly.
129	+	@test unsafe fn maximum() throws (testing::TestError) {
130	+	let mut machine: platform::Platform = undefined;
131	+	set machine.ramCount = 1; set machine.reservedCount = 0;
132	+	set machine.ram[0] = range::Range { start: 0x80000000, end: 0x90000000 };
133	+	try! frames::initialize(&mut POOL, &machine);
134	+	try testing::expect(POOL.count == 65536);
135	+	let token = try! frames::reserve(&mut POOL, 65536);
136	+	let run = frames::commit(token);
137	+	try! frames::release(&mut POOL, run);
138	+	set machine.ram[0].end += 4096;
139	+	let mut failed = false;
140	+	try frames::initialize(&mut POOL, &machine) catch err {
141	+	try testing::expect(err == abi::Error::Exhausted); set failed = true;
142	+	};
143	+	try testing::expect(failed and POOL.count == 0);
144	+	set machine.ram[0] = range::Range { start: 0xfffffffffffff001, end: 0xffffffffffffffff };
145	+	try! frames::initialize(&mut POOL, &machine);
146	+	try testing::expect(POOL.count == 0);
147	+	}
148	+
149	+	/// Sorted reservation skipping agrees with byte-range exclusion across banks and gaps.
150	+	@test unsafe fn reservationSweep() throws (testing::TestError) {
151	+	let mut machine: platform::Platform = undefined;
152	+	set machine.ramCount = 3;
153	+	set machine.ram[0] = range::Range { start: 0x80040000, end: 0x80060000 };
154	+	set machine.ram[1] = range::Range { start: 0x80000000, end: 0x80010000 };
155	+	set machine.ram[2] = range::Range { start: 0x80010000, end: 0x80020000 };
156	+	set machine.reservedCount = 6;
157	+	set machine.reserved[0] = range::Range { start: 0x80050001, end: 0x8005f000 };
158	+	set machine.reserved[1] = range::Range { start: 0x80001234, end: 0x80004321 };
159	+	set machine.reserved[2] = range::Range { start: 0x80004000, end: 0x80009001 };
160	+	set machine.reserved[3] = range::Range { start: 0x7ffff000, end: 0x80000001 };
161	+	set machine.reserved[4] = range::Range { start: 0x80018000, end: 0x80048000 };
162	+	set machine.reserved[5] = range::Range { start: 0x80002000, end: 0x80003000 };
163	+	try! frames::initialize(&mut POOL, &machine);
164	+	let mut count: u32 = 0;
165	+	for i in 0..96 {
166	+	let start = 0x80000000 + i as u64 * 4096;
167	+	if platform::available(&machine, range::Range { start, end: start + 4096 }) {
168	+	try testing::expect(count < POOL.count and POOL.addresses[count] == start and POOL.free[count]);
169	+	set count += 1;
170	+	}
171	+	}
172	+	try testing::expect(count == POOL.count);
173	+	}

kernel/kernel/tests/pages.rad added +196 -0

1	+	//! Page allocation, zeroing, split atomicity, and retained-pointer lifetimes.
2	+
3	+	use std::testing;
4	+	use kernel::abi;
5	+	use kernel::capability;
6	+	use kernel::pages;
7	+	use kernel::backing;
8	+	use kernel::slots;
9	+	use kernel::limits;
10	+
11	+	/// Page and allocation metadata workspace.
12	+	unsafe static STORE: pages::Store = undefined;
13	+	/// Test domain's capability table.
14	+	unsafe static TABLE: capability::Table = undefined;
15	+	/// Mapped bytes with room for page alignment and eight full pages.
16	+	static RAM: [u8; 36864] = [0; 36864];
17	+	/// Byte offset to the first page-aligned test frame.
18	+	static OFFSET: u32 = 0;
19	+
20	+	/// Initialize metadata over mapped hosted memory, without synthetic addresses.
21	+	unsafe fn initialize() {
22	+	for i in 0..RAM.len {
23	+	set RAM[i] = 0xa5;
24	+	}
25	+	let pointer: *u8 = &RAM[0];
26	+	let address = pointer as u64;
27	+	let aligned = (address + 4095) & ~4095;
28	+	set OFFSET = (aligned - address) as u32;
29	+	slots::initialize(&mut STORE.slots[..]);
30	+	backing::initialize(&mut STORE.backings);
31	+	set STORE.backings.pool.count = 8;
32	+	for i in 0..8 {
33	+	set STORE.backings.pool.addresses[i] = aligned + i as u64 * 4096;
34	+	set STORE.backings.pool.free[i] = true;
35	+	}
36	+	let owner = abi::Ref { index: 0, generation: 1 };
37	+	capability::initialize(&mut TABLE, owner);
38	+	try! backing::registerDomain(&mut STORE.backings, owner);
39	+	let authority = try! capability::install(&mut TABLE, capability::Entry {
40	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::ALLOCATE),
41	+	});
42	+	}
43	+
44	+	/// Drop the test's handle and its object reference in the same serialized operation.
45	+	unsafe fn drop(handle: abi::Handle) {
46	+	let entry = try! capability::invalidate(&mut TABLE, handle);
47	+	try! pages::release(&mut STORE, entry.object);
48	+	}
49	+
50	+	/// New allocations are zeroed completely and carry the documented default rights.
51	+	@test unsafe fn allocation() throws (testing::TestError) {
52	+	initialize();
53	+	let handle = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 2);
54	+	let entry = try! capability::get(&TABLE, handle);
55	+	try testing::expect(*entry.rights == (abi::READ \| abi::WRITE \| abi::GRANT \| abi::TRANSFER));
56	+	let page = try! pages::get(&STORE, entry.object);
57	+	try testing::expect(page.count == 2 and page.origin == TABLE.owner and page.handles == 1);
58	+	for i in OFFSET..OFFSET + 8192 {
59	+	try testing::expect(RAM[i] == 0);
60	+	}
61	+	try testing::expect(RAM[OFFSET + 8192] == 0xa5);
62	+	let mut invalid = false;
63	+	try pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 0) catch err {
64	+	try testing::expect(err == abi::Error::InvalidArg); set invalid = true;
65	+	};
66	+	try testing::expect(invalid);
67	+	for count in &[65537 as u64, 0xffffffffffffffff] {
68	+	let mut exhausted = false;
69	+	try pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), count) catch err {
70	+	try testing::expect(err == abi::Error::OutOfMemory); set exhausted = true;
71	+	};
72	+	try testing::expect(exhausted and STORE.backings.pool.free[2]);
73	+	}
74	+	}
75	+
76	+	/// Splitting preserves contents and backing identity while changing only page extents.
77	+	@test unsafe fn split() throws (testing::TestError) {
78	+	initialize();
79	+	let handle = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 4);
80	+	set RAM[OFFSET + 8192] = 73;
81	+	let original = try! capability::get(&TABLE, handle);
82	+	for count in &[0 as u64, 4, 5, 0xffffffffffffffff] {
83	+	let mut failed = false;
84	+	try pages::split(&mut STORE, &mut TABLE, handle, count) catch err {
85	+	try testing::expect(err == abi::Error::InvalidArg); set failed = true;
86	+	};
87	+	try testing::expect(failed);
88	+	}
89	+	let rightHandle = try! pages::split(&mut STORE, &mut TABLE, handle, 2);
90	+	let rightEntry = try! capability::get(&TABLE, rightHandle);
91	+	let left = try! pages::get(&STORE, original.object);
92	+	let right = try! pages::get(&STORE, rightEntry.object);
93	+	try testing::expect(left.count == 2 and right.count == 2 and right.base == left.base + 8192);
94	+	try testing::expect(left.backing == right.backing and left.origin == right.origin);
95	+	try testing::expect(original.rights == rightEntry.rights and RAM[OFFSET + 8192] == 73);
96	+	try testing::expect(STORE.backings.records[left.backing.index].pages == 2);
97	+	try! pages::retain(&mut STORE, original.object, TABLE.owner);
98	+	let mut shared = false;
99	+	try pages::split(&mut STORE, &mut TABLE, handle, 1) catch err {
100	+	try testing::expect(err == abi::Error::Busy); set shared = true;
101	+	};
102	+	try testing::expect(shared);
103	+	try! pages::release(&mut STORE, original.object);
104	+	}
105	+
106	+	/// Neither handle drop nor origin exit can release frames exposed to another live domain.
107	+	@test unsafe fn retainedPointers() throws (testing::TestError) {
108	+	initialize();
109	+	let receiver = abi::Ref { index: 1, generation: 1 };
110	+	try! backing::registerDomain(&mut STORE.backings, receiver);
111	+	let handle = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 8);
112	+	let entry = try! capability::get(&TABLE, handle);
113	+	try! pages::retain(&mut STORE, entry.object, receiver);
114	+	set RAM[OFFSET] = 99;
115	+	drop(handle);
116	+	try! pages::release(&mut STORE, entry.object);
117	+	try! backing::endDomain(&mut STORE.backings, TABLE.owner);
118	+	try testing::expect(not STORE.backings.pool.free[0] and RAM[OFFSET] == 99);
119	+	try! backing::endDomain(&mut STORE.backings, receiver);
120	+	try testing::expect(STORE.backings.pool.free[0]);
121	+	let owner = abi::Ref { index: 0, generation: 2 };
122	+	capability::initialize(&mut TABLE, owner);
123	+	try! backing::registerDomain(&mut STORE.backings, owner);
124	+	let authority = try! capability::install(&mut TABLE, capability::Entry {
125	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::ALLOCATE),
126	+	});
127	+	let replacement = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 8);
128	+	try testing::expect(RAM[OFFSET] == 0);
129	+	}
130	+
131	+	/// A full capability table prevents allocation and splitting before payload mutation.
132	+	@test unsafe fn fullTable() throws (testing::TestError) {
133	+	initialize();
134	+	let handle = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 4);
135	+	let entry = try! capability::get(&TABLE, handle);
136	+	let owner = TABLE.owner;
137	+	for i in 2..limits::HANDLES {
138	+	let filler = try! capability::install(&mut TABLE, capability::Entry {
139	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(0),
140	+	});
141	+	}
142	+	let mut failed: u32 = 0;
143	+	try pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 1) catch err {
144	+	try testing::expect(err == abi::Error::Exhausted); set failed += 1;
145	+	};
146	+	try pages::split(&mut STORE, &mut TABLE, handle, 2) catch err {
147	+	try testing::expect(err == abi::Error::Exhausted); set failed += 1;
148	+	};
149	+	let page = try! pages::get(&STORE, entry.object);
150	+	try testing::expect(failed == 2 and page.count == 4 and STORE.backings.pool.free[4]);
151	+	}
152	+
153	+	/// Failure at each storage stage releases all earlier reservations.
154	+	@test unsafe fn rollback() throws (testing::TestError) {
155	+	initialize();
156	+	let full = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 8);
157	+	let mut failed = false;
158	+	try pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 1) catch err {
159	+	try testing::expect(err == abi::Error::OutOfMemory); set failed = true;
160	+	};
161	+	try testing::expect(failed and TABLE.slots[2].state == slots::State::Free);
162	+	try testing::expect(STORE.slots[1].state == slots::State::Free and STORE.backings.slots[1].state == slots::State::Free);
163	+	for stage in 0..2 {
164	+	initialize();
165	+	if stage == 0 {
166	+	for i in 0..limits::ALLOCATIONS {
167	+	set STORE.backings.slots[i].state = slots::State::Retired;
168	+	}
169	+	} else {
170	+	for i in 0..limits::PAGES {
171	+	set STORE.slots[i].state = slots::State::Retired;
172	+	}
173	+	}
174	+	let mut exhausted = false;
175	+	try pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 1) catch err {
176	+	try testing::expect(err == abi::Error::Exhausted); set exhausted = true;
177	+	};
178	+	try testing::expect(exhausted and TABLE.slots[1].state == slots::State::Free and STORE.backings.pool.free[0]);
179	+	if stage == 0 {
180	+	try testing::expect(STORE.slots[0].state == slots::State::Free);
181	+	}
182	+	}
183	+	}
184	+
185	+	/// A Domain handle with Allocate rights must name the caller to allocate RAM.
186	+	@test unsafe fn allocationAuthority() throws (testing::TestError) {
187	+	initialize();
188	+	let other = try! capability::install(&mut TABLE, capability::Entry {
189	+	kind: abi::Kind::Domain, object: abi::Ref { index: 1, generation: 1 }, rights: abi::Rights(abi::ALLOCATE),
190	+	});
191	+	let mut denied = false;
192	+	try pages::allocate(&mut STORE, &mut TABLE, other, 1) catch err {
193	+	try testing::expect(err == abi::Error::Denied); set denied = true;
194	+	};
195	+	try testing::expect(denied and STORE.backings.pool.free[0] and TABLE.slots[2].state == slots::State::Free);
196	+	}

kernel/tools/build.rad +64 -15

//! Build a physical kernel image from trusted binary RIL and startup assembly.

use std::sys;
use std::io;
use std::mem;
use std::lang::il;
use std::sys::unix;
use std::lang::alloc;
use std::lang::strings;
use std::lang::il::binary;
use std::lang::il::binary::program;

use std::collections::dict;
use std::arch::rv64;
use std::arch::rv64::asm;
use std::arch::rv64::image;

/// Code segment base for the native platform profile.
constant CODE_ADDRESS: u64 = 0x81000000;

/// Persistent native code-generation workspace.
static CODE: [u8; 16777216] = [0; 16777216];
/// Reusable function workspace.
static SCRATCH: [u8; 16777216] = [0; 16777216];
/// Decoded package storage.
static DECODE: [u8; 16777216] = [0; 16777216];
static DECODE: [u8; 67108864] = [0; 67108864];
/// Binary package input.
static INPUT: [u8; 1048576] = [0; 1048576];
static INPUT: [u8; 8388608] = [0; 8388608];
/// Combined package data descriptors.
unsafe static GLOBALS: [il::Data; 4096] = undefined;
/// Combined startup and boundary assembly.
static SOURCE: [u8; 65536] = [0; 65536];
/// Assembler workspace.
static ASSEMBLY: [u8; 4194304] = [0; 4194304];
/// Assembled startup words.
static TEXT: [u32; 16384] = [0; 16384];
/// Assembly identifiers.
unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };
/// Data symbol placement workspace.
unsafe static SYMBOLS: [data::DataSym; 1024] = undefined;
unsafe static SYMBOLS: [data::DataSym; 4096] = undefined;
/// Data name lookup workspace.
unsafe static ENTRIES: [dict::Entry; data::DATA_SYM_TABLE_SIZE] = undefined;
/// Initialized read-only bytes.
static RO: [u8; 1048576] = [0; 1048576];
/// Initialized writable bytes.
static RW: [u8; 1048576] = [0; 1048576];

/// Decode one package into persistent storage shared by the native link.
unsafe fn load(path: &[u8], arena: &mut alloc::Arena) -> binary::Package {
    let inputLength = unix::readFile(path, &mut INPUT[..]) else panic "boot RIL";
    return try! program::decode(&INPUT[..inputLength], arena, binary::Limits { registers: 8192, blocks: 4096 });
}

/// Print the bounded backend failure reported by a native build.
fn report(error: rv64::Error) {
    match error {
        case rv64::Error::Allocation => io::printLn("native build: allocation exhausted"),
        case rv64::Error::Capacity => io::printLn("native build: output capacity exhausted"),
        case rv64::Error::Symbol => io::printLn("native build: unresolved symbol"),
        case rv64::Error::Relocation => io::printLn("native build: relocation out of range"),
        case rv64::Error::Image(_) => io::printLn("native build: invalid image layout"),
        case rv64::Error::Data(_) => io::printLn("native build: invalid data layout"),
    }
}

/// Assemble entry code and lower the kernel at explicit physical addresses.
@default unsafe fn main(env: *sys::Env) -> i32 {
    assert env.args.len == 4;
    let inputLength = unix::readFile(env.args[1], &mut INPUT[..]) else panic "kernel RIL";
    assert env.args.len == 5;
    let mut decoder = alloc::new(&mut DECODE[..]);
    let package = try! program::decode(&INPUT[..inputLength], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
    assert package.dependencies.len == 0;
    let sourceLength = unix::readFile(env.args[2], &mut SOURCE[..]) else panic "kernel assembly";
    let library = load(env.args[1], &mut decoder);
    let kernel = load(env.args[2], &mut decoder);
    assert mem::eq(library.name, "std") and mem::eq(kernel.name, "kernel");
    assert library.dependencies.len == 0;
    for dependency in kernel.dependencies {
        assert mem::eq(dependency, "std");
    }
    let sourceLength = unix::readFile(env.args[3], &mut SOURCE[..]) else panic "kernel assembly";
    let mut assembly = alloc::new(&mut ASSEMBLY[..]);
    let empty: *mut [u8] = &mut [];
    let startup = try! asm::assemble(asm::scanner::SourceKind::String, &SOURCE[..sourceLength],
        &mut TEXT[..], &mut empty[..], &mut assembly, &mut STRINGS, 0);
    let mut arena = alloc::new(&mut CODE[..]);
    let mut scratch = alloc::new(&mut SCRATCH[..]);
    let mut generator = try! rv64::beginProgram(rv64::ProgramOptions {
        entryPatch: rv64::EntryPatch::None, debug: false,
        placement: image::Placement::Physical {
            code: 0x80010000, roData: 0x80200000, rwData: 0x80400000, entry: 0x80010000,
            code: CODE_ADDRESS, roData: 0, rwData: 0, entry: CODE_ADDRESS,
        },
    }, &mut arena);
    rv64::addAssembly(&mut generator, startup);
    for func in package.program.fns {
        rv64::generateFunction(&mut generator, func, &mut scratch);
        alloc::reset(&mut scratch);
    let mut dataCount: u32 = 0;
    for package in &[library, kernel] {
        for item in package.program.data {
            assert dataCount < GLOBALS.len;
            set GLOBALS[dataCount] = item; set dataCount += 1;
        }
        for func in package.program.fns {
            rv64::generateFunction(&mut generator, func, &mut scratch);
            if let error = generator.e.error {
                report(error); io::printLn(func.name); return 1;
            }
            alloc::reset(&mut scratch);
        }
    }
    for call in &generator.e.pendingCalls[..] {
        if dict::get(&generator.e.labels.funcs, call.target) == nil {
            io::print("undefined kernel function: "); io::printLn(call.target); return 1;
        }
    }
    let output = try! rv64::finishProgram(&mut generator, package.program.data,
    let roAddress = (CODE_ADDRESS + generator.e.codeLen as u64 * 4 + 4095) & ~4095;
    let mut symbols: u32 = 0;
    let roSize = try! data::layoutSection(&GLOBALS[..dataCount], &mut SYMBOLS[..], &mut symbols, roAddress, true);
    let rwAddress = (roAddress + roSize as u64 + 4095) & ~4095;
    set generator.placement = image::Placement::Physical {
        code: CODE_ADDRESS, roData: roAddress, rwData: rwAddress, entry: CODE_ADDRESS,
    };
    let output = try rv64::finishProgram(&mut generator, &GLOBALS[..dataCount],
        rv64::Storage { dataSyms: &mut SYMBOLS[..], dataSymEntries: &mut ENTRIES[..] },
        &[], &mut RO[..], &mut RW[..]);
        &[], &mut RO[..], &mut RW[..]) catch error {
            report(error);
            return 1;
        };
    let header = try! image::header(output.layout);
    let fd = unix::openOpts(env.args[3], unix::OpenFlags(*unix::O_WRONLY | *unix::O_CREAT | *unix::O_TRUNC), 420);
    let fd = unix::openOpts(env.args[4], unix::OpenFlags(*unix::O_WRONLY | *unix::O_CREAT | *unix::O_TRUNC), 420);
    assert fd >= 0;
    let written = unix::writeAll(fd, &header[..]) and unix::writeAll(fd, @sliceOf(output.code.ptr as *u8, output.code.len * 4))
        and unix::writeAll(fd, &RO[..output.roDataSize]) and unix::writeAll(fd, &RW[..output.rwDataSize]);
    let closed = unix::close(fd) == 0;
    assert written and closed;

test/boot/run +6 -4

# Boot the kernel through its production initialization on each supported hart count.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras > "$work/boot.ras"
"$emulator" -run bin/kernel.build.rv64 -- bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras > "$work/boot.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
for harts in 1 2 8; do
    if ! "$emulator" -machine -max-steps=10000000 -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then
    # The deterministic emulator shares its tick budget across all harts.
    steps=$((10000000 * harts))
    if ! "$emulator" -machine -max-steps="$steps" -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then
        cat "$work/log" >&2
        exit 1
    fi
    if ! grep -q '^kernel: platform ready$' "$work/log"; then
        cat "$work/log" >&2
        exit 1
    fi
    printf 'kernel boot: %s harts passed\n' "$harts"
done
status=0
"$emulator" -machine -harts=8 -max-steps=10000000 -run bin/kernel.rv64 > "$work/log" 2>&1 || status=$?
"$emulator" -machine -harts=8 -max-steps=80000000 -run bin/kernel.rv64 > "$work/log" 2>&1 || status=$?
if [ "$status" -ne 2 ] || [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
    cat "$work/log" >&2
    exit 1
fi
printf 'kernel startup: all eight harts reached machine idle without traps\n'

test/pages/machine.ras added +35 -0

1	+	//! Physical page zeroing with canaries immediately outside the allocation.
2	+	.text;
3	+	li %s0 0x40010000;
4	+	slli %s0 %s0 1;
5	+	li %t0 4096;
6	+	add %s1 %s0 %t0;
7	+	li %t0 91;
8	+	sd %t0 -8(%s0);
9	+	sd %t0 0(%s1);
10	+	mv %t1 %s0;
11	+	@fill
12	+	sd %t0 0(%t1);
13	+	addi %t1 %t1 8;
14	+	bne %t1 %s1 @fill;
15	+	mv %a0 %s0;
16	+	li %a1 1;
17	+	call @kernel::pages::zero;
18	+	mv %t1 %s0;
19	+	@check
20	+	ld %t0 0(%t1);
21	+	bnez %t0 @fail;
22	+	addi %t1 %t1 8;
23	+	bne %t1 %s1 @check;
24	+	li %t0 91;
25	+	ld %t1 -8(%s0);
26	+	bne %t0 %t1 @fail;
27	+	ld %t1 0(%s1);
28	+	bne %t0 %t1 @fail;
29	+	li %t0 0x10001000;
30	+	li %t1 0x5555;
31	+	sw %t1 0(%t0);
32	+	@fail
33	+	li %t0 0x10001000;
34	+	li %t1 0x13333;
35	+	sw %t1 0(%t0);

test/pages/run added +10 -0

1	+	#!/bin/sh
2	+	# Check complete physical-page zeroing and both adjacent canaries.
3	+	set -eu
4	+	emulator=${RAD_EMULATOR:-emulator}
5	+	work=$(mktemp -d)
6	+	trap 'rm -rf "$work"' EXIT HUP INT TERM
7	+	cat test/pages/machine.ras kernel/kernel/pages.ras > "$work/pages.ras"
8	+	"$emulator" -run bin/sync.build.rv64 -- "$work/pages.ras" "$work/pages.rv64"
9	+	"$emulator" -machine -max-steps=1000000 -run "$work/pages.rv64"
10	+	printf 'physical pages: complete zeroing and adjacent canaries passed\n'

7	7		export mod limits;
8	8		export mod slots;
9	9		export mod range;
10	10		export mod sync;
11	11		export mod platform;
12		-	@test export mod tests;
13	12		export mod trap;
14		-	export mod boot;
15	13		export mod capability;
	14	+	export mod frames;
	15	+	export mod backing;
	16	+	export mod pages;
	17	+	export mod boot;
	18	+	@test export mod tests;

4	4		use super::fdt;
5	5		use super::range;
6	6		use super::limits;
7	7		use super::sync;
8	8		use super::trap;
	9	+	use super::pages;
9	10
10	11		/// Platform data published by hart zero before secondary initialization.
11	12		export unsafe static PLATFORM: platform::Platform = undefined;
12	13		/// Release/acquire publication flag for PLATFORM.
13	14		static READY: u64 = 0;

64	65		let blob = @sliceOf(tree, size);
65	66		try! platform::decode(&blob[..], &mut PLATFORM);
66	67		let treeRange = range::new(treeAddress, size as u64) else panic "FDT range";
67	68		assert platform::inRam(&PLATFORM, treeRange);
68	69		try! platform::protect(&mut PLATFORM, treeRange);
	70	+	try! pages::initialize(&mut pages::STORE, &PLATFORM);
69	71		print("kernel: platform ready\n");
70	72		sync::storeRelease(&mut READY, 1);
71	73		} else {
72	74		while sync::loadAcquire(&READY) == 0 {
73	75		}

17	17		csrw mstatus %zero;
18	18		csrw mscratch %sp;
19	19		mv %a2 %sp;
20	20		addi %sp %sp -16;
21	21		sd %ra 0(%sp);
22		-	call @kernel::boot::enter;
	22	+	la %t0 @kernel::boot::enter;
	23	+	jalr %ra %t0 0;
23	24		ld %ra 0(%sp);
24	25		addi %sp %sp 16;
25	26		ret;
26	27
27	28		@kernel::boot::read8

37	37		if entry.kind == abi::Kind::Empty or entry.object.generation == 0 {
38	38		throw abi::Error::InvalidArg;
39	39		}
40	40		let rights = try abi::rights(*entry.rights as u64);
41	41		let reservation = try slots::reserve(&mut table.slots[..]);
	42	+	return publish(table, reservation, entry);
	43	+	}
	44	+
	45	+	/// Publish valid authority into a reservation from this table.
	46	+	/// The caller has validated metadata and acquired the object's lifetime reference.
	47	+	export fn publish(table: &mut Table, reservation: slots::Reservation, entry: Entry) -> abi::Handle {
42	48		let object = slots::reference(&reservation);
	49	+	assert slots::matches(&table.slots[..], object, slots::State::Reserved);
43	50		set table.entries[object.index] = entry;
44	51		let live = try! slots::commit(&mut table.slots[..], reservation);
45	52		return try! abi::handle(entry.kind, live);
46	53		}
47	54

5	5		export mod slots;
6	6		export mod fdt;
7	7		export mod platform;
8	8		export mod trap;
9	9		export mod capability;
	10	+	export mod frames;
	11	+	export mod backing;
	12	+	export mod pages;

1	1		//! Build a physical kernel image from trusted binary RIL and startup assembly.
2	2
3	3		use std::sys;
4	4		use std::io;
	5	+	use std::mem;
	6	+	use std::lang::il;
5	7		use std::sys::unix;
6	8		use std::lang::alloc;
7	9		use std::lang::strings;
8	10		use std::lang::il::binary;
9	11		use std::lang::il::binary::program;

11	13		use std::collections::dict;
12	14		use std::arch::rv64;
13	15		use std::arch::rv64::asm;
14	16		use std::arch::rv64::image;
15	17
	18	+	/// Code segment base for the native platform profile.
	19	+	constant CODE_ADDRESS: u64 = 0x81000000;
	20	+
16	21		/// Persistent native code-generation workspace.
17	22		static CODE: [u8; 16777216] = [0; 16777216];
18	23		/// Reusable function workspace.
19	24		static SCRATCH: [u8; 16777216] = [0; 16777216];
20	25		/// Decoded package storage.
21		-	static DECODE: [u8; 16777216] = [0; 16777216];
	26	+	static DECODE: [u8; 67108864] = [0; 67108864];
22	27		/// Binary package input.
23		-	static INPUT: [u8; 1048576] = [0; 1048576];
	28	+	static INPUT: [u8; 8388608] = [0; 8388608];
	29	+	/// Combined package data descriptors.
	30	+	unsafe static GLOBALS: [il::Data; 4096] = undefined;
24	31		/// Combined startup and boundary assembly.
25	32		static SOURCE: [u8; 65536] = [0; 65536];
26	33		/// Assembler workspace.
27	34		static ASSEMBLY: [u8; 4194304] = [0; 4194304];
28	35		/// Assembled startup words.
29	36		static TEXT: [u32; 16384] = [0; 16384];
30	37		/// Assembly identifiers.
31	38		unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };
32	39		/// Data symbol placement workspace.
33		-	unsafe static SYMBOLS: [data::DataSym; 1024] = undefined;
	40	+	unsafe static SYMBOLS: [data::DataSym; 4096] = undefined;
34	41		/// Data name lookup workspace.
35	42		unsafe static ENTRIES: [dict::Entry; data::DATA_SYM_TABLE_SIZE] = undefined;
36	43		/// Initialized read-only bytes.
37	44		static RO: [u8; 1048576] = [0; 1048576];
38	45		/// Initialized writable bytes.
39	46		static RW: [u8; 1048576] = [0; 1048576];
40	47
	48	+	/// Decode one package into persistent storage shared by the native link.
	49	+	unsafe fn load(path: &[u8], arena: &mut alloc::Arena) -> binary::Package {
	50	+	let inputLength = unix::readFile(path, &mut INPUT[..]) else panic "boot RIL";
	51	+	return try! program::decode(&INPUT[..inputLength], arena, binary::Limits { registers: 8192, blocks: 4096 });
	52	+	}
	53	+
	54	+	/// Print the bounded backend failure reported by a native build.
	55	+	fn report(error: rv64::Error) {
	56	+	match error {
	57	+	case rv64::Error::Allocation => io::printLn("native build: allocation exhausted"),
	58	+	case rv64::Error::Capacity => io::printLn("native build: output capacity exhausted"),
	59	+	case rv64::Error::Symbol => io::printLn("native build: unresolved symbol"),
	60	+	case rv64::Error::Relocation => io::printLn("native build: relocation out of range"),
	61	+	case rv64::Error::Image(_) => io::printLn("native build: invalid image layout"),
	62	+	case rv64::Error::Data(_) => io::printLn("native build: invalid data layout"),
	63	+	}
	64	+	}
	65	+
41	66		/// Assemble entry code and lower the kernel at explicit physical addresses.
42	67		@default unsafe fn main(env: *sys::Env) -> i32 {
43		-	assert env.args.len == 4;
44		-	let inputLength = unix::readFile(env.args[1], &mut INPUT[..]) else panic "kernel RIL";
	68	+	assert env.args.len == 5;
45	69		let mut decoder = alloc::new(&mut DECODE[..]);
46		-	let package = try! program::decode(&INPUT[..inputLength], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
47		-	assert package.dependencies.len == 0;
48		-	let sourceLength = unix::readFile(env.args[2], &mut SOURCE[..]) else panic "kernel assembly";
	70	+	let library = load(env.args[1], &mut decoder);
	71	+	let kernel = load(env.args[2], &mut decoder);
	72	+	assert mem::eq(library.name, "std") and mem::eq(kernel.name, "kernel");
	73	+	assert library.dependencies.len == 0;
	74	+	for dependency in kernel.dependencies {
	75	+	assert mem::eq(dependency, "std");
	76	+	}
	77	+	let sourceLength = unix::readFile(env.args[3], &mut SOURCE[..]) else panic "kernel assembly";
49	78		let mut assembly = alloc::new(&mut ASSEMBLY[..]);
50	79		let empty: *mut [u8] = &mut [];
51	80		let startup = try! asm::assemble(asm::scanner::SourceKind::String, &SOURCE[..sourceLength],
52	81		&mut TEXT[..], &mut empty[..], &mut assembly, &mut STRINGS, 0);
53	82		let mut arena = alloc::new(&mut CODE[..]);
54	83		let mut scratch = alloc::new(&mut SCRATCH[..]);
55	84		let mut generator = try! rv64::beginProgram(rv64::ProgramOptions {
56	85		entryPatch: rv64::EntryPatch::None, debug: false,
57	86		placement: image::Placement::Physical {
58		-	code: 0x80010000, roData: 0x80200000, rwData: 0x80400000, entry: 0x80010000,
	87	+	code: CODE_ADDRESS, roData: 0, rwData: 0, entry: CODE_ADDRESS,
59	88		},
60	89		}, &mut arena);
61	90		rv64::addAssembly(&mut generator, startup);
62		-	for func in package.program.fns {
63		-	rv64::generateFunction(&mut generator, func, &mut scratch);
64		-	alloc::reset(&mut scratch);
	91	+	let mut dataCount: u32 = 0;
	92	+	for package in &[library, kernel] {
	93	+	for item in package.program.data {
	94	+	assert dataCount < GLOBALS.len;
	95	+	set GLOBALS[dataCount] = item; set dataCount += 1;
	96	+	}
	97	+	for func in package.program.fns {
	98	+	rv64::generateFunction(&mut generator, func, &mut scratch);
	99	+	if let error = generator.e.error {
	100	+	report(error); io::printLn(func.name); return 1;
	101	+	}
	102	+	alloc::reset(&mut scratch);
	103	+	}
65	104		}
66	105		for call in &generator.e.pendingCalls[..] {
67	106		if dict::get(&generator.e.labels.funcs, call.target) == nil {
68	107		io::print("undefined kernel function: "); io::printLn(call.target); return 1;
69	108		}
70	109		}
71		-	let output = try! rv64::finishProgram(&mut generator, package.program.data,
	110	+	let roAddress = (CODE_ADDRESS + generator.e.codeLen as u64 * 4 + 4095) & ~4095;
	111	+	let mut symbols: u32 = 0;
	112	+	let roSize = try! data::layoutSection(&GLOBALS[..dataCount], &mut SYMBOLS[..], &mut symbols, roAddress, true);
	113	+	let rwAddress = (roAddress + roSize as u64 + 4095) & ~4095;
	114	+	set generator.placement = image::Placement::Physical {
	115	+	code: CODE_ADDRESS, roData: roAddress, rwData: rwAddress, entry: CODE_ADDRESS,
	116	+	};
	117	+	let output = try rv64::finishProgram(&mut generator, &GLOBALS[..dataCount],
72	118		rv64::Storage { dataSyms: &mut SYMBOLS[..], dataSymEntries: &mut ENTRIES[..] },
73		-	&[], &mut RO[..], &mut RW[..]);
	119	+	&[], &mut RO[..], &mut RW[..]) catch error {
	120	+	report(error);
	121	+	return 1;
	122	+	};
74	123		let header = try! image::header(output.layout);
75		-	let fd = unix::openOpts(env.args[3], unix::OpenFlags(unix::O_WRONLY \| unix::O_CREAT \| *unix::O_TRUNC), 420);
	124	+	let fd = unix::openOpts(env.args[4], unix::OpenFlags(unix::O_WRONLY \| unix::O_CREAT \| *unix::O_TRUNC), 420);
76	125		assert fd >= 0;
77	126		let written = unix::writeAll(fd, &header[..]) and unix::writeAll(fd, @sliceOf(output.code.ptr as u8, output.code.len 4))
78	127		and unix::writeAll(fd, &RO[..output.roDataSize]) and unix::writeAll(fd, &RW[..output.rwDataSize]);
79	128		let closed = unix::close(fd) == 0;
80	129		assert written and closed;

2	2		# Boot the kernel through its production initialization on each supported hart count.
3	3		set -eu
4	4		emulator=${RAD_EMULATOR:-emulator}
5	5		work=$(mktemp -d)
6	6		trap 'rm -rf "$work"' EXIT HUP INT TERM
7		-	cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras > "$work/boot.ras"
8		-	"$emulator" -run bin/kernel.build.rv64 -- bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
	7	+	cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras > "$work/boot.ras"
	8	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
9	9		for harts in 1 2 8; do
10		-	if ! "$emulator" -machine -max-steps=10000000 -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then
	10	+	# The deterministic emulator shares its tick budget across all harts.
	11	+	steps=$((10000000 * harts))
	12	+	if ! "$emulator" -machine -max-steps="$steps" -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then
11	13		cat "$work/log" >&2
12	14		exit 1
13	15		fi
14	16		if ! grep -q '^kernel: platform ready$' "$work/log"; then
15	17		cat "$work/log" >&2
16	18		exit 1
17	19		fi
18	20		printf 'kernel boot: %s harts passed\n' "$harts"
19	21		done
20	22		status=0
21		-	"$emulator" -machine -harts=8 -max-steps=10000000 -run bin/kernel.rv64 > "$work/log" 2>&1 \|\| status=$?
	23	+	"$emulator" -machine -harts=8 -max-steps=80000000 -run bin/kernel.rv64 > "$work/log" 2>&1 \|\| status=$?
22	24		if [ "$status" -ne 2 ] \|\| [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
23	25		cat "$work/log" >&2
24	26		exit 1
25	27		fi
26	28		printf 'kernel startup: all eight harts reached machine idle without traps\n'