kernel: Initialize page heaps outside the lock

bf2423a6d156768266ae6c5971ec3a277b609e2d43db07bb8605d3f56f295a3e
Reserve an exclusive Page capability and retain it through a context
lease. Clear the page and initialize its heap header outside the
metadata lock, then finish conversion under the lock.

Keep the backing live during domain teardown and reject conversion
while an active image load retains an overlapping source token.
Cover private reservations and dead continuation cleanup.

Assisted-by: Codex:gpt-6.1-sol
Alexis Sellier committed ago 1 parent 050eda0c
kernel/kernel/calls.rad +72 -6
631 631
        }
632 632
        case abi::Operation::PageSplit as u64 =>
633 633
            return *try pages::split(
634 634
                &mut shared.pages, table, handle, arguments[1]
635 635
            ),
636 -
        case abi::Operation::PageHeap as u64 => {
637 -
            if arguments[1] <> 0 or arguments[2] <> 0 or arguments[3] <> 0 {
638 -
                throw abi::Error::InvalidArg;
639 -
            }
640 -
            return try pages::consumeHeap(&mut shared.pages, table, handle);
641 -
        }
642 636
        case abi::Operation::DomainYield as u64 => {
643 637
            if arguments[1] <> 0 or arguments[2] <> 0 or arguments[3] <> 0 {
644 638
                throw abi::Error::InvalidArg;
645 639
            }
646 640
            let current = running.current else panic "execute: no current context";
1053 1047
        throw abi::Error::BadHandle;
1054 1048
    }
1055 1049
    return pages::publish(&mut shared.pages, table, ready);
1056 1050
}
1057 1051
1052 +
/// Reserve an exclusive Page for heap initialization.
1053 +
export fn reserveHeap(
1054 +
    shared: &mut state::Shared,
1055 +
    domain: &domains::Domain,
1056 +
    table: &mut capability::Table,
1057 +
    arguments: &[u64],
1058 +
    current: ?abi::CurrentContextInfo
1059 +
) -> lifecycle::PageHeld throws (abi::Error) {
1060 +
    let running = current else panic "reserveHeap: no current context";
1061 +
    try admit(&*shared, domain, &*table, abi::Operation::PageHeap as u64, abi::Handle(
1062 +
        arguments[0]
1063 +
    ));
1064 +
    if arguments[1] <> 0 or arguments[2] <> 0 or arguments[3] <> 0 {
1065 +
        throw abi::Error::InvalidArg;
1066 +
    }
1067 +
    let extent = try pages::exclusiveExtent(&shared.pages, &*table, abi::Handle(arguments[0]));
1068 +
    if lifecycle::sourceBusy(&shared.lifecycle, extent.start, extent.end) {
1069 +
        throw abi::Error::Busy;
1070 +
    }
1071 +
    let pending = try pages::reserveHeap(&shared.pages, table, abi::Handle(arguments[0]));
1072 +
    return lifecycle::holdHeap(
1073 +
        &mut shared.lifecycle, &mut shared.domains, abi::reference(running.context), pending
1074 +
    );
1075 +
}
1076 +
1077 +
/// Finish heap conversion and release the caller's reservation.
1078 +
export fn publishHeap(
1079 +
    shared: &mut state::Shared,
1080 +
    table: &mut capability::Table,
1081 +
    lease: lifecycle::Lease,
1082 +
    access: physical::Access
1083 +
) -> u64 throws (abi::Error) {
1084 +
    let ready = lifecycle::takeHeap(&mut shared.lifecycle, &mut shared.domains, lease, access);
1085 +
    let alive = live(&shared.domains, table.owner);
1086 +
    let address = pages::finishHeap(&mut shared.pages, table, ready);
1087 +
    if not alive {
1088 +
        throw abi::Error::BadHandle;
1089 +
    }
1090 +
    return address;
1091 +
}
1092 +
1058 1093
/// Read a context start record and reserve the context under the calling context.
1059 1094
export fn reserveContext(
1060 1095
    shared: &mut state::Shared,
1061 1096
    domain: &domains::Domain,
1062 1097
    memory: &mut domains::Memory,
1384 1419
            return *result;
1385 1420
        }
1386 1421
    }
1387 1422
}
1388 1423
1424 +
/// Convert a Page to a heap with private initialization outside the lock.
1425 +
fn createHeap(owner: abi::Ref, arguments: &[u64]) -> u64 throws (abi::Error) {
1426 +
    let guard = sync::enter();
1427 +
    let domain = try locked::lookup(&guard, owner) catch error {
1428 +
        sync::leave(guard);
1429 +
        throw error;
1430 +
    };
1431 +
    let begun = try locked::beginHeap(&guard, &domain, arguments) catch error {
1432 +
        sync::leave(guard);
1433 +
        throw error;
1434 +
    };
1435 +
    sync::leave(guard);
1436 +
    match begun {
1437 +
        case lifecycle::PageHeld::Held(lease, access) => {
1438 +
            pages::initializeHeap(&access);
1439 +
            let publicationGuard = sync::enter();
1440 +
            let result = try locked::publishHeap(
1441 +
                &publicationGuard, &domain, lease, access
1442 +
            ) catch error {
1443 +
                sync::leave(publicationGuard);
1444 +
                throw error;
1445 +
            };
1446 +
            sync::leave(publicationGuard);
1447 +
            return result;
1448 +
        }
1449 +
    }
1450 +
}
1451 +
1389 1452
/// Create a context: reserve under the lock, clear its stack unlocked, then publish.
1390 1453
fn createContext(owner: abi::Ref, arguments: &[u64]) -> u64 throws (abi::Error) {
1391 1454
    let guard = sync::enter();
1392 1455
    let domain = try locked::lookup(&guard, owner) catch error {
1393 1456
        sync::leave(guard);
1468 1531
            return access::Reply { value: try createDomain(owner, arguments), extra: 0 };
1469 1532
        }
1470 1533
        case abi::Operation::PageAllocate as u64 => {
1471 1534
            return access::Reply { value: try allocatePage(owner, arguments), extra: 0 };
1472 1535
        }
1536 +
        case abi::Operation::PageHeap as u64 => {
1537 +
            return access::Reply { value: try createHeap(owner, arguments), extra: 0 };
1538 +
        }
1473 1539
        case abi::Operation::ContextCreate as u64 => {
1474 1540
            return access::Reply { value: try createContext(owner, arguments), extra: 0 };
1475 1541
        }
1476 1542
        else => return try submitCall(owner, operation, arguments),
1477 1543
    }
kernel/kernel/lifecycle.rad +55 -0
63 63
/// stopped continuation releases frames from these records alone; its stranded
64 64
/// tokens are never used, because the context never resumes.
65 65
export union Allocation: Copy {
66 66
    /// Page frames and unpublished capability metadata.
67 67
    Page(pages::Allocation),
68 +
    /// Page retained for private heap initialization.
69 +
    Heap(pages::HeapAllocation),
68 70
    /// Additional context slot and private kernel stack.
69 71
    Context(domains::ContextAllocation),
70 72
    /// Reserved domain generation with its retained creation payload.
71 73
    Domain(abi::Ref),
72 74
    /// Private compiler workspace and unpublished output.
176 178
            );
177 179
        }
178 180
    }
179 181
}
180 182
183 +
/// Check whether an active load retains a token for an extent.
184 +
export fn sourceBusy(calls: &Calls, start: u64, end: u64) -> bool {
185 +
    for word in 0 .. CONTEXT_WORDS {
186 +
        if calls.active[word] == 0 {
187 +
            continue;
188 +
        }
189 +
        for offset in 0 .. 32 as u32 {
190 +
            if (calls.active[word] & (1 << offset)) == 0 {
191 +
                continue;
192 +
            }
193 +
            match calls.records[word * 32 + offset].allocation {
194 +
                case Allocation::Load(loading) => {
195 +
                    if loader::sourceOverlaps(&loading.input, start, end) {
196 +
                        return true;
197 +
                    }
198 +
                }
199 +
                else => {}
200 +
            }
201 +
        }
202 +
    }
203 +
    return false;
204 +
}
205 +
206 +
/// Retain a heap conversion and its caller while the memory is initialized.
207 +
export fn holdHeap(
208 +
    calls: &mut Calls,
209 +
    store: &mut domains::Store,
210 +
    context: abi::Ref,
211 +
    reservation: pages::HeapReservation
212 +
) -> PageHeld {
213 +
    match reservation {
214 +
        case pages::HeapReservation::Held(allocation, access) =>
215 +
            return PageHeld::Held(
216 +
                hold(calls, store, context, Allocation::Heap(allocation)), access
217 +
            ),
218 +
    }
219 +
}
220 +
181 221
/// Transfer an unpublished context into its caller's retained storage.
182 222
/// The token goes to the continuation, and only the context record is retained.
183 223
export fn holdContext(
184 224
    calls: &mut Calls,
185 225
    store: &mut domains::Store,
510 550
        }
511 551
        else => panic "takePage: page reservation required",
512 552
    }
513 553
}
514 554
555 +
/// Return retained heap memory under metadata serialization.
556 +
export fn takeHeap(
557 +
    calls: &mut Calls, store: &mut domains::Store, lease: Lease, access: physical::Access
558 +
) -> pages::HeapReservation {
559 +
    match take(calls, store, lease) {
560 +
        case Allocation::Heap(allocation) => {
561 +
            assert physical::covers(&access, allocation.base, allocation.size);
562 +
            return pages::HeapReservation::Held(allocation, access);
563 +
        }
564 +
        else => panic "takeHeap: heap reservation required",
565 +
    }
566 +
}
567 +
515 568
/// Return a context reservation for publication or cancellation under the lock.
516 569
/// The continuation returns its token, which must cover the retained stack.
517 570
export fn takeContext(
518 571
    calls: &mut Calls, store: &mut domains::Store, lease: Lease, access: physical::Access
519 572
) -> domains::ContextReservation {
586 639
                        ),
587 640
                    case Allocation::Page(allocation) =>
588 641
                        pages::cancelUsing(
589 642
                            memory, &mut owner.memory.table, allocation, frames::retire
590 643
                        ),
644 +
                    case Allocation::Heap(allocation) =>
645 +
                        pages::releaseHeap(memory, &mut owner.memory.table, allocation),
591 646
                    case Allocation::Context(allocation) =>
592 647
                        domains::contextCancelUsing(
593 648
                            store, memory, allocation, frames::retire
594 649
                        ),
595 650
                    case Allocation::Domain(object) =>
kernel/kernel/loader.rad +5 -0
635 635
            }
636 636
        },
637 637
    }
638 638
}
639 639
640 +
/// Test whether a load's source overlaps a physical extent.
641 +
export fn sourceOverlaps(input: &Input, start: u64, end: u64) -> bool {
642 +
    return input.address < end and start < input.address + input.length as u64;
643 +
}
644 +
640 645
/// Domain that owns the load, which is kept live while the input is held.
641 646
export fn owner(input: &Input) -> abi::Ref {
642 647
    return input.owner;
643 648
}
644 649
kernel/kernel/locked.rad +23 -0
246 246
    unsafe {
247 247
        return try calls::publishPage(&mut state::SHARED, &mut domain.memory.table, lease, access);
248 248
    }
249 249
}
250 250
251 +
/// Reserve a Page for heap initialization.
252 +
export fn beginHeap(
253 +
    guard: &sync::Guard, domain: &domains::Domain, arguments: &[u64]
254 +
) -> lifecycle::PageHeld throws (abi::Error) {
255 +
    // The guard serializes the shared metadata, and the record came from `get`.
256 +
    unsafe {
257 +
        let shared = &mut state::SHARED;
258 +
        let table = &mut domain.memory.table;
259 +
        let current = dispatch::running(guard, &shared.budgets);
260 +
        return try calls::reserveHeap(shared, domain, table, arguments, current);
261 +
    }
262 +
}
263 +
264 +
/// Publish an initialized heap.
265 +
export fn publishHeap(
266 +
    guard: &sync::Guard, domain: &domains::Domain, lease: lifecycle::Lease, access: physical::Access
267 +
) -> u64 throws (abi::Error) {
268 +
    // The guard serializes the shared metadata, and the record came from `get`.
269 +
    unsafe {
270 +
        return try calls::publishHeap(&mut state::SHARED, &mut domain.memory.table, lease, access);
271 +
    }
272 +
}
273 +
251 274
/// Reserve a context. The result carries the token to clear its stack with.
252 275
export fn beginContext(
253 276
    guard: &sync::Guard, domain: &domains::Domain, arguments: &[u64]
254 277
) -> lifecycle::ContextHeld throws (abi::Error) {
255 278
    // The guard serializes the shared metadata, and the record came from `get`.
kernel/kernel/pages.rad +75 -20
132 132
133 133
/// Bytes at the start of a heap Page that hold its header, padded so that the pool is aligned
134 134
/// for any element.
135 135
export constant HEAP_POOL_OFFSET: u64 = heap::HEADER_SIZE as u64;
136 136
137 -
/// Take the memory of a Page for a heap, and consume its handle.
138 -
///
139 -
/// The header at the start of the Page holds zero used bytes, the size of the pool, and the
140 -
/// address of the pool, which follows the header; this is the layout of a static heap. The Page
141 -
/// is cleared first, so the pool is zero whatever the kernel wrote into it before. On success
142 -
/// nothing can name the memory again, so only verified growth reaches it, and the frames stay
143 -
/// with the allocation until the domains that hold exposure end. Returns the address of the
144 -
/// heap, which is the address of its header. See [`exclusiveExtent`] for the conditions.
145 -
export fn consumeHeap(
146 -
    store: &mut Store, table: &mut capability::Table, handle: abi::Handle
147 -
) -> u64 throws (abi::Error) {
148 -
    let extent = try exclusiveExtent(&*store, &*table, handle);
137 +
/// A Page capability retained while its heap memory is initialized.
138 +
export record HeapAllocation: Copy {
139 +
    /// Reserved capability slot.
140 +
    handle: abi::Ref,
141 +
    /// Page object retained by the capability.
142 +
    page: abi::Ref,
143 +
    /// Start of the Page extent.
144 +
    base: u64,
145 +
    /// Size of the Page extent.
146 +
    size: u64,
147 +
}
148 +
149 +
/// Exclusive Page memory and the capability that retains it.
150 +
export union HeapReservation: Once {
151 +
    /// Retained metadata and the token for initialization.
152 +
    Held(HeapAllocation, physical::Access),
153 +
}
154 +
155 +
/// Reserve an exclusive Page for heap initialization under metadata serialization.
156 +
/// The reserved capability cannot be used, and retains the Page and its backing
157 +
/// until conversion finishes. See [`exclusiveExtent`] for the conditions.
158 +
export fn reserveHeap(
159 +
    store: &Store, table: &mut capability::Table, handle: abi::Handle
160 +
) -> HeapReservation throws (abi::Error) {
161 +
    let extent = try exclusiveExtent(store, &*table, handle);
162 +
    let entry = try capability::get(&*table, handle);
163 +
    let slot = (try abi::decode(handle)).object;
149 164
    let size = extent.end - extent.start;
150 165
    let access = try physical::access(&store.backings.pool, extent.start, size);
151 -
    // The pool must hold zeros, which a heap hands out. A Page that was never reached through
152 -
    // `page.slice` is zero already, unless the kernel itself wrote into it for a call, so the
153 -
    // bytes are cleared here instead of trusting that.
154 -
    try physical::zeroAt(&access, 0, size);
155 -
    try physical::writeWordsAt(&access, 0, &[
156 -
        0, size - HEAP_POOL_OFFSET, extent.start + HEAP_POOL_OFFSET
166 +
    set table.slots[slot.index].state = slots::State::Reserved;
167 +
    return HeapReservation::Held(
168 +
        HeapAllocation {
169 +
            handle: slot, page: entry.object, base: extent.start, size
170 +
        },
171 +
        access
172 +
    );
173 +
}
174 +
175 +
/// Initialize exclusive heap memory outside metadata serialization.
176 +
/// The header holds zero used bytes, the pool size, and the pool address. The
177 +
/// pool follows the header and is cleared before verified growth can reach it.
178 +
export fn initializeHeap(access: &physical::Access) {
179 +
    let size = physical::size(access);
180 +
    let base = physical::start(access);
181 +
    physical::clear(access);
182 +
    try! physical::writeWordsAt(access, 0, &[
183 +
        0, size - HEAP_POOL_OFFSET, base + HEAP_POOL_OFFSET
157 184
    ]);
158 -
    try drop(store, table, handle);
159 -
    return extent.start;
185 +
}
186 +
187 +
/// Initialize the memory of a heap reservation.
188 +
export fn clearHeap(reservation: &HeapReservation) {
189 +
    match reservation {
190 +
        case HeapReservation::Held(allocation, access) => initializeHeap(access),
191 +
    }
192 +
}
193 +
194 +
/// Release the retained capability and page under metadata serialization.
195 +
export fn releaseHeap(
196 +
    store: &mut Store, table: &mut capability::Table, allocation: HeapAllocation
197 +
) {
198 +
    assert slots::matches(&table.slots[..], allocation.handle, slots::State::Reserved);
199 +
    assert table.entries[allocation.handle.index].object == allocation.page;
200 +
    try! release(store, allocation.page);
201 +
    try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(allocation.handle));
202 +
}
203 +
204 +
/// Publish an initialized heap and consume its capability.
205 +
export fn finishHeap(
206 +
    store: &mut Store, table: &mut capability::Table, reservation: HeapReservation
207 +
) -> u64 {
208 +
    match reservation {
209 +
        case HeapReservation::Held(allocation, access) => {
210 +
            assert physical::covers(&access, allocation.base, allocation.size);
211 +
            releaseHeap(store, table, allocation);
212 +
            return allocation.base;
213 +
        }
214 +
    }
160 215
}
161 216
162 217
/// Private frames and unpublished slots held while memory is cleared.
163 218
export record Allocation: Copy {
164 219
    /// Calling domain whose table contains the reserved handle.
kernel/kernel/tests/calls.rad +53 -0
761 761
        assert frames::reclaim(&mut shared.pages.backings.pool, 256) > 0;
762 762
    }
763 763
    assert free() == before and MEMORY.table.slots[1].state == slots::State::Free;
764 764
}
765 765
766 +
/// Heap reservations retain their backing until a dead caller's work ends.
767 +
@test unsafe fn heapReservation() throws (testing::TestError) {
768 +
    for stopped in [false, true] {
769 +
        let owner = initialize();
770 +
        let handle = abi::Handle(try! allocate(owner, 1));
771 +
        let shared = &mut state::SHARED;
772 +
        let domain = try! domains::get(&shared.domains, owner);
773 +
        let entry = try! capability::get(&MEMORY.table, handle);
774 +
        let page = try! pages::get(&shared.pages, entry.object);
775 +
        let held = try! calls::reserveHeap(
776 +
            shared, &domain, &mut MEMORY.table, &[*handle, 0, 0, 0], running()
777 +
        );
778 +
        assert not pages::accessible(&shared.pages, &MEMORY.table, page.base, 8, abi::Rights(
779 +
            abi::READ
780 +
        ));
781 +
        set shared.domains.records[owner.index].state = domains::Lifecycle::Dead;
782 +
        assert not try! domains::quiescent(&shared.domains, owner);
783 +
        match held {
784 +
            case lifecycle::PageHeld::Held(lease, access) => {
785 +
                if stopped {
786 +
                    set shared.domains.contexts[CALLER.index].state =
787 +
                        domains::ContextState::Stopped;
788 +
                    set shared.domains.contexts[CALLER.index].hart = nil;
789 +
                    assert lifecycle::leaseContext(lease) == CALLER;
790 +
                    assert lifecycle::cancelAllocations(
791 +
                        &mut shared.lifecycle,
792 +
                        &mut shared.domains,
793 +
                        &mut shared.pages,
794 +
                        &mut shared.loader,
795 +
                        &mut shared.registry
796 +
                    );
797 +
                } else {
798 +
                    pages::initializeHeap(&access);
799 +
                    let mut failed = false;
800 +
                    try calls::publishHeap(shared, &mut MEMORY.table, lease, access) catch error {
801 +
                        assert error == abi::Error::BadHandle;
802 +
                        set failed = true;
803 +
                    };
804 +
                    assert failed;
805 +
                }
806 +
            }
807 +
        }
808 +
        assert not shared.domains.contexts[CALLER.index].reservation;
809 +
        assert not frames::available(&shared.pages.backings.pool, 0);
810 +
        let mut invalid = false;
811 +
        try capability::get(&MEMORY.table, handle) catch error {
812 +
            assert error == abi::Error::BadHandle;
813 +
            set invalid = true;
814 +
        };
815 +
        assert invalid;
816 +
    }
817 +
}
818 +
766 819
/// Context calls validate descriptor storage and preserve capacity on failure.
767 820
@test unsafe fn contextBuffers() throws (testing::TestError) {
768 821
    let owner = initialize();
769 822
    let page = try! allocate(owner, 1);
770 823
    let base = &RAM[OFFSET] as u64;
kernel/kernel/tests/loader.rad +4 -0
159 159
        set DOMAINS.contexts[0].state = domains::ContextState::Ready;
160 160
        set DOMAINS.contexts[0].hart = nil;
161 161
        let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut MEMORY.table, request);
162 162
        match lifecycle::holdLoad(&mut CALLS, &mut DOMAINS, owner, pending) {
163 163
            case lifecycle::LoadHeld::Held(lease, held) => {
164 +
                let entry = try! capability::get(&MEMORY.table, SOURCE);
165 +
                let page = try! pages::get(&PAGES, entry.object);
166 +
                assert lifecycle::sourceBusy(&CALLS, page.base, page.base + 4096);
167 +
                assert not lifecycle::sourceBusy(&CALLS, page.base + 4096, page.base + 8192);
164 168
                let mut tokens = held;
165 169
                if stage > 0 {
166 170
                    let output = try! lifecycle::reserveOutput(
167 171
                        &mut CALLS, &lease, &mut PAGES, &mut PACKAGES
168 172
                    );
kernel/kernel/tests/pages.rad +31 -8
455 455
    try! capability::get(&TABLE, shared);
456 456
    try! capability::get(&TABLE, pair);
457 457
    try! capability::get(&TABLE, right);
458 458
}
459 459
460 +
/// Convert one test Page to a heap.
461 +
unsafe fn consumeHeap(handle: abi::Handle) -> u64 throws (abi::Error) {
462 +
    let pending = try pages::reserveHeap(&STORE, &mut TABLE, handle);
463 +
    pages::clearHeap(&pending);
464 +
    return pages::finishHeap(&mut STORE, &mut TABLE, pending);
465 +
}
466 +
460 467
/// A heap consumes its handle, receives a header and a pool, and keeps its frames until the
461 468
/// domain that is exposed to them ends.
462 469
@test unsafe fn heapConsumption() throws (testing::TestError) {
463 470
    initialize();
464 471
    let heap = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 2);
465 472
    let entry = try! capability::get(&TABLE, heap);
466 473
    let page = try! pages::get(&STORE, entry.object);
467 -
    let address = try! pages::consumeHeap(&mut STORE, &mut TABLE, heap);
474 +
    let address = try! consumeHeap(heap);
468 475
    assert address == page.base;
469 476
    try capability::get(&TABLE, heap) catch error {
470 477
        assert error == abi::Error::BadHandle;
471 478
    };
472 479
    // The header holds no used bytes, the size of the pool, and the address of the pool.
483 490
    assert frames::available(&STORE.backings.pool, 0) and frames::available(
484 491
        &STORE.backings.pool, 1
485 492
    );
486 493
}
487 494
495 +
/// A reserved heap cannot be reached until conversion finishes.
496 +
@test unsafe fn heapReservation() throws (testing::TestError) {
497 +
    initialize();
498 +
    let handle = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 2);
499 +
    let entry = try! capability::get(&TABLE, handle);
500 +
    let page = try! pages::get(&STORE, entry.object);
501 +
    let pending = try! pages::reserveHeap(&STORE, &mut TABLE, handle);
502 +
    assert not pages::accessible(&STORE, &TABLE, page.base, 8, abi::Rights(abi::READ));
503 +
    try capability::get(&TABLE, handle) catch error {
504 +
        assert error == abi::Error::BadHandle;
505 +
    };
506 +
    pages::clearHeap(&pending);
507 +
    assert pages::finishHeap(&mut STORE, &mut TABLE, pending) == page.base;
508 +
    assert not frames::available(&STORE.backings.pool, 0);
509 +
}
510 +
488 511
/// A Page that the kernel wrote into, as it does for the buffer of a call, becomes a zeroed heap.
489 512
@test unsafe fn heapClearsPool() throws (testing::TestError) {
490 513
    initialize();
491 514
    let heap = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 2);
492 515
    let entry = try! capability::get(&TABLE, heap);
493 516
    let page = try! pages::get(&STORE, entry.object);
494 517
    let access = try! physical::access(&STORE.backings.pool, page.base, 8192);
495 518
    try! physical::writeAt(&access, 100, &[1, 2, 3, 4]);
496 519
    try! physical::writeAt(&access, 5000, &[9]);
497 -
    try! pages::consumeHeap(&mut STORE, &mut TABLE, heap);
520 +
    try! consumeHeap(heap);
498 521
    let mut offset = pages::HEAP_POOL_OFFSET;
499 522
    while offset < 8192 {
500 523
        assert try! physical::read64At(&access, offset) == 0;
501 524
        set offset += 8;
502 525
    }
509 532
    let shared = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 1);
510 533
    let second = try! pages::selfGrant(
511 534
        &mut STORE, &mut TABLE, shared, (abi::READ | abi::WRITE) as u64
512 535
    );
513 536
    let mut refused: u32 = 0;
514 -
    try pages::consumeHeap(&mut STORE, &mut TABLE, shared) catch error {
537 +
    try consumeHeap(shared) catch error {
515 538
        assert error == abi::Error::Busy;
516 539
        set refused += 1;
517 540
    };
518 541
    drop(second);
519 542
    let accessed = try! capability::get(&TABLE, shared);
520 543
    let page = try! pages::get(&STORE, accessed.object);
521 544
    try! backing::markAccessed(&mut STORE.backings, page.backing);
522 -
    try pages::consumeHeap(&mut STORE, &mut TABLE, shared) catch error {
545 +
    try consumeHeap(shared) catch error {
523 546
        assert error == abi::Error::Denied;
524 547
        set refused += 1;
525 548
    };
526 549
    let pair = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 2);
527 550
    let right = try! pages::split(&mut STORE, &mut TABLE, pair, 1);
528 -
    try pages::consumeHeap(&mut STORE, &mut TABLE, pair) catch error {
551 +
    try consumeHeap(pair) catch error {
529 552
        assert error == abi::Error::Busy;
530 553
        set refused += 1;
531 554
    };
532 -
    try pages::consumeHeap(&mut STORE, &mut TABLE, right) catch error {
555 +
    try consumeHeap(right) catch error {
533 556
        assert error == abi::Error::Busy;
534 557
        set refused += 1;
535 558
    };
536 559
    let readonly = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 1);
537 560
    let readonlySlot = try! abi::decode(readonly);
538 561
    set TABLE.entries[readonlySlot.object.index].rights = abi::Rights(abi::READ);
539 -
    try pages::consumeHeap(&mut STORE, &mut TABLE, readonly) catch error {
562 +
    try consumeHeap(readonly) catch error {
540 563
        assert error == abi::Error::Denied;
541 564
        set refused += 1;
542 565
    };
543 566
    assert refused == 5;
544 567
    // Nothing was consumed.
553 576
@test unsafe fn heapBackingClosed() throws (testing::TestError) {
554 577
    initialize();
555 578
    let heap = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 1);
556 579
    let entry = try! capability::get(&TABLE, heap);
557 580
    let page = try! pages::get(&STORE, entry.object);
558 -
    try! pages::consumeHeap(&mut STORE, &mut TABLE, heap);
581 +
    try! consumeHeap(heap);
559 582
    let other = abi::Ref { index: 1, generation: 1 };
560 583
    try! backing::registerDomain(&mut STORE.backings, other);
561 584
    try backing::expose(&mut STORE.backings, page.backing, other) catch error {
562 585
        assert error == abi::Error::BadHandle;
563 586
        return;