//! One domain executing separate user contexts on every online hart.

use std::mem;

use std::arch::rv64::shared;

use kernel::abi;

use kernel::invariants;

use kernel::frames;

use kernel::slots;

use kernel::backing;

use kernel::pages;

use kernel::capability;

use kernel::registry;

use kernel::loader;

use kernel::domains;

use kernel::budgets;

use kernel::dispatch;

use kernel::boot;

use kernel::sync;

use kernel::remote;

use kernel::calls;

use kernel::events;

use kernel::timers;

use kernel::interrupts;

use kernel::plic;

use kernel::lifecycle;

use kernel::dispatchinput;

/// Bootstrap resource authority.

unsafe static TABLE: capability::Table = undefined;

/// Release/acquire barrier after fixture publication.

static READY: u64 = 0;

/// Shared user domain generation.

unsafe static USER: abi::Ref = undefined;

/// Pending domain in the user's creation subtree.

unsafe static DESCENDANT: abi::Ref = undefined;

/// Per-hart user execution contexts.

unsafe static CONTEXTS: [abi::Ref; 8] = undefined;

/// Mapped progress counters in the user's private state.

static COUNTERS: u64 = 0;

/// Per-context migration observations in the user's private state.

static MIGRATIONS: u64 = 0;

/// Shared handshake for the independently activated context on hart one.

static WAKE_STATE: u64 = 0;

/// Bootstrap management capability for the pending idle-wakeup domain.

unsafe static IDLE_HANDLE: abi::Handle = undefined;

/// Context whose Wait call must release the remote hart.

unsafe static IDLE_CONTEXT: abi::Ref = undefined;

/// Validated initial stack pointer for the idle-wakeup domain.

static IDLE_STACK: u64 = 0;

/// Generation of the idle-wakeup domain.

unsafe static IDLE_DOMAIN: abi::Ref = undefined;

/// Completion checker's live domain generation.

unsafe static CHECKER: abi::Ref = undefined;

/// Unbound window that expires before the checker starts.

unsafe static EXPIRED: abi::Handle = undefined;

/// Checker's authority to activate and notify the idle-wakeup domain.

unsafe static RECEIVER: abi::Handle = undefined;

/// Unbound authority reserved for the idle-wakeup check on hart one.

unsafe static IDLE_WINDOW: abi::Handle = undefined;

/// Validated physical memory mapping.

fn memory(address: u64) -> *mut u8;

/// Current kernel package-state table.

fn kernelGp() -> u64;

/// M-mode entry of the completion checker.

fn completion() -> u64;

/// Write one diagnostic byte to the fixture UART.

fn put(byte: u8);

/// Set the fixture UART interrupt-enable register.

fn uartInterrupt(value: u8);

/// Finish the machine test successfully.

fn finish();

/// Publish fixture state on the last initialized hart and wait on every hart.

export unsafe fn start(last: bool) {

    if last {

        setup();

        sync::storeRelease(&mut READY, 1);

    }

    while sync::loadAcquire(&READY) == 0 {

    }

}

/// Create one context per online hart before any dispatch can access the fixture.

unsafe fn setup() {

    let pending = try! slots::reserve(&mut domains::STORE.slots[..]);

    let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);

    capability::initialize(&mut TABLE, owner);

    try! backing::registerDomain(&mut pages::STORE.backings, owner);

    let authority = try! capability::install(&mut TABLE, capability::Entry {

        kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE | abi::ALLOCATE),

    });

    let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);

    let sourceEntry = try! capability::get(&TABLE, source);

    let page = try! pages::get(&pages::STORE, sourceEntry.object);

    let bytes = @sliceOf(memory(page.base), page.count * 4096);

    let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);

    let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,

        loader::Request { authority, source, offset: 0, length: length as u64 });

    let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);

    let cap = try! capability::get(&TABLE, handle);

    set USER = cap.object;

    createDescendant();

    let mut user = try! domains::get(&domains::STORE, USER);

    let self = try! capability::install(&mut user.memory.table, capability::Entry {

        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE),

    });

    assert boot::PLATFORM.irqSources == 31;

    let irq = try! interrupts::seed(&mut interrupts::STORE, &mut user.memory.table, 7);

    sync::storeRelease(memory(dataAddress(&user, &"spin::interrupt"[..])) as *mut u64, *irq);

    let target = try! registry::exported(&registry::STORE, user.image, &"spin::worker"[..]);

    let case shared::Target::Function(entry) = target else panic "worker entry";

    set COUNTERS = dataAddress(&user, &"spin::count"[..]);

    set MIGRATIONS = dataAddress(&user, &"spin::migrations"[..]);

    let notification = @sliceOf(memory(dataAddress(&user, &"spin::notification"[..])), 8);

    for i in 0..8 {

        set notification[i] = ((*self) >> (i as u64 * 8)) as u8;

    }

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) == 0 {

            continue;

        }

        let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);

        let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut user.memory.table, stack, (abi::READ | abi::WRITE) as u64);

        let permission = try! capability::get(&TABLE, stack);

        let page = try! pages::get(&pages::STORE, permission.object);

        let argument = @sliceOf(memory(page.base), 8);

        for i in 0..8 {

            set argument[i] = ((hart as u64) >> (i as u64 * 8)) as u8;

        }

        if hart == 0 {

            try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, page.base + 4096, page.base, 8);

            set CONTEXTS[hart] = user.initial;

        } else {

            set CONTEXTS[hart] = try! domains::contextCreate(&mut domains::STORE, &mut pages::STORE, &registry::STORE,

                &TABLE, handle, abi::ContextStart { entry, stack: page.base + 4096, args: page.base, size: 8 });

        }

    }

    let checkerHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);

    let checkerCap = try! capability::get(&TABLE, checkerHandle);

    let mut checker = try! domains::get(&domains::STORE, checkerCap.object);

    set CHECKER = checkerCap.object;

    set domains::STORE.records[checkerCap.object.index].state = domains::Lifecycle::Active;

    set domains::STORE.contexts[checker.initial.index].frame.pc = completion();

    set domains::STORE.contexts[checker.initial.index].frame.status = 0x1880;

    set domains::STORE.contexts[checker.initial.index].frame.registers[2] = domains::STORE.contexts[checker.initial.index].kernelStack.end;

    set domains::STORE.contexts[checker.initial.index].frame.registers[3] = kernelGp();

    idleSetup(authority, image);

    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));

    let start = clock + 1000000;

    let mut harts: u64 = 0;

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {

            set harts += 1;

        }

    }

    let end = start + 10000000 * harts * harts;

    let migrationEnd = end + 10000000 * harts * harts;

    let windowBase = dataAddress(&user, &"spin::windows"[..]);

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) == 0 {

            continue;

        }

        let begin = clock if hart == 0 else start;

        let mut budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, begin, 0xffffffffffffffff);

        if hart == 0 {

            let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, start, clock);

            set EXPIRED = try! budgets::transfer(&mut budgets::STORE, &mut TABLE, &mut checker.memory.table,

                budget, budgets::DEFAULT_RIGHTS as u64, clock);

            set budget = next;

        }

        let final = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, end, clock);

        let checking = try! budgets::split(&mut budgets::STORE, &mut TABLE, final, migrationEnd, clock);

        let future = try! budgets::split(&mut budgets::STORE, &mut TABLE, checking, migrationEnd + 10000000000, clock);

        let transferred = try! budgets::transfer(&mut budgets::STORE, &mut TABLE, &mut user.memory.table,

            future, budgets::DEFAULT_RIGHTS as u64, clock);

        let caller = (hart as u64 + harts - 1) % harts;

        let output = @sliceOf(memory(windowBase + caller * 8), 8);

        for i in 0..8 {

            set output[i] = ((*transferred) >> (i as u64 * 8)) as u8;

        }

        let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,

            budgets::Binding { budget, domain: handle, context: CONTEXTS[hart] }, clock);

        let rotated = ((hart as u64 + 1) % harts) as u32;

        let moved = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,

            budgets::Binding { budget: final, domain: handle, context: CONTEXTS[rotated] }, clock);

        if hart == 0 {

            let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,

                budgets::Binding { budget: checking, domain: checkerHandle, context: checker.initial }, clock);

        }

        if hart == 1 {

            set IDLE_WINDOW = checking;

        }

    }

}

/// Create an actual pending descendant before secondary harts enter dispatch.

unsafe fn createDescendant() {

    let mut user = try! domains::get(&domains::STORE, USER);

    let root = user.image;

    let authority = try! capability::install(&mut user.memory.table, capability::Entry {

        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::CREATE),

    });

    let image = try! registry::install(&registry::STORE, &mut user.memory.table, root);

    let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,

        &registry::STORE, &mut user.memory.table, authority, image);

    set DESCENDANT = (try! capability::get(&user.memory.table, handle)).object;

}

/// Resolve one exported private-data address in the fixture's user instance.

unsafe fn dataAddress(user: &domains::Domain, name: &[u8]) -> u64 {

    let target = try! registry::exported(&registry::STORE, user.image, name);

    let case shared::Target::Data(data) = target else panic "fixture data export";

    return user.graph.table[data.slot] + data.offset as u64;

}

/// Prepare a pending domain and its stack for activation on an idle hart.

unsafe fn idleSetup(authority: abi::Handle, image: abi::Handle) {

    set IDLE_HANDLE = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,

        &registry::STORE, &mut TABLE, authority, image);

    let permit = try! capability::get(&TABLE, IDLE_HANDLE);

    set IDLE_DOMAIN = permit.object;

    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);

    let allocation = try! capability::install(&mut domain.memory.table, capability::Entry {

        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::ALLOCATE),

    });

    set IDLE_CONTEXT = domain.initial;

    set WAKE_STATE = dataAddress(&domain, &"spin::wakeState"[..]);

    sync::storeRelease(memory(dataAddress(&domain, &"spin::idle"[..])) as *mut u64, 1);

    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);

    let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut domain.memory.table, stack, (abi::READ | abi::WRITE) as u64);

    let permission = try! capability::get(&TABLE, stack);

    let page = try! pages::get(&pages::STORE, permission.object);

    set IDLE_STACK = page.base + 4096;

    let mut checker = try! domains::get(&domains::STORE, CHECKER);

    set RECEIVER = try! capability::install(&mut checker.memory.table, capability::Entry {

        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::WAKE | abi::EXECUTE | abi::DESTROY),

    });

}

/// Check shared progress and context isolation after the user windows finish.

export unsafe fn verify() {

    let guard = sync::enter();

    let user = try! domains::get(&domains::STORE, USER);

    let mut counts: [u64; 8] = [0; 8];

    let mut notifications: u32 = 0;

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {

            set notifications += 1;

        }

    }

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) == 0 {

            continue;

        }

        let value = sync::loadAcquire(memory(COUNTERS + hart as u64 * 8) as *u64);

        set counts[hart] = value;

        let context = try! domains::context(&domains::STORE, USER, CONTEXTS[hart]);

        assert context.owner == USER;

        let mode = context.frame.status & 0x1800;

        assert mode == 0 or mode == 0x1800;

        if mode == 0x1800 {

            assert context.frame.registers[3] == kernelGp();

            assert context.frame.registers[2] >= context.kernelStack.start and context.frame.registers[2] <= context.kernelStack.end;

        } else {

            assert context.frame.registers[3] == user.graph.table.ptr as u64;

        }

        let destination = (hart + notifications - 1) % notifications;

        assert context.hart == nil or context.hart == destination;

        assert sync::loadAcquire(memory(MIGRATIONS + hart as u64 * 8) as *u64) == destination as u64 + 1;

        assert context.pending == nil;

    }

    assert domains::STORE.events.queues[USER.index].tail == notifications + 1;

    assert user.memory.ring.data[0].kind == events::INTERRUPT and user.memory.ring.data[0].code == 7;

    sync::leave(guard);

    for byte in "smp completed transactions:" {

        put(byte);

    }

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) == 0 {

            continue;

        }

        put(32); number(counts[hart]);

    }

    put(10);

    for hart in 0..8 {

        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {

            assert counts[hart] == 16;

        }

    }

    for target in 0..8 {

        if (boot::PLATFORM.harts & (1 << target)) == 0 {

            continue;

        }

        for action in &[remote::Action::Reschedule, remote::Action::InstructionSync] {

            let posting = sync::enter();

            let id = try! dispatch::request(target, action);

            sync::leave(posting);

            let mut acknowledged = false;

            for attempt in 0..100000 {

                let guard = sync::enter();

                if try! remote::completed(&remote::STORE, 0, id) {

                    try! remote::release(&mut remote::STORE, 0, id);

                    set acknowledged = true;

                }

                sync::leave(guard);

                if acknowledged {

                    break;

                }

            }

            assert acknowledged;

        }

    }

    for byte in "smp request acknowledgements passed\n" {

        put(byte);

    }

    expiredCall();

    if (boot::PLATFORM.harts & 2) <> 0 {

        idleWakeup();

    }

    interruptOwnership();

    interruptStorm();

    if (boot::PLATFORM.harts & 2) <> 0 {

        terminalWakeup();

        cancelAllocation();

        reclaimedDomains();

    }

    invariants::check(&TABLE);

    for byte in "smp final ownership and interval invariants passed\n" {

        put(byte);

    }

    let maximum = sync::maximum();

    assert maximum > 0 and maximum < 1000000;

    for byte in "smp metadata instructions: 0x" {

        put(byte);

    }

    number(maximum);

    put(10);

    let waiting = sync::maximumWait();

    assert waiting > 0;

    for byte in "smp acquisition instructions: 0x" {

        put(byte);

    }

    number(waiting);

    put(10);

    finish();

}

/// Validate elapsed authority against the current clock at the serialized boundary.

unsafe fn expiredCall() {

    let guard = sync::enter();

    let checker = try! domains::get(&domains::STORE, CHECKER);

    let before = try! budgets::query(&budgets::STORE, &checker.memory.table, EXPIRED);

    assert before.end < dispatch::clock();

    sync::leave(guard);

    let mut rejected = false;

    try calls::synchronized(CHECKER, 70, &[*EXPIRED, before.start + (before.end - before.start) / 2, 0, 0], 0) catch error {

        assert error == abi::Error::Busy; set rejected = true;

    };

    assert rejected;

    let checking = sync::enter();

    let after = try! budgets::query(&budgets::STORE, &checker.memory.table, EXPIRED);

    assert after.start == before.start and after.end == before.end;

    sync::leave(checking);

    for byte in "smp resumed call rejected expired authority\n" {

        put(byte);

    }

}

/// Transfer a queued IRQ, rearm after consumption, and mask it before capability drop.

unsafe fn interruptOwnership() {

    let guard = sync::enter();

    let mut user = try! domains::get(&domains::STORE, USER);

    let mut checker = try! domains::get(&domains::STORE, CHECKER);

    let device = try! plic::mapping(&boot::PLATFORM) else panic "missing PLIC";

    let enabled = memory(device.base + 0x2000 + device.context as u64 * 0x80) as *u32;

    assert (sync::read32(enabled) & (1 << 7)) == 0;

    assert interrupts::STORE.records[6].state == interrupts::State::Queued;

    let original = interrupts::STORE.records[6].handle;

    let target = try! capability::install(&mut user.memory.table, capability::Entry {

        kind: abi::Kind::Domain, object: CHECKER, rights: abi::Rights(0),

    });

    let back = try! capability::install(&mut checker.memory.table, capability::Entry {

        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(0),

    });

    let moved = try! calls::invoke(USER, 11, &[*original, *target, (abi::READ | abi::TRANSFER) as u64, 0], 0);

    assert interrupts::STORE.records[6].owner == CHECKER;

    let first = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));

    assert interrupts::STORE.records[6].state == interrupts::State::Queued;

    assert (sync::read32(enabled) & (1 << 7)) == 0;

    let event = try! events::pop(&mut checker.memory.ring) else panic "missing transferred IRQ";

    assert event.kind == events::INTERRUPT and event.code == 7;

    let before = domains::STORE.events.queues[USER.index].tail;

    let returned = try! calls::invoke(CHECKER, 11, &[moved, *back, (abi::READ | abi::TRANSFER) as u64, 0], 0);

    let second = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));

    assert domains::STORE.events.queues[USER.index].tail == before;

    assert (sync::read32(enabled) & (1 << 7)) == 0;

    let consumed = try! events::pop(&mut user.memory.ring) else panic "missing original IRQ";

    assert consumed.kind == events::INTERRUPT and consumed.code == 7;

    let rearmed = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));

    assert interrupts::STORE.records[6].state == interrupts::State::Idle;

    assert (sync::read32(enabled) & (1 << 7)) <> 0;

    let dropped = try! calls::invoke(USER, 12, &[returned, 0, 0, 0], 0);

    assert dropped == 0 and interrupts::STORE.slots[6].state == slots::State::Free;

    assert (sync::read32(enabled) & (1 << 7)) == 0;

    sync::leave(guard);

    for byte in "smp IRQ transfer, rearm, and drop passed\n" {

        put(byte);

    }

}

/// Keep a level source asserted across full queues and repeated driver progress.

unsafe fn interruptStorm() {

    let guard = sync::enter();

    let mut checker = try! domains::get(&domains::STORE, CHECKER);

    let handle = try! interrupts::seed(&mut interrupts::STORE, &mut checker.memory.table, 10);

    let device = try! plic::mapping(&boot::PLATFORM) else panic "missing PLIC";

    let enabled = memory(device.base + 0x2000 + device.context as u64 * 0x80) as *u32;

    sync::leave(guard);

    for i in 0..events::ORDINARY {

        let filling = sync::enter();

        assert try! domains::notify(&mut domains::STORE, CHECKER,

            events::Event { kind: events::WAKEUP, reserved: 0, code: i, value: 0 });

        sync::leave(filling);

    }

    let arming = sync::enter();

    let ready = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));

    uartInterrupt(2);

    sync::leave(arming);

    let mut pending = false;

    for attempt in 0..10000 {

        let checking = sync::enter();

        set pending = interrupts::STORE.records[9].state == interrupts::State::Pending;

        if pending {

            assert (sync::read32(enabled) & (1 << 10)) == 0;

            let queue = domains::STORE.events.queues[CHECKER.index];

            assert queue.tail - queue.head == events::ORDINARY;

        }

        sync::leave(checking);

        if pending {

            break;

        }

    }

    assert pending;

    for i in 0..events::ORDINARY {

        let draining = sync::enter();

        let mut receiver = try! domains::get(&domains::STORE, CHECKER);

        let event = try! events::pop(&mut receiver.memory.ring) else panic "missing saturation event";

        assert event.kind == events::WAKEUP and event.code == i;

        sync::leave(draining);

    }

    let posting = sync::enter();

    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));

    try! timers::request(&mut timers::STORE, &domains::STORE, CHECKER, 0, clock, 2000000, 0x77);

    let request = try! dispatch::request(0, remote::Action::Reschedule);

    sync::leave(posting);

    let acknowledged = try! dispatch::awaitRequest(0, request);

    assert acknowledged == 0;

    let mut received: u32 = 0;

    let mut timed = false;

    for attempt in 0..1000 {

        let consuming = sync::enter();

        let mut receiver = try! domains::get(&domains::STORE, CHECKER);

        if let event = try! events::pop(&mut receiver.memory.ring) {

            if event.kind == events::INTERRUPT {

                assert event.code == 10;

                set received += 1;

            } else {

                assert event.kind == events::TIMEOUT and event.code == 0x77;

                assert dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) >= clock + 2000000;

                set timed = true;

            }

        }

        let notified = plic::service(dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)));

        sync::leave(consuming);

        if timed and received >= 16 {

            break;

        }

    }

    let stopping = sync::enter();

    uartInterrupt(0);

    let dropped = try! calls::invoke(CHECKER, 12, &[*handle, 0, 0, 0], 0);

    assert dropped == 0 and (sync::read32(enabled) & (1 << 10)) == 0;

    sync::leave(stopping);

    assert timed and received >= 16;

    for byte in "smp IRQ saturation and storm timeout passed\n" {

        put(byte);

    }

}

/// Resume an actual U-mode Wait call through an IPI to an idle remote hart.

unsafe fn idleWakeup() {

    let guard = sync::enter();

    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));

    let handle = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,

        budgets::Binding { budget: IDLE_WINDOW, domain: IDLE_HANDLE, context: IDLE_CONTEXT }, clock);

    let request = try! dispatch::request(1, remote::Action::Reschedule);

    sync::leave(guard);

    let acknowledged = try! dispatch::awaitRequest(0, request);

    assert acknowledged == 0;

    let activated = try! calls::synchronized(CHECKER, 21, &[*RECEIVER, IDLE_STACK, 0, 0], clock);

    assert activated == 0 and waiting();

    let now = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));

    let result = try! calls::synchronized(CHECKER, 50, &[*RECEIVER, 0x55, 0, 0], now);

    assert result == 0;

    let mut resumed = false;

    for attempt in 0..100000 {

        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 3 {

            set resumed = true;

            break;

        }

    }

    assert resumed;

    for byte in "smp idle-hart activation and wakeup passed\n" {

        put(byte);

    }

    remoteTimeout();

}

/// Deliver a timeout from hart zero to a waiting user context on idle hart one.

unsafe fn remoteTimeout() {

    let guard = sync::enter();

    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);

    let event = try! events::pop(&mut domain.memory.ring) else panic "missing wakeup";

    assert event.kind == events::WAKEUP and event.code == 0x55;

    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 4);

    sync::leave(guard);

    assert waiting();

    let posting = sync::enter();

    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));

    try! timers::request(&mut timers::STORE, &domains::STORE, IDLE_DOMAIN, 0, clock, 100000, 0x66);

    let request = try! dispatch::request(0, remote::Action::Reschedule);

    sync::leave(posting);

    let acknowledged = try! dispatch::awaitRequest(0, request);

    assert acknowledged == 0;

    let mut resumed = false;

    for attempt in 0..100000 {

        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 6 {

            set resumed = true;

            break;

        }

    }

    assert resumed;

    for byte in "smp remote timeout passed\n" {

        put(byte);

    }

}

/// Wake a parent waiting on hart one through terminal delivery on hart zero.

unsafe fn terminalWakeup() {

    let guard = sync::enter();

    let mut receiver = try! domains::get(&domains::STORE, IDLE_DOMAIN);

    let timeout = try! events::pop(&mut receiver.memory.ring) else panic "missing idle timeout";

    assert timeout.kind == events::TIMEOUT and timeout.code == 0x66;

    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 7);

    sync::leave(guard);

    assert waiting();

    let stopping = sync::enter();

    let mut checker = try! domains::get(&domains::STORE, CHECKER);

    let child = try! capability::install(&mut checker.memory.table, capability::Entry {

        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::DESTROY),

    });

    let parent = try! calls::invoke(CHECKER, 23, &[*child, *RECEIVER, 0, 0], 0);

    assert parent == 0;

    let cancelled = try! dispatch::request(1, remote::Action::Budget(remote::Call {

        owner: USER, context: CONTEXTS[0], operation: 70, arguments: [0; 4],

    }));

    set domains::STORE.contexts[CONTEXTS[0].index].pending = cancelled;

    let readiness = try! remote::reserve(&mut remote::STORE, 0, 2);

    let batch = remote::publish(&mut remote::STORE, readiness, remote::Action::Ready(CONTEXTS[1]));

    dispatch::startReady(CONTEXTS[1], &batch);

    let stopped = try! domains::terminateTree(&mut domains::STORE, USER, events::CHILD_EXIT, 23);

    assert stopped == 0;

    assert domains::STORE.records[DESCENDANT.index].state == domains::Lifecycle::Dead;

    assert try! events::acknowledged(&domains::STORE.events, DESCENDANT);

    let request = try! dispatch::request(0, remote::Action::Reschedule);

    sync::leave(stopping);

    let acknowledged = try! dispatch::awaitRequest(0, request);

    assert acknowledged == 0;

    let mut resumed = false;

    for attempt in 0..100000 {

        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 9 {

            set resumed = true;

            break;

        }

    }

    assert resumed;

    let checking = sync::enter();

    assert remote::STORE.slots[cancelled.index].state == slots::State::Free;

    assert remote::STORE.slots[batch.ids[1].index].state == slots::State::Free;

    assert domains::STORE.contexts[CONTEXTS[0].index].pending == nil;

    assert domains::STORE.contexts[CONTEXTS[1].index].pending == nil;

    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);

    let event = try! events::pop(&mut domain.memory.ring) else panic "missing remote exit";

    assert event.kind == events::CHILD_EXIT and event.code == 23 and event.value == abi::id(USER);

    sync::leave(checking);

    for byte in "smp cascade cancelled calls and woke remote parent\n" {

        put(byte);

    }

}

/// Stop a real page-clear continuation and recover its unpublished frames.

unsafe fn cancelAllocation() {

    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 10);

    let mut allocation: pages::Allocation = undefined;

    let mut request: ?abi::Ref = nil;

    for attempt in 0..100000 {

        let guard = sync::enter();

        let index = IDLE_CONTEXT.index;

        if (lifecycle::CALLS.active[index / 32] & (1 << (index % 32))) <> 0 {

            match lifecycle::CALLS.records[index].allocation {

                case lifecycle::Allocation::Page(value) => {

                    set allocation = value;

                },

                else => panic "page allocation required",

            }

            assert allocation.owner == IDLE_DOMAIN and allocation.run.count == 4096;

            sync::leave(guard);

            let result = try! calls::synchronized(CHECKER, 22, &[*RECEIVER, 0, 0, 0], dispatch::clock());

            assert result == 0;

            let checking = sync::enter();

            assert domains::STORE.records[IDLE_DOMAIN.index].state == domains::Lifecycle::Dead;

            set request = try! dispatch::request(1, remote::Action::Reschedule);

            sync::leave(checking);

            break;

        }

        sync::leave(guard);

    }

    let stopped = request else panic "page call did not reserve frames";

    let acknowledged = try! dispatch::awaitRequest(0, stopped);

    assert acknowledged == 0;

    let snapshot = sync::enter();

    let context = try! domains::context(&domains::STORE, IDLE_DOMAIN, IDLE_CONTEXT);

    for byte in "smp cancelled page status and progress: " {

        put(byte);

    }

    number(context.frame.status); put(32);

    number(sync::loadAcquire(memory(WAKE_STATE) as *u64)); put(10);

    assert context.hart == nil and context.state == domains::ContextState::Stopped;

    assert (context.frame.status & 0x1800) == 0x1800;

    assert sync::loadAcquire(memory(WAKE_STATE) as *u64) == 11;

    sync::leave(snapshot);

    let mut reclaimed = false;

    for attempt in 0..1000 {

        let checking = sync::enter();

        set reclaimed = pages::STORE.backings.pool.retiredCount == 0;

        if reclaimed {

            sync::leave(checking);

            break;

        }

        let progress = try! dispatch::request(0, remote::Action::Reschedule);

        sync::leave(checking);

        let result = try! dispatch::awaitRequest(0, progress);

        assert result == 0;

    }

    assert reclaimed;

    let guard = sync::enter();

    assert (lifecycle::CALLS.active[IDLE_CONTEXT.index / 32] & (1 << (IDLE_CONTEXT.index % 32))) == 0;

    assert pages::STORE.slots[allocation.page.index].state == slots::State::Free;

    assert pages::STORE.backings.slots[allocation.backing.index].state == slots::State::Free;

    for i in allocation.run.first..allocation.run.first + allocation.run.count {

        assert frames::available(&pages::STORE.backings.pool, i);

    }

    sync::leave(guard);

    for byte in "smp cancelled preempted page allocation\n" {

        put(byte);

    }

}

/// Wait for dispatch-driven teardown and verify all dead execution ownership is gone.

unsafe fn reclaimedDomains() {

    for attempt in 0..100000 {

        let guard = sync::enter();

        let mut done = pages::STORE.backings.pool.retiredCount == 0;

        for owner in [USER, IDLE_DOMAIN, DESCENDANT] {

            if slots::matches(&domains::STORE.slots[..], owner, slots::State::Live) {

                set done = false;

            }

        }

        if done {

            assert domains::STORE.dead == 0;

            for owner in [USER, IDLE_DOMAIN, DESCENDANT] {

                assert domains::STORE.slots[owner.index].state == slots::State::Free;

                assert pages::STORE.backings.domains[owner.index] == 0;

                for i in 0..domains::STORE.contextSlots.len {

                    if domains::STORE.contextSlots[i].state == slots::State::Live {

                        assert domains::STORE.contexts[i].owner <> owner;

                    }

                }

                for i in 0..budgets::STORE.slots.len {

                    if budgets::STORE.slots[i].state == slots::State::Live {

                        assert budgets::STORE.windows[i].owner <> owner;

                    }

                }

                for i in 0..interrupts::STORE.slots.len {

                    if interrupts::STORE.slots[i].state == slots::State::Live {

                        assert interrupts::STORE.records[i].owner <> owner;

                    }

                }

            }

            sync::leave(guard);

            for byte in "smp reaped dead domains and recovered ownership\n" {

                put(byte);

            }

            return;

        }

        let progress = try! dispatch::request(0, remote::Action::Reschedule);

        sync::leave(guard);

        let result = try! dispatch::awaitRequest(0, progress);