test/smp/kernel/dispatchcheck.rad 39.6 KiB raw
1
//! One domain executing separate user contexts on every online hart.
2
export mod inputs;
3
export mod cases;
4
export mod invariants;
5
6
use std::arch::rv64;
7
use std::arch::rv64::shared;
8
use std::fmt;
9
use kernel::abi;
10
use kernel::boot;
11
use kernel::budgets;
12
use kernel::calls;
13
use kernel::capability;
14
use kernel::dispatch;
15
use kernel::domains;
16
use kernel::events;
17
use kernel::fixture;
18
use kernel::frames;
19
use kernel::instances;
20
use kernel::interrupts;
21
use kernel::lifecycle;
22
use kernel::limits;
23
use kernel::pages;
24
use kernel::plic;
25
use kernel::registers;
26
use kernel::registry;
27
use kernel::remote;
28
use kernel::slots;
29
use kernel::state;
30
use kernel::sync;
31
use kernel::timers;
32
33
/// Delay before concurrent user windows, in timer ticks.
34
constant START_DELAY: u64 = 1000000;
35
36
/// Base work and migration duration, scaled by the square of hart count.
37
constant WORK_TICKS: u64 = 10000000;
38
39
/// Reserved duration for final machine checks.
40
constant CHECK_TICKS: u64 = 10000000000;
41
42
/// Maximum bounded polls for remote progress.
43
constant POLL_ATTEMPTS: u32 = 100000;
44
45
/// Maximum polls for a level interrupt to reach a saturated event queue.
46
constant IRQ_PENDING_ATTEMPTS: u32 = 10000;
47
/// Maximum consumed events while checking interrupt saturation and timeout.
48
constant IRQ_EVENTS: u32 = 1000;
49
/// Maximum maintenance requests after a cancelled page allocation.
50
constant RECLAIM_ATTEMPTS: u32 = 1000;
51
/// Level-triggered UART source asserted during the storm check.
52
constant UART_IRQ: u32 = 10;
53
/// Minimum repeated UART events needed to check progress during a storm.
54
constant STORM_EVENTS: u32 = 16;
55
/// Byte offset of the first PLIC context enable word.
56
constant PLIC_ENABLES: u64 = 0x2000;
57
/// Byte stride between PLIC context enable words.
58
constant PLIC_ENABLE_STRIDE: u64 = 0x80;
59
/// Timer ticks before the remote idle-context timeout.
60
constant REMOTE_TIMEOUT: u64 = 100000;
61
62
/// Timeout while the fixture holds a level interrupt asserted.
63
constant IRQ_TIMEOUT: u64 = 2000000;
64
65
// Setup initializes retained state for online harts before releasing READY.
66
// Other harts must acquire READY before they read this state.
67
/// Bootstrap resource authority.
68
unsafe static TABLE: capability::Table = undefined;
69
/// Shared user domain generation.
70
unsafe static USER: abi::Ref = undefined;
71
/// Pending domain in the user's creation subtree.
72
unsafe static DESCENDANT: abi::Ref = undefined;
73
/// Per-hart user execution contexts.
74
unsafe static CONTEXTS: [abi::Ref; limits::HARTS] = undefined;
75
/// Mapped progress counters in the user's private state.
76
unsafe static COUNTERS: u64 = 0;
77
/// Per-context migration observations in the user's private state.
78
unsafe static MIGRATIONS: u64 = 0;
79
/// Shared handshake for the independently activated context on hart one.
80
unsafe static WAKE_STATE: u64 = 0;
81
/// Bootstrap management capability for the pending idle-wakeup domain.
82
unsafe static IDLE_HANDLE: abi::Handle = undefined;
83
/// Context whose Wait call must release the remote hart.
84
unsafe static IDLE_CONTEXT: abi::Ref = undefined;
85
/// Stack Page handle in the idle-wakeup domain's table.
86
unsafe static IDLE_STACK: u64 = 0;
87
/// Generation of the idle-wakeup domain.
88
unsafe static IDLE_DOMAIN: abi::Ref = undefined;
89
/// Completion checker's live domain generation.
90
unsafe static CHECKER: abi::Ref = undefined;
91
/// Unbound window that hart zero retires before the checker starts.
92
unsafe static EXPIRED: abi::Handle = undefined;
93
/// Checker's authority to activate and notify the idle-wakeup domain.
94
unsafe static RECEIVER: abi::Handle = undefined;
95
/// Unbound authority reserved for the idle-wakeup check on hart one.
96
unsafe static IDLE_WINDOW: ?abi::Handle = nil;
97
/// Release/acquire barrier after fixture publication.
98
static READY: u64 = 0;
99
/// Validated physical memory mapping.
100
fn memory(address: u64) -> *mut u8;
101
/// Current kernel package-state table.
102
fn kernelGp() -> u64;
103
/// M-mode entry of the completion checker.
104
fn completion() -> u64;
105
/// Write one diagnostic byte to the fixture UART.
106
fn put(byte: u8);
107
/// Set the fixture UART interrupt-enable register.
108
fn uartInterrupt(value: u8);
109
/// Finish the machine test successfully.
110
fn finish();
111
/// Sleep until this hart's software interrupt is pending. Interrupts stay
112
/// globally masked, so the interrupt wakes the hart without trapping.
113
fn park();
114
115
/// Publish fixture state on the last initialized hart and wait on every hart.
116
export unsafe fn start(last: bool) {
117
    if last {
118
        setup();
119
        sync::storeRelease(&mut READY, 1);
120
        // Every online hart finished dispatch::initialize before arriving.
121
        for hart in 0 .. limits::HARTS {
122
            if hart <> dispatch::hart() and (boot::PLATFORM.harts & (1 << hart))
123
                <> 0
124
            {
125
                dispatch::signal(hart);
126
            }
127
        }
128
    }
129
    while sync::loadAcquire(&READY) == 0 {
130
        park();
131
    }
132
}
133
134
/// Create one context per online hart before any dispatch can access the
135
/// fixture.
136
unsafe fn setup() {
137
    let authority = fixture::bootstrap(&mut TABLE);
138
    let image = fixture::load(&mut TABLE, authority, &inputs::INPUT[..]);
139
    let handle = try! domains::create(
140
        &mut state::SHARED.domains,
141
        &mut state::SHARED.pages.backings,
142
        &state::SHARED.registry,
143
        &mut TABLE,
144
        authority,
145
        image
146
    );
147
    let cap = try! capability::get(&TABLE, handle);
148
    set USER = cap.object;
149
    createDescendant();
150
    let mut user = try! domains::get(&state::SHARED.domains, USER);
151
    let self = try! capability::install(&mut user.memory.table, capability::Entry {
152
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE)
153
    });
154
    assert boot::PLATFORM.irqSources == 31;
155
    let irq = try! interrupts::seed(
156
        &mut state::SHARED.interrupts, &mut user.memory.table, cases::IRQ_SOURCE
157
    );
158
    sync::storeRelease(
159
        memory(dataAddress(
160
            USER, &"spin::interrupt"[..]
161
        )) as *mut u64,
162
        *irq
163
    );
164
    let target = try! registry::exported(
165
        &state::SHARED.registry, user.image, &"spin::worker"[..]
166
    );
167
    let case shared::Target::Function(entry) = target else panic "setup: worker entry";
168
    set COUNTERS = dataAddress(USER, &"spin::count"[..]);
169
    set MIGRATIONS = dataAddress(USER, &"spin::migrations"[..]);
170
    let notification = @sliceOf(
171
        memory(dataAddress(
172
            USER, &"spin::notification"[..]
173
        )),
174
        8
175
    );
176
    for i in 0 .. fixture::WORD_BYTES {
177
        set notification[i] = (*self >> (i as u64 * 8)) as u8;
178
    }
179
    for hart in 0 .. limits::HARTS {
180
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
181
            continue;
182
        }
183
        let moved = fixture::stack(
184
            &mut TABLE, authority, &mut user.memory.table
185
        );
186
        if hart == 0 {
187
            try! fixture::activate(&TABLE, handle, moved, abi::Handle(0), 0);
188
            set CONTEXTS[hart] = user.initial;
189
        } else {
190
            set CONTEXTS[hart] =
191
                try! domains::contextCreate(
192
                    &mut state::SHARED.domains,
193
                    &mut state::SHARED.pages,
194
                    &state::SHARED.registry,
195
                    USER,
196
                    abi::ContextStart {
197
                        entry, stack: *moved, args: hart as u64, size: 8
198
                    },
199
                    &mut user.memory.table
200
                );
201
        }
202
    }
203
    let checkerHandle = try! domains::create(
204
        &mut state::SHARED.domains,
205
        &mut state::SHARED.pages.backings,
206
        &state::SHARED.registry,
207
        &mut TABLE,
208
        authority,
209
        image
210
    );
211
    let checkerCap = try! capability::get(&TABLE, checkerHandle);
212
    let mut checker = try! domains::get(
213
        &state::SHARED.domains, checkerCap.object
214
    );
215
    set CHECKER = checkerCap.object;
216
    fixture::machine(checkerCap.object, completion(), kernelGp());
217
    idleSetup(authority, image);
218
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
219
    let start = clock + START_DELAY;
220
    let mut harts: u64 = 0;
221
    for hart in 0 .. limits::HARTS {
222
        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
223
            set harts += 1;
224
        }
225
    }
226
    let end = start + WORK_TICKS * harts * harts;
227
    let migrationEnd = end + WORK_TICKS * harts * harts;
228
    let windowBase = dataAddress(USER, &"spin::windows"[..]);
229
    for hart in 0 .. limits::HARTS {
230
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
231
            continue;
232
        }
233
        let begin = clock if hart == 0 else start;
234
        let mut budget = try! budgets::seed(
235
            &mut state::SHARED.budgets, &mut TABLE, hart, begin, 0xffffffffffffffff
236
        );
237
        if hart == 0 {
238
            let next = try! budgets::split(
239
                &mut state::SHARED.budgets, &mut TABLE, budget, start, clock
240
            );
241
            set EXPIRED = try! budgets::transfer(
242
                &mut state::SHARED.budgets,
243
                &mut TABLE,
244
                &mut checker.memory.table,
245
                budget,
246
                budgets::DEFAULT_RIGHTS as u64,
247
                clock
248
            );
249
            set budget = next;
250
        }
251
        let final = try! budgets::split(
252
            &mut state::SHARED.budgets, &mut TABLE, budget, end, clock
253
        );
254
        let checking = try! budgets::split(
255
            &mut state::SHARED.budgets, &mut TABLE, final, migrationEnd, clock
256
        );
257
        let future = try! budgets::split(
258
            &mut state::SHARED.budgets, &mut TABLE, checking, migrationEnd + CHECK_TICKS, clock
259
        );
260
        let transferred = try! budgets::transfer(
261
            &mut state::SHARED.budgets,
262
            &mut TABLE,
263
            &mut user.memory.table,
264
            future,
265
            budgets::DEFAULT_RIGHTS as u64,
266
            clock
267
        );
268
        let caller = (hart as u64 + harts - 1) % harts;
269
        let output = @sliceOf(memory(windowBase + caller * 8), 8);
270
        for i in 0 .. fixture::WORD_BYTES {
271
            set output[i] = (*transferred >> (i as u64 * 8)) as u8;
272
        }
273
        let bound = try! calls::rebind(
274
            &mut state::SHARED,
275
            &mut TABLE,
276
            budgets::Binding {
277
                budget, domain: handle, context: CONTEXTS[hart]
278
            },
279
            clock
280
        );
281
        let rotated = ((hart as u64 + 1) % harts) as u32;
282
        let moved = try! calls::rebind(
283
            &mut state::SHARED,
284
            &mut TABLE,
285
            budgets::Binding {
286
                budget: final, domain: handle, context: CONTEXTS[rotated]
287
            },
288
            clock
289
        );
290
        if hart == 0 {
291
            let bound = try! calls::rebind(
292
                &mut state::SHARED,
293
                &mut TABLE,
294
                budgets::Binding {
295
                    budget: checking, domain: checkerHandle, context: checker.initial
296
                },
297
                clock
298
            );
299
        }
300
        if hart == 1 {
301
            set IDLE_WINDOW = checking;
302
        }
303
    }
304
}
305
306
/// Create an actual pending descendant before secondary harts enter dispatch.
307
unsafe fn createDescendant() {
308
    let mut user = try! domains::get(&state::SHARED.domains, USER);
309
    let root = user.image;
310
    let authority = try! capability::install(&mut user.memory.table, capability::Entry {
311
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::CREATE)
312
    });
313
    let image = try! registry::install(
314
        &state::SHARED.registry, &mut user.memory.table, root
315
    );
316
    let handle = try! domains::create(
317
        &mut state::SHARED.domains,
318
        &mut state::SHARED.pages.backings,
319
        &state::SHARED.registry,
320
        &mut user.memory.table,
321
        authority,
322
        image
323
    );
324
    set DESCENDANT = (try! capability::get(&user.memory.table, handle)).object;
325
}
326
327
/// Resolve one exported private-data address in the fixture's user instance.
328
unsafe fn dataAddress(owner: abi::Ref, name: &[u8]) -> u64 {
329
    let user = try! domains::get(&state::SHARED.domains, owner);
330
    let target = try! registry::exported(
331
        &state::SHARED.registry, user.image, name
332
    );
333
    let case shared::Target::Data(data) = target else panic "dataAddress: fixture data export";
334
    return instances::base(
335
        &state::SHARED.domains.tables[owner.index], data.slot
336
    ) + data.offset as u64;
337
}
338
339
/// Prepare a pending domain and its stack for activation on an idle hart.
340
unsafe fn idleSetup(authority: abi::Handle, image: abi::Handle) {
341
    set IDLE_HANDLE = try! domains::create(
342
        &mut state::SHARED.domains,
343
        &mut state::SHARED.pages.backings,
344
        &state::SHARED.registry,
345
        &mut TABLE,
346
        authority,
347
        image
348
    );
349
    let permit = try! capability::get(&TABLE, IDLE_HANDLE);
350
    set IDLE_DOMAIN = permit.object;
351
    let mut domain = try! domains::get(&state::SHARED.domains, IDLE_DOMAIN);
352
    let allocation = try! capability::install(&mut domain.memory.table, capability::Entry {
353
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::ALLOCATE)
354
    });
355
    set IDLE_CONTEXT = domain.initial;
356
    set WAKE_STATE = dataAddress(IDLE_DOMAIN, &"spin::wakeState"[..]);
357
    sync::storeRelease(
358
        memory(dataAddress(
359
            IDLE_DOMAIN, &"spin::idle"[..]
360
        )) as *mut u64,
361
        1
362
    );
363
    let moved = fixture::stack(
364
        &mut TABLE, authority, &mut domain.memory.table
365
    );
366
    set IDLE_STACK = *moved;
367
    let mut checker = try! domains::get(&state::SHARED.domains, CHECKER);
368
    set RECEIVER = try! capability::install(&mut checker.memory.table, capability::Entry {
369
        kind: abi::Kind::Domain,
370
        object: IDLE_DOMAIN,
371
        rights: abi::Rights(abi::WAKE | abi::EXECUTE | abi::DESTROY),
372
    });
373
}
374
375
/// Check shared progress and context isolation after the user windows finish.
376
/// Only the scheduled checker runs these phases. Kernel metadata is locked by
377
/// each phase; user progress words use acquire loads.
378
export unsafe fn verify() {
379
    checkUserContexts();
380
    checkAcknowledgements();
381
    expiredCall();
382
    if (boot::PLATFORM.harts & 2) <> 0 {
383
        idleWakeup();
384
    }
385
    interruptOwnership();
386
    interruptStorm();
387
    if (boot::PLATFORM.harts & 2) <> 0 {
388
        terminalWakeup();
389
        cancelAllocation();
390
        reclaimedDomains();
391
    }
392
    invariants::check(&TABLE);
393
    for byte in "smp final ownership and interval invariants passed\n" {
394
        put(byte);
395
    }
396
    finish();
397
}
398
399
/// Check saved contexts, migration destinations, and all per-hart progress.
400
unsafe fn checkUserContexts() {
401
    let guard = sync::enter();
402
    let user = try! domains::get(&state::SHARED.domains, USER);
403
    let mut counts: [u64; limits::HARTS] = [0; limits::HARTS];
404
    let mut notifications: u32 = 0;
405
    for hart in 0 .. limits::HARTS {
406
        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
407
            set notifications += 1;
408
        }
409
    }
410
    for hart in 0 .. limits::HARTS {
411
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
412
            continue;
413
        }
414
        let value = sync::loadAcquire(
415
            memory(
416
                COUNTERS
417
                    + hart as u64 * 8
418
            ) as *u64
419
        );
420
        set counts[hart] = value;
421
        let context = try! domains::context(
422
            &state::SHARED.domains, USER, CONTEXTS[hart]
423
        );
424
        assert context.owner == USER;
425
        let gp = context.frame.registers[*rv64::GP as u32];
426
        let sp = context.frame.registers[*rv64::SP as u32];
427
        let mode = context.frame.status & fixture::MACHINE_PRIVILEGE;
428
        assert mode == 0 or mode == fixture::MACHINE_PRIVILEGE;
429
        if mode == fixture::MACHINE_PRIVILEGE {
430
            assert gp == kernelGp();
431
            assert sp >= context.kernelStack.start
432
                and sp <= context.kernelStack.end;
433
        } else {
434
            assert gp
435
                == instances::address(&state::SHARED.domains.tables[USER.index]);
436
        }
437
        let destination = (hart + notifications - 1) % notifications;
438
        assert context.hart == nil or context.hart == destination;
439
        assert sync::loadAcquire(
440
            memory(
441
                MIGRATIONS
442
                    + hart as u64 * 8
443
            ) as *u64
444
        ) == destination as u64
445
            + 1;
446
        assert context.pending == nil;
447
    }
448
    assert state::SHARED.domains.events.queues[USER.index].tail
449
        == notifications + 1;
450
    assert user.memory.ring.data[0].kind == events::INTERRUPT
451
        and user.memory.ring.data[0].code == cases::IRQ_SOURCE;
452
    sync::leave(guard);
453
    for byte in "smp completed transactions:" {
454
        put(byte);
455
    }
456
    for hart in 0 .. limits::HARTS {
457
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
458
            continue;
459
        }
460
        put(32);
461
        number(counts[hart]);
462
    }
463
    put(10);
464
    for hart in 0 .. limits::HARTS {
465
        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
466
            assert counts[hart] == cases::TRANSACTIONS as u64;
467
        }
468
    }
469
}
470
471
/// Post one request per online hart and check acknowledgement ownership.
472
unsafe fn checkAcknowledgements() {
473
    for target in 0 .. limits::HARTS {
474
        if (boot::PLATFORM.harts & (1 << target)) == 0 {
475
            continue;
476
        }
477
        let posting = sync::enter();
478
        let id = try! dispatch::request(
479
            &mut state::SHARED.remote, target, remote::Action::Reschedule
480
        );
481
        sync::leave(posting);
482
        let mut acknowledged = false;
483
        for attempt in 0 .. POLL_ATTEMPTS {
484
            let guard = sync::enter();
485
            if try! remote::completed(&state::SHARED.remote, 0, id) {
486
                try! remote::release(&mut state::SHARED.remote, 0, id);
487
                set acknowledged = true;
488
            }
489
            sync::leave(guard);
490
            if acknowledged {
491
                break;
492
            }
493
        }
494
        assert acknowledged;
495
    }
496
    for byte in "smp request acknowledgements passed\n" {
497
        put(byte);
498
    }
499
}
500
501
/// Dispatch on the window's hart retires elapsed authority, so a later call
502
/// finds a stale handle.
503
unsafe fn expiredCall() {
504
    let guard = sync::enter();
505
    let checker = try! domains::get(&state::SHARED.domains, CHECKER);
506
    let mut retired = false;
507
    try budgets::query(
508
        &state::SHARED.budgets, &checker.memory.table, EXPIRED
509
    ) catch error {
510
        assert error == abi::Error::BadHandle;
511
        set retired = true;
512
    };
513
    assert retired;
514
    sync::leave(guard);
515
    let mut rejected = false;
516
    try calls::synchronized(
517
        CHECKER, abi::Operation::BudgetSplit as u32, &[*EXPIRED, 1, 0, 0], 0
518
    ) catch error {
519
        assert error == abi::Error::BadHandle;
520
        set rejected = true;
521
    };
522
    assert rejected;
523
    for byte in "smp resumed call rejected expired authority\n" {
524
        put(byte);
525
    }
526
}
527
528
/// Transfer a queued IRQ, rearm after consumption, and mask it before
529
/// capability drop.
530
unsafe fn interruptOwnership() {
531
    let guard = sync::enter();
532
    let mut user = try! domains::get(&state::SHARED.domains, USER);
533
    let mut checker = try! domains::get(&state::SHARED.domains, CHECKER);
534
    let device = try! plic::mapping(&boot::PLATFORM) else panic "interruptOwnership: missing PLIC";
535
    let enabled = PLIC_ENABLES + device.context as u64 * PLIC_ENABLE_STRIDE;
536
    let bit = 1 as u32 << cases::IRQ_SOURCE;
537
    assert (registers::read32(device.registers, enabled) & bit) == 0;
538
    assert state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].state
539
        == interrupts::State::Queued;
540
    let original =
541
        state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].handle;
542
    let target = try! capability::install(&mut user.memory.table, capability::Entry {
543
        kind: abi::Kind::Domain, object: CHECKER, rights: abi::Rights(0)
544
    });
545
    let back = try! capability::install(&mut checker.memory.table, capability::Entry {
546
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(0)
547
    });
548
    let moved = try! calls::invoke(
549
        USER,
550
        abi::Operation::CapabilityTransfer as u32,
551
        &[
552
            *original, *target,
553
            (abi::READ | abi::TRANSFER) as u64, 0,
554
        ],
555
        0
556
    );
557
    assert state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].owner
558
        == CHECKER;
559
    let first = plic::service(
560
        &mut state::SHARED.interrupts,
561
        &mut state::SHARED.domains,
562
        &state::SHARED.budgets,
563
        dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0))
564
    );
565
    assert state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].state
566
        == interrupts::State::Queued;
567
    assert (registers::read32(device.registers, enabled) & bit) == 0;
568
    let event = try! events::pop(&mut checker.memory.ring)
569
        else panic "interruptOwnership: missing transferred IRQ";
570
    assert event.kind == events::INTERRUPT and event.code == cases::IRQ_SOURCE;
571
    let before = state::SHARED.domains.events.queues[USER.index].tail;
572
    let returned = try! calls::invoke(
573
        CHECKER,
574
        abi::Operation::CapabilityTransfer as u32,
575
        &[
576
            moved, *back,
577
            (abi::READ | abi::TRANSFER) as u64, 0,
578
        ],
579
        0
580
    );
581
    let second = plic::service(
582
        &mut state::SHARED.interrupts,
583
        &mut state::SHARED.domains,
584
        &state::SHARED.budgets,
585
        dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0))
586
    );
587
    assert state::SHARED.domains.events.queues[USER.index].tail == before;
588
    assert (registers::read32(device.registers, enabled) & bit) == 0;
589
    let consumed = try! events::pop(&mut user.memory.ring)
590
        else panic "interruptOwnership: missing original IRQ";
591
    assert consumed.kind == events::INTERRUPT
592
        and consumed.code == cases::IRQ_SOURCE;
593
    let rearmed = plic::service(
594
        &mut state::SHARED.interrupts,
595
        &mut state::SHARED.domains,
596
        &state::SHARED.budgets,
597
        dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0))
598
    );
599
    assert state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].state
600
        == interrupts::State::Idle;
601
    assert (registers::read32(device.registers, enabled) & bit) <> 0;
602
    let dropped = try! calls::invoke(
603
        USER, abi::Operation::CapabilityDrop as u32, &[returned, 0, 0, 0], 0
604
    );
605
    assert dropped == 0
606
        and state::SHARED.interrupts.slots[cases::IRQ_SOURCE - 1].state
607
            == slots::State::Free;
608
    assert (registers::read32(device.registers, enabled) & bit) == 0;
609
    sync::leave(guard);
610
    for byte in "smp IRQ transfer, rearm, and drop passed\n" {
611
        put(byte);
612
    }
613
}
614
615
/// Keep a level source asserted across full queues and repeated driver
616
/// progress.
617
unsafe fn interruptStorm() {
618
    let guard = sync::enter();
619
    let mut checker = try! domains::get(&state::SHARED.domains, CHECKER);
620
    let handle = try! interrupts::seed(
621
        &mut state::SHARED.interrupts, &mut checker.memory.table, UART_IRQ
622
    );
623
    let device = try! plic::mapping(&boot::PLATFORM) else panic "interruptStorm: missing PLIC";
624
    let enabled = PLIC_ENABLES + device.context as u64 * PLIC_ENABLE_STRIDE;
625
    sync::leave(guard);
626
    for i in 0 .. events::ORDINARY {
627
        let filling = sync::enter();
628
        assert try! domains::serve(&mut state::SHARED.domains, CHECKER, domains::RingRequest::Send(
629
            events::Event {
630
                kind: events::WAKEUP, reserved: 0, code: i, value: 0
631
            }
632
        ));
633
        sync::leave(filling);
634
    }
635
    let arming = sync::enter();
636
    let ready = plic::service(
637
        &mut state::SHARED.interrupts,
638
        &mut state::SHARED.domains,
639
        &state::SHARED.budgets,
640
        dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0))
641
    );
642
    uartInterrupt(2);
643
    sync::leave(arming);
644
    let mut pending = false;
645
    for attempt in 0 .. IRQ_PENDING_ATTEMPTS {
646
        let checking = sync::enter();
647
        set pending =
648
            state::SHARED.interrupts.records[UART_IRQ - 1].state
649
                == interrupts::State::Pending;
650
        if pending {
651
            assert (registers::read32(
652
                device.registers, enabled
653
            ) & (1 << UART_IRQ)) == 0;
654
            let queue = state::SHARED.domains.events.queues[CHECKER.index];
655
            assert queue.tail - queue.head == events::ORDINARY;
656
        }
657
        sync::leave(checking);
658
        if pending {
659
            break;
660
        }
661
    }
662
    assert pending;
663
    for i in 0 .. events::ORDINARY {
664
        let draining = sync::enter();
665
        let mut receiver = try! domains::get(&state::SHARED.domains, CHECKER);
666
        let event = try! events::pop(&mut receiver.memory.ring)
667
            else panic "interruptStorm: missing saturation event";
668
        assert event.kind == events::WAKEUP and event.code == i;
669
        sync::leave(draining);
670
    }
671
    let posting = sync::enter();
672
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
673
    try! timers::request(
674
        &mut state::SHARED.timers,
675
        &state::SHARED.domains,
676
        CHECKER,
677
        0,
678
        clock,
679
        IRQ_TIMEOUT,
680
        cases::STORM_TOKEN
681
    );
682
    let request = try! dispatch::request(
683
        &mut state::SHARED.remote, 0, remote::Action::Reschedule
684
    );
685
    sync::leave(posting);
686
    let acknowledged = try! dispatch::awaitRequest(0, request);
687
    assert acknowledged == 0;
688
    let mut received: u32 = 0;
689
    let mut timed = false;
690
    for attempt in 0 .. IRQ_EVENTS {
691
        let consuming = sync::enter();
692
        let mut receiver = try! domains::get(&state::SHARED.domains, CHECKER);
693
        if let event = try! events::pop(&mut receiver.memory.ring) {
694
            if event.kind == events::INTERRUPT {
695
                assert event.code == UART_IRQ;
696
                set received += 1;
697
            } else {
698
                assert event.kind == events::TIMEOUT
699
                    and event.code == cases::STORM_TOKEN;
700
                assert dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0))
701
                    >= clock + IRQ_TIMEOUT;
702
                set timed = true;
703
            }
704
        }
705
        let notified = plic::service(
706
            &mut state::SHARED.interrupts,
707
            &mut state::SHARED.domains,
708
            &state::SHARED.budgets,
709
            dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0))
710
        );
711
        sync::leave(consuming);
712
        if timed and received >= STORM_EVENTS {
713
            break;
714
        }
715
    }
716
    let stopping = sync::enter();
717
    uartInterrupt(0);
718
    let dropped = try! calls::invoke(
719
        CHECKER, abi::Operation::CapabilityDrop as u32, &[*handle, 0, 0, 0], 0
720
    );
721
    assert dropped == 0 and (registers::read32(
722
        device.registers, enabled
723
    ) & (1 << UART_IRQ)) == 0;
724
    sync::leave(stopping);
725
    assert timed and received >= STORM_EVENTS;
726
    for byte in "smp IRQ saturation and storm timeout passed\n" {
727
        put(byte);
728
    }
729
}
730
731
/// Resume an actual U-mode Wait call through an IPI to an idle remote hart.
732
unsafe fn idleWakeup() {
733
    let window = IDLE_WINDOW else panic "idleWakeup: hart one window required";
734
    let guard = sync::enter();
735
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
736
    let handle = try! calls::rebind(
737
        &mut state::SHARED,
738
        &mut TABLE,
739
        budgets::Binding {
740
            budget: window, domain: IDLE_HANDLE, context: IDLE_CONTEXT
741
        },
742
        clock
743
    );
744
    let request = try! dispatch::request(
745
        &mut state::SHARED.remote, 1, remote::Action::Reschedule
746
    );
747
    sync::leave(guard);
748
    let acknowledged = try! dispatch::awaitRequest(0, request);
749
    assert acknowledged == 0;
750
    let activated = try! calls::synchronized(
751
        CHECKER,
752
        abi::Operation::DomainActivate as u32,
753
        &[
754
            *RECEIVER,
755
            IDLE_STACK, 0, 0,
756
        ],
757
        clock
758
    );
759
    assert activated == 0 and waiting();
760
    let now = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
761
    let result = try! calls::synchronized(
762
        CHECKER,
763
        abi::Operation::EventWakeup as u32,
764
        &[
765
            *RECEIVER,
766
            cases::WAKE_TOKEN as u64, 0, 0,
767
        ],
768
        now
769
    );
770
    assert result == 0;
771
    let mut resumed = false;
772
    for attempt in 0 .. POLL_ATTEMPTS {
773
        if sync::loadAcquire(memory(WAKE_STATE) as *u64)
774
            == cases::WakeStage::WakeResumed as u64
775
        {
776
            set resumed = true;
777
            break;
778
        }
779
    }
780
    assert resumed;
781
    for byte in "smp idle-hart activation and wakeup passed\n" {
782
        put(byte);
783
    }
784
    remoteTimeout();
785
}
786
787
/// Deliver a timeout from hart zero to a waiting user context on idle hart one.
788
unsafe fn remoteTimeout() {
789
    let guard = sync::enter();
790
    let mut domain = try! domains::get(&state::SHARED.domains, IDLE_DOMAIN);
791
    let event = try! events::pop(&mut domain.memory.ring)
792
        else panic "remoteTimeout: missing wakeup";
793
    assert event.kind == events::WAKEUP and event.code == cases::WAKE_TOKEN;
794
    sync::storeRelease(
795
        memory(WAKE_STATE) as *mut u64, cases::WakeStage::AllowTimeout as u64
796
    );
797
    sync::leave(guard);
798
    assert waiting();
799
    let posting = sync::enter();
800
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
801
    try! timers::request(
802
        &mut state::SHARED.timers,
803
        &state::SHARED.domains,
804
        IDLE_DOMAIN,
805
        0,
806
        clock,
807
        REMOTE_TIMEOUT,
808
        cases::TIMEOUT_TOKEN
809
    );
810
    let request = try! dispatch::request(
811
        &mut state::SHARED.remote, 0, remote::Action::Reschedule
812
    );
813
    sync::leave(posting);
814
    let acknowledged = try! dispatch::awaitRequest(0, request);
815
    assert acknowledged == 0;
816
    let mut resumed = false;
817
    for attempt in 0 .. POLL_ATTEMPTS {
818
        if sync::loadAcquire(memory(WAKE_STATE) as *u64)
819
            == cases::WakeStage::TimeoutResumed as u64
820
        {
821
            set resumed = true;
822
            break;
823
        }
824
    }
825
    assert resumed;
826
    for byte in "smp remote timeout passed\n" {
827
        put(byte);
828
    }
829
}
830
831
/// Wake a parent waiting on hart one through terminal delivery on hart zero.
832
unsafe fn terminalWakeup() {
833
    let guard = sync::enter();
834
    let mut receiver = try! domains::get(&state::SHARED.domains, IDLE_DOMAIN);
835
    let timeout = try! events::pop(&mut receiver.memory.ring)
836
        else panic "terminalWakeup: missing idle timeout";
837
    assert timeout.kind == events::TIMEOUT
838
        and timeout.code == cases::TIMEOUT_TOKEN;
839
    sync::storeRelease(
840
        memory(WAKE_STATE) as *mut u64, cases::WakeStage::AllowTerminalWait as u64
841
    );
842
    sync::leave(guard);
843
    assert waiting();
844
    let stopping = sync::enter();
845
    let mut checker = try! domains::get(&state::SHARED.domains, CHECKER);
846
    let child = try! capability::install(&mut checker.memory.table, capability::Entry {
847
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::DESTROY)
848
    });
849
    let parent = try! calls::invoke(
850
        CHECKER, abi::Operation::DomainReparent as u32, &[*child, *RECEIVER, 0, 0], 0
851
    );
852
    assert parent == 0;
853
    let cancelled = try! dispatch::request(&mut state::SHARED.remote, 1, remote::Action::Budget(
854
        remote::Call {
855
            owner: USER,
856
            context: CONTEXTS[0],
857
            operation: abi::Operation::BudgetSplit as u32,
858
            arguments: [0; 4],
859
        }
860
    ));
861
    set state::SHARED.domains.contexts[CONTEXTS[0].index].pending = cancelled;
862
    let readiness = try! remote::reserve(&mut state::SHARED.remote, 0, 2);
863
    let batch = remote::publish(&mut state::SHARED.remote, readiness, remote::Action::Ready(
864
        CONTEXTS[1]
865
    ));
866
    dispatch::startReady(&mut state::SHARED.domains, CONTEXTS[1], &batch);
867
    let stopped = try! domains::terminateTree(
868
        &mut state::SHARED.domains, USER, events::CHILD_EXIT, 23
869
    );
870
    assert stopped == 0;
871
    assert state::SHARED.domains.records[DESCENDANT.index].state
872
        == domains::Lifecycle::Dead;
873
    // The descendant reported to USER, which dies with it, so its event
874
    // moves to USER's parent.
875
    assert state::SHARED.domains.events.terminals[DESCENDANT.index].receiver
876
        == IDLE_DOMAIN;
877
    let request = try! dispatch::request(
878
        &mut state::SHARED.remote, 0, remote::Action::Reschedule
879
    );
880
    sync::leave(stopping);
881
    let acknowledged = try! dispatch::awaitRequest(0, request);
882
    assert acknowledged == 0;
883
    let mut resumed = false;
884
    for attempt in 0 .. POLL_ATTEMPTS {
885
        if sync::loadAcquire(memory(WAKE_STATE) as *u64)
886
            == cases::WakeStage::TerminalResumed as u64
887
        {
888
            set resumed = true;
889
            break;
890
        }
891
    }
892
    assert resumed;
893
    let checking = sync::enter();
894
    assert state::SHARED.remote.slots[cancelled.index].state
895
        == slots::State::Free;
896
    assert state::SHARED.remote.slots[batch.ids[1].index].state
897
        == slots::State::Free;
898
    assert state::SHARED.domains.contexts[CONTEXTS[0].index].pending == nil;
899
    assert state::SHARED.domains.contexts[CONTEXTS[1].index].pending == nil;
900
    let mut domain = try! domains::get(&state::SHARED.domains, IDLE_DOMAIN);
901
    let mut seen: u32 = 0;
902
    for i in 0 .. 2 {
903
        let event = try! events::pop(&mut domain.memory.ring)
904
            else panic "terminalWakeup: missing remote exit";
905
        if event.value == abi::id(USER) {
906
            assert event.kind == events::CHILD_EXIT and event.code == 23;
907
            set seen |= 1;
908
        } else {
909
            assert event.kind == events::FAULT
910
                and event.code == abi::FAULT_DESTROYED
911
                and event.value == abi::id(DESCENDANT);
912
            set seen |= 2;
913
        }
914
    }
915
    assert seen == 3;
916
    sync::leave(checking);
917
    for byte in "smp cascade cancelled calls and woke remote parent\n" {
918
        put(byte);
919
    }
920
}
921
922
/// Stop a real page-clear continuation and recover its unpublished frames.
923
unsafe fn cancelAllocation() {
924
    sync::storeRelease(
925
        memory(WAKE_STATE) as *mut u64, cases::WakeStage::AllowAllocation as u64
926
    );
927
    let mut allocation: pages::Allocation = undefined;
928
    let mut request: ?abi::Ref = nil;
929
    for attempt in 0 .. POLL_ATTEMPTS {
930
        let guard = sync::enter();
931
        let index = IDLE_CONTEXT.index;
932
        let active = state::SHARED.lifecycle.active[index / fixture::CONTEXT_BITS];
933
        let mask = 1 as u32 << (index % fixture::CONTEXT_BITS);
934
        if (active & mask) <> 0 {
935
            match state::SHARED.lifecycle.records[index].allocation {
936
                case lifecycle::Allocation::Page(value) => {
937
                    set allocation = value;
938
                }
939
                else => panic "cancelAllocation: page allocation required",
940
            }
941
            assert allocation.owner == IDLE_DOMAIN and allocation.run.count
942
                == cases::ALLOCATION_PAGES;
943
            sync::leave(guard);
944
            let result = try! calls::synchronized(
945
                CHECKER,
946
                abi::Operation::DomainDestroy as u32,
947
                &[
948
                    *RECEIVER, 0, 0,
949
                    0,
950
                ],
951
                dispatch::clock()
952
            );
953
            assert result == 0;
954
            let checking = sync::enter();
955
            assert state::SHARED.domains.records[IDLE_DOMAIN.index].state
956
                == domains::Lifecycle::Dead;
957
            set request = try! dispatch::request(
958
                &mut state::SHARED.remote, 1, remote::Action::Reschedule
959
            );
960
            sync::leave(checking);
961
            break;
962
        }
963
        sync::leave(guard);
964
    }
965
    let stopped = request else panic "cancelAllocation: page call did not reserve frames";
966
    let acknowledged = try! dispatch::awaitRequest(0, stopped);
967
    assert acknowledged == 0;
968
    let snapshot = sync::enter();
969
    let context = try! domains::context(
970
        &state::SHARED.domains, IDLE_DOMAIN, IDLE_CONTEXT
971
    );
972
    for byte in "smp cancelled page status and progress: " {
973
        put(byte);
974
    }
975
    number(context.frame.status);
976
    put(32);
977
    number(sync::loadAcquire(memory(WAKE_STATE) as *u64));
978
    put(10);
979
    assert context.hart == nil
980
        and context.state == domains::ContextState::Stopped;
981
    assert (context.frame.status & fixture::MACHINE_PRIVILEGE)
982
        == fixture::MACHINE_PRIVILEGE;
983
    assert sync::loadAcquire(memory(WAKE_STATE) as *u64)
984
        == cases::WakeStage::BeforeAllocation as u64;
985
    sync::leave(snapshot);
986
    let mut reclaimed = false;
987
    for attempt in 0 .. RECLAIM_ATTEMPTS {
988
        let checking = sync::enter();
989
        set reclaimed = state::SHARED.pages.backings.pool.retiredCount == 0;
990
        if reclaimed {
991
            sync::leave(checking);
992
            break;
993
        }
994
        let progress = try! dispatch::request(
995
            &mut state::SHARED.remote, 0, remote::Action::Reschedule
996
        );
997
        sync::leave(checking);
998
        let result = try! dispatch::awaitRequest(0, progress);
999
        assert result == 0;
1000
    }
1001
    assert reclaimed;
1002
    let guard = sync::enter();
1003
    let active = state::SHARED.lifecycle.active[IDLE_CONTEXT.index / fixture::CONTEXT_BITS];
1004
    let mask = 1 as u32 << (IDLE_CONTEXT.index % fixture::CONTEXT_BITS);
1005
    assert (active & mask) == 0;
1006
    assert state::SHARED.pages.slots[allocation.page.index].state
1007
        == slots::State::Free;
1008
    let backing = allocation.backing.index;
1009
    assert state::SHARED.pages.backings.slots[backing].state
1010
        == slots::State::Free;
1011
    for i in allocation.run.first .. allocation.run.first + allocation.run.count {
1012
        assert frames::available(&state::SHARED.pages.backings.pool, i);
1013
    }
1014
    sync::leave(guard);
1015
    for byte in "smp cancelled preempted page allocation\n" {
1016
        put(byte);
1017
    }
1018
}
1019
1020
/// Wait for dispatch-driven teardown and verify all dead execution ownership is
1021
/// gone.
1022
unsafe fn reclaimedDomains() {
1023
    for attempt in 0 .. POLL_ATTEMPTS {
1024
        let guard = sync::enter();
1025
        let mut done = state::SHARED.pages.backings.pool.retiredCount == 0;
1026
        for owner in [USER, IDLE_DOMAIN, DESCENDANT] {
1027
            if slots::matches(
1028
                &state::SHARED.domains.slots[..], owner, slots::State::Live
1029
            ) {
1030
                set done = false;
1031
            }
1032
        }
1033
        if done {
1034
            assert state::SHARED.domains.dead == 0;
1035
            for owner in [USER, IDLE_DOMAIN, DESCENDANT] {
1036
                assert state::SHARED.domains.slots[owner.index].state
1037
                    == slots::State::Free;
1038
                assert state::SHARED.pages.backings.domains[owner.index] == 0;
1039
                for i in 0 .. state::SHARED.domains.contextSlots.len {
1040
                    if state::SHARED.domains.contextSlots[i].state
1041
                        == slots::State::Live
1042
                    {
1043
                        assert state::SHARED.domains.contexts[i].owner <> owner;
1044
                    }
1045
                }
1046
                for i in 0 .. state::SHARED.budgets.slots.len {
1047
                    if state::SHARED.budgets.slots[i].state
1048
                        == slots::State::Live
1049
                    {
1050
                        assert state::SHARED.budgets.windows[i].owner <> owner;
1051
                    }
1052
                }
1053
                for i in 0 .. state::SHARED.interrupts.slots.len {
1054
                    if state::SHARED.interrupts.slots[i].state
1055
                        == slots::State::Live
1056
                    {
1057
                        assert state::SHARED.interrupts.records[i].owner
1058
                            <> owner;
1059
                    }
1060
                }
1061
            }
1062
            sync::leave(guard);
1063
            for byte in "smp reaped dead domains and recovered ownership\n" {
1064
                put(byte);
1065
            }
1066
            return;
1067
        }
1068
        let progress = try! dispatch::request(
1069
            &mut state::SHARED.remote, 0, remote::Action::Reschedule
1070
        );
1071
        sync::leave(guard);
1072
        let result = try! dispatch::awaitRequest(0, progress);
1073
        assert result == 0;
1074
    }
1075
    panic "reclaimedDomains: domain reaper did not finish";
1076
}
1077
1078
/// Confirm the waiting context has released hart ownership before its wakeup.
1079
unsafe fn waiting() -> bool {
1080
    for attempt in 0 .. POLL_ATTEMPTS {
1081
        let guard = sync::enter();
1082
        let context = try! domains::context(
1083
            &state::SHARED.domains, IDLE_DOMAIN, IDLE_CONTEXT
1084
        );
1085
        let stopped = context.state == domains::ContextState::Waiting
1086
            and context.hart == nil;
1087
        sync::leave(guard);
1088
        if stopped {
1089
            return true;
1090
        }
1091
    }
1092
    return false;
1093
}
1094
1095
/// Print one full-width hexadecimal diagnostic value.
1096
fn number(value: u64) {
1097
    let mut text: [u8; 16] = [0; 16];
1098
    for byte in fmt::formatHexU64(value, &mut text[..]) {
1099
        put(byte);
1100
    }
1101
}