.githooks/
compiler/
kernel/
lib/
scripts/
seed/
sublime/
test/
boot/
bootstrap/
compiler/
cycles/
dispatch/
format/
host/
integration/
kernel/
loader/
manifests/
mmio/
native/
packages/
runtime/
scheduling/
shared/
slots/
smp/
kernel/
dispatchcheck.rad
39.6 KiB
invariants.rad
3.6 KiB
cases.rad
1.6 KiB
machine.ras
1.5 KiB
spin.rad
3.9 KiB
std/
sync/
termination/
tests/
trap/
verify/
Makefile
9.2 KiB
kernel.rad
776 B
runner.rad
11.6 KiB
vim/
.gitignore
333 B
.gitsigners
224 B
AGENTS.md
8.3 KiB
CONTRIBUTING
2.1 KiB
LICENSE
1.1 KiB
Makefile
2.5 KiB
README
3.4 KiB
STYLE
3.2 KiB
std.lib
2.1 KiB
std.lib.test
1.1 KiB
test/smp/kernel/dispatchcheck.rad
raw
| 1 | //! One domain executing separate user contexts on every online hart. |
| 2 | export mod inputs; |
| 3 | export mod cases; |
| 4 | export mod invariants; |
| 5 | |
| 6 | use std::arch::rv64; |
| 7 | use std::arch::rv64::shared; |
| 8 | use std::fmt; |
| 9 | use kernel::abi; |
| 10 | use kernel::boot; |
| 11 | use kernel::budgets; |
| 12 | use kernel::calls; |
| 13 | use kernel::capability; |
| 14 | use kernel::dispatch; |
| 15 | use kernel::domains; |
| 16 | use kernel::events; |
| 17 | use kernel::fixture; |
| 18 | use kernel::frames; |
| 19 | use kernel::instances; |
| 20 | use kernel::interrupts; |
| 21 | use kernel::lifecycle; |
| 22 | use kernel::limits; |
| 23 | use kernel::pages; |
| 24 | use kernel::plic; |
| 25 | use kernel::registers; |
| 26 | use kernel::registry; |
| 27 | use kernel::remote; |
| 28 | use kernel::slots; |
| 29 | use kernel::state; |
| 30 | use kernel::sync; |
| 31 | use kernel::timers; |
| 32 | |
| 33 | /// Delay before concurrent user windows, in timer ticks. |
| 34 | constant START_DELAY: u64 = 1000000; |
| 35 | |
| 36 | /// Base work and migration duration, scaled by the square of hart count. |
| 37 | constant WORK_TICKS: u64 = 10000000; |
| 38 | |
| 39 | /// Reserved duration for final machine checks. |
| 40 | constant CHECK_TICKS: u64 = 10000000000; |
| 41 | |
| 42 | /// Maximum bounded polls for remote progress. |
| 43 | constant POLL_ATTEMPTS: u32 = 100000; |
| 44 | |
| 45 | /// Maximum polls for a level interrupt to reach a saturated event queue. |
| 46 | constant IRQ_PENDING_ATTEMPTS: u32 = 10000; |
| 47 | /// Maximum consumed events while checking interrupt saturation and timeout. |
| 48 | constant IRQ_EVENTS: u32 = 1000; |
| 49 | /// Maximum maintenance requests after a cancelled page allocation. |
| 50 | constant RECLAIM_ATTEMPTS: u32 = 1000; |
| 51 | /// Level-triggered UART source asserted during the storm check. |
| 52 | constant UART_IRQ: u32 = 10; |
| 53 | /// Minimum repeated UART events needed to check progress during a storm. |
| 54 | constant STORM_EVENTS: u32 = 16; |
| 55 | /// Byte offset of the first PLIC context enable word. |
| 56 | constant PLIC_ENABLES: u64 = 0x2000; |
| 57 | /// Byte stride between PLIC context enable words. |
| 58 | constant PLIC_ENABLE_STRIDE: u64 = 0x80; |
| 59 | /// Timer ticks before the remote idle-context timeout. |
| 60 | constant REMOTE_TIMEOUT: u64 = 100000; |
| 61 | |
| 62 | /// Timeout while the fixture holds a level interrupt asserted. |
| 63 | constant IRQ_TIMEOUT: u64 = 2000000; |
| 64 | |
| 65 | // Setup initializes retained state for online harts before releasing READY. |
| 66 | // Other harts must acquire READY before they read this state. |
| 67 | /// Bootstrap resource authority. |
| 68 | unsafe static TABLE: capability::Table = undefined; |
| 69 | /// Shared user domain generation. |
| 70 | unsafe static USER: abi::Ref = undefined; |
| 71 | /// Pending domain in the user's creation subtree. |
| 72 | unsafe static DESCENDANT: abi::Ref = undefined; |
| 73 | /// Per-hart user execution contexts. |
| 74 | unsafe static CONTEXTS: [abi::Ref; limits::HARTS] = undefined; |
| 75 | /// Mapped progress counters in the user's private state. |
| 76 | unsafe static COUNTERS: u64 = 0; |
| 77 | /// Per-context migration observations in the user's private state. |
| 78 | unsafe static MIGRATIONS: u64 = 0; |
| 79 | /// Shared handshake for the independently activated context on hart one. |
| 80 | unsafe static WAKE_STATE: u64 = 0; |
| 81 | /// Bootstrap management capability for the pending idle-wakeup domain. |
| 82 | unsafe static IDLE_HANDLE: abi::Handle = undefined; |
| 83 | /// Context whose Wait call must release the remote hart. |
| 84 | unsafe static IDLE_CONTEXT: abi::Ref = undefined; |
| 85 | /// Stack Page handle in the idle-wakeup domain's table. |
| 86 | unsafe static IDLE_STACK: u64 = 0; |
| 87 | /// Generation of the idle-wakeup domain. |
| 88 | unsafe static IDLE_DOMAIN: abi::Ref = undefined; |
| 89 | /// Completion checker's live domain generation. |
| 90 | unsafe static CHECKER: abi::Ref = undefined; |
| 91 | /// Unbound window that hart zero retires before the checker starts. |
| 92 | unsafe static EXPIRED: abi::Handle = undefined; |
| 93 | /// Checker's authority to activate and notify the idle-wakeup domain. |
| 94 | unsafe static RECEIVER: abi::Handle = undefined; |
| 95 | /// Unbound authority reserved for the idle-wakeup check on hart one. |
| 96 | unsafe static IDLE_WINDOW: ?abi::Handle = nil; |
| 97 | /// Release/acquire barrier after fixture publication. |
| 98 | static READY: u64 = 0; |
| 99 | /// Validated physical memory mapping. |
| 100 | fn memory(address: u64) -> *mut u8; |
| 101 | /// Current kernel package-state table. |
| 102 | fn kernelGp() -> u64; |
| 103 | /// M-mode entry of the completion checker. |
| 104 | fn completion() -> u64; |
| 105 | /// Write one diagnostic byte to the fixture UART. |
| 106 | fn put(byte: u8); |
| 107 | /// Set the fixture UART interrupt-enable register. |
| 108 | fn uartInterrupt(value: u8); |
| 109 | /// Finish the machine test successfully. |
| 110 | fn finish(); |
| 111 | /// Sleep until this hart's software interrupt is pending. Interrupts stay |
| 112 | /// globally masked, so the interrupt wakes the hart without trapping. |
| 113 | fn park(); |
| 114 | |
| 115 | /// Publish fixture state on the last initialized hart and wait on every hart. |
| 116 | export unsafe fn start(last: bool) { |
| 117 | if last { |
| 118 | setup(); |
| 119 | sync::storeRelease(&mut READY, 1); |
| 120 | // Every online hart finished dispatch::initialize before arriving. |
| 121 | for hart in 0 .. limits::HARTS { |
| 122 | if hart <> dispatch::hart() and (boot::PLATFORM.harts & (1 << hart)) |
| 123 | <> 0 |
| 124 | { |
| 125 | dispatch::signal(hart); |
| 126 | } |
| 127 | } |
| 128 | } |
| 129 | while sync::loadAcquire(&READY) == 0 { |
| 130 | park(); |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | /// Create one context per online hart before any dispatch can access the |
| 135 | /// fixture. |
| 136 | unsafe fn setup() { |
| 137 | let authority = fixture::bootstrap(&mut TABLE); |
| 138 | let image = fixture::load(&mut TABLE, authority, &inputs::INPUT[..]); |
| 139 | let handle = try! domains::create( |
| 140 | &mut state::SHARED.domains, |
| 141 | &mut state::SHARED.pages.backings, |
| 142 | &state::SHARED.registry, |
| 143 | &mut TABLE, |
| 144 | authority, |
| 145 | image |
| 146 | ); |
| 147 | let cap = try! capability::get(&TABLE, handle); |
| 148 | set USER = cap.object; |
| 149 | createDescendant(); |
| 150 | let mut user = try! domains::get(&state::SHARED.domains, USER); |
| 151 | let self = try! capability::install(&mut user.memory.table, capability::Entry { |
| 152 | kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE) |
| 153 | }); |
| 154 | assert boot::PLATFORM.irqSources == 31; |
| 155 | let irq = try! interrupts::seed( |
| 156 | &mut state::SHARED.interrupts, &mut user.memory.table, cases::IRQ_SOURCE |
| 157 | ); |
| 158 | sync::storeRelease( |
| 159 | memory(dataAddress( |
| 160 | USER, &"spin::interrupt"[..] |
| 161 | )) as *mut u64, |
| 162 | *irq |
| 163 | ); |
| 164 | let target = try! registry::exported( |
| 165 | &state::SHARED.registry, user.image, &"spin::worker"[..] |
| 166 | ); |
| 167 | let case shared::Target::Function(entry) = target else panic "setup: worker entry"; |
| 168 | set COUNTERS = dataAddress(USER, &"spin::count"[..]); |
| 169 | set MIGRATIONS = dataAddress(USER, &"spin::migrations"[..]); |
| 170 | let notification = @sliceOf( |
| 171 | memory(dataAddress( |
| 172 | USER, &"spin::notification"[..] |
| 173 | )), |
| 174 | 8 |
| 175 | ); |
| 176 | for i in 0 .. fixture::WORD_BYTES { |
| 177 | set notification[i] = (*self >> (i as u64 * 8)) as u8; |
| 178 | } |
| 179 | for hart in 0 .. limits::HARTS { |
| 180 | if (boot::PLATFORM.harts & (1 << hart)) == 0 { |
| 181 | continue; |
| 182 | } |
| 183 | let moved = fixture::stack( |
| 184 | &mut TABLE, authority, &mut user.memory.table |
| 185 | ); |
| 186 | if hart == 0 { |
| 187 | try! fixture::activate(&TABLE, handle, moved, abi::Handle(0), 0); |
| 188 | set CONTEXTS[hart] = user.initial; |
| 189 | } else { |
| 190 | set CONTEXTS[hart] = |
| 191 | try! domains::contextCreate( |
| 192 | &mut state::SHARED.domains, |
| 193 | &mut state::SHARED.pages, |
| 194 | &state::SHARED.registry, |
| 195 | USER, |
| 196 | abi::ContextStart { |
| 197 | entry, stack: *moved, args: hart as u64, size: 8 |
| 198 | }, |
| 199 | &mut user.memory.table |
| 200 | ); |
| 201 | } |
| 202 | } |
| 203 | let checkerHandle = try! domains::create( |
| 204 | &mut state::SHARED.domains, |
| 205 | &mut state::SHARED.pages.backings, |
| 206 | &state::SHARED.registry, |
| 207 | &mut TABLE, |
| 208 | authority, |
| 209 | image |
| 210 | ); |
| 211 | let checkerCap = try! capability::get(&TABLE, checkerHandle); |
| 212 | let mut checker = try! domains::get( |
| 213 | &state::SHARED.domains, checkerCap.object |
| 214 | ); |
| 215 | set CHECKER = checkerCap.object; |
| 216 | fixture::machine(checkerCap.object, completion(), kernelGp()); |
| 217 | idleSetup(authority, image); |
| 218 | let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)); |
| 219 | let start = clock + START_DELAY; |
| 220 | let mut harts: u64 = 0; |
| 221 | for hart in 0 .. limits::HARTS { |
| 222 | if (boot::PLATFORM.harts & (1 << hart)) <> 0 { |
| 223 | set harts += 1; |
| 224 | } |
| 225 | } |
| 226 | let end = start + WORK_TICKS * harts * harts; |
| 227 | let migrationEnd = end + WORK_TICKS * harts * harts; |
| 228 | let windowBase = dataAddress(USER, &"spin::windows"[..]); |
| 229 | for hart in 0 .. limits::HARTS { |
| 230 | if (boot::PLATFORM.harts & (1 << hart)) == 0 { |
| 231 | continue; |
| 232 | } |
| 233 | let begin = clock if hart == 0 else start; |
| 234 | let mut budget = try! budgets::seed( |
| 235 | &mut state::SHARED.budgets, &mut TABLE, hart, begin, 0xffffffffffffffff |
| 236 | ); |
| 237 | if hart == 0 { |
| 238 | let next = try! budgets::split( |
| 239 | &mut state::SHARED.budgets, &mut TABLE, budget, start, clock |
| 240 | ); |
| 241 | set EXPIRED = try! budgets::transfer( |
| 242 | &mut state::SHARED.budgets, |
| 243 | &mut TABLE, |
| 244 | &mut checker.memory.table, |
| 245 | budget, |
| 246 | budgets::DEFAULT_RIGHTS as u64, |
| 247 | clock |
| 248 | ); |
| 249 | set budget = next; |
| 250 | } |
| 251 | let final = try! budgets::split( |
| 252 | &mut state::SHARED.budgets, &mut TABLE, budget, end, clock |
| 253 | ); |
| 254 | let checking = try! budgets::split( |
| 255 | &mut state::SHARED.budgets, &mut TABLE, final, migrationEnd, clock |
| 256 | ); |
| 257 | let future = try! budgets::split( |
| 258 | &mut state::SHARED.budgets, &mut TABLE, checking, migrationEnd + CHECK_TICKS, clock |
| 259 | ); |
| 260 | let transferred = try! budgets::transfer( |
| 261 | &mut state::SHARED.budgets, |
| 262 | &mut TABLE, |
| 263 | &mut user.memory.table, |
| 264 | future, |
| 265 | budgets::DEFAULT_RIGHTS as u64, |
| 266 | clock |
| 267 | ); |
| 268 | let caller = (hart as u64 + harts - 1) % harts; |
| 269 | let output = @sliceOf(memory(windowBase + caller * 8), 8); |
| 270 | for i in 0 .. fixture::WORD_BYTES { |
| 271 | set output[i] = (*transferred >> (i as u64 * 8)) as u8; |
| 272 | } |
| 273 | let bound = try! calls::rebind( |
| 274 | &mut state::SHARED, |
| 275 | &mut TABLE, |
| 276 | budgets::Binding { |
| 277 | budget, domain: handle, context: CONTEXTS[hart] |
| 278 | }, |
| 279 | clock |
| 280 | ); |
| 281 | let rotated = ((hart as u64 + 1) % harts) as u32; |
| 282 | let moved = try! calls::rebind( |
| 283 | &mut state::SHARED, |
| 284 | &mut TABLE, |
| 285 | budgets::Binding { |
| 286 | budget: final, domain: handle, context: CONTEXTS[rotated] |
| 287 | }, |
| 288 | clock |
| 289 | ); |
| 290 | if hart == 0 { |
| 291 | let bound = try! calls::rebind( |
| 292 | &mut state::SHARED, |
| 293 | &mut TABLE, |
| 294 | budgets::Binding { |
| 295 | budget: checking, domain: checkerHandle, context: checker.initial |
| 296 | }, |
| 297 | clock |
| 298 | ); |
| 299 | } |
| 300 | if hart == 1 { |
| 301 | set IDLE_WINDOW = checking; |
| 302 | } |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | /// Create an actual pending descendant before secondary harts enter dispatch. |
| 307 | unsafe fn createDescendant() { |
| 308 | let mut user = try! domains::get(&state::SHARED.domains, USER); |
| 309 | let root = user.image; |
| 310 | let authority = try! capability::install(&mut user.memory.table, capability::Entry { |
| 311 | kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::CREATE) |
| 312 | }); |
| 313 | let image = try! registry::install( |
| 314 | &state::SHARED.registry, &mut user.memory.table, root |
| 315 | ); |
| 316 | let handle = try! domains::create( |
| 317 | &mut state::SHARED.domains, |
| 318 | &mut state::SHARED.pages.backings, |
| 319 | &state::SHARED.registry, |
| 320 | &mut user.memory.table, |
| 321 | authority, |
| 322 | image |
| 323 | ); |
| 324 | set DESCENDANT = (try! capability::get(&user.memory.table, handle)).object; |
| 325 | } |
| 326 | |
| 327 | /// Resolve one exported private-data address in the fixture's user instance. |
| 328 | unsafe fn dataAddress(owner: abi::Ref, name: &[u8]) -> u64 { |
| 329 | let user = try! domains::get(&state::SHARED.domains, owner); |
| 330 | let target = try! registry::exported( |
| 331 | &state::SHARED.registry, user.image, name |
| 332 | ); |
| 333 | let case shared::Target::Data(data) = target else panic "dataAddress: fixture data export"; |
| 334 | return instances::base( |
| 335 | &state::SHARED.domains.tables[owner.index], data.slot |
| 336 | ) + data.offset as u64; |
| 337 | } |
| 338 | |
| 339 | /// Prepare a pending domain and its stack for activation on an idle hart. |
| 340 | unsafe fn idleSetup(authority: abi::Handle, image: abi::Handle) { |
| 341 | set IDLE_HANDLE = try! domains::create( |
| 342 | &mut state::SHARED.domains, |
| 343 | &mut state::SHARED.pages.backings, |
| 344 | &state::SHARED.registry, |
| 345 | &mut TABLE, |
| 346 | authority, |
| 347 | image |
| 348 | ); |
| 349 | let permit = try! capability::get(&TABLE, IDLE_HANDLE); |
| 350 | set IDLE_DOMAIN = permit.object; |
| 351 | let mut domain = try! domains::get(&state::SHARED.domains, IDLE_DOMAIN); |
| 352 | let allocation = try! capability::install(&mut domain.memory.table, capability::Entry { |
| 353 | kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::ALLOCATE) |
| 354 | }); |
| 355 | set IDLE_CONTEXT = domain.initial; |
| 356 | set WAKE_STATE = dataAddress(IDLE_DOMAIN, &"spin::wakeState"[..]); |
| 357 | sync::storeRelease( |
| 358 | memory(dataAddress( |
| 359 | IDLE_DOMAIN, &"spin::idle"[..] |
| 360 | )) as *mut u64, |
| 361 | 1 |
| 362 | ); |
| 363 | let moved = fixture::stack( |
| 364 | &mut TABLE, authority, &mut domain.memory.table |
| 365 | ); |
| 366 | set IDLE_STACK = *moved; |
| 367 | let mut checker = try! domains::get(&state::SHARED.domains, CHECKER); |
| 368 | set RECEIVER = try! capability::install(&mut checker.memory.table, capability::Entry { |
| 369 | kind: abi::Kind::Domain, |
| 370 | object: IDLE_DOMAIN, |
| 371 | rights: abi::Rights(abi::WAKE | abi::EXECUTE | abi::DESTROY), |
| 372 | }); |
| 373 | } |
| 374 | |
| 375 | /// Check shared progress and context isolation after the user windows finish. |
| 376 | /// Only the scheduled checker runs these phases. Kernel metadata is locked by |
| 377 | /// each phase; user progress words use acquire loads. |
| 378 | export unsafe fn verify() { |
| 379 | checkUserContexts(); |
| 380 | checkAcknowledgements(); |
| 381 | expiredCall(); |
| 382 | if (boot::PLATFORM.harts & 2) <> 0 { |
| 383 | idleWakeup(); |
| 384 | } |
| 385 | interruptOwnership(); |
| 386 | interruptStorm(); |
| 387 | if (boot::PLATFORM.harts & 2) <> 0 { |
| 388 | terminalWakeup(); |
| 389 | cancelAllocation(); |
| 390 | reclaimedDomains(); |
| 391 | } |
| 392 | invariants::check(&TABLE); |
| 393 | for byte in "smp final ownership and interval invariants passed\n" { |
| 394 | put(byte); |
| 395 | } |
| 396 | finish(); |
| 397 | } |
| 398 | |
| 399 | /// Check saved contexts, migration destinations, and all per-hart progress. |
| 400 | unsafe fn checkUserContexts() { |
| 401 | let guard = sync::enter(); |
| 402 | let user = try! domains::get(&state::SHARED.domains, USER); |
| 403 | let mut counts: [u64; limits::HARTS] = [0; limits::HARTS]; |
| 404 | let mut notifications: u32 = 0; |
| 405 | for hart in 0 .. limits::HARTS { |
| 406 | if (boot::PLATFORM.harts & (1 << hart)) <> 0 { |
| 407 | set notifications += 1; |
| 408 | } |
| 409 | } |
| 410 | for hart in 0 .. limits::HARTS { |
| 411 | if (boot::PLATFORM.harts & (1 << hart)) == 0 { |
| 412 | continue; |
| 413 | } |
| 414 | let value = sync::loadAcquire( |
| 415 | memory( |
| 416 | COUNTERS |
| 417 | + hart as u64 * 8 |
| 418 | ) as *u64 |
| 419 | ); |
| 420 | set counts[hart] = value; |
| 421 | let context = try! domains::context( |
| 422 | &state::SHARED.domains, USER, CONTEXTS[hart] |
| 423 | ); |
| 424 | assert context.owner == USER; |
| 425 | let gp = context.frame.registers[*rv64::GP as u32]; |
| 426 | let sp = context.frame.registers[*rv64::SP as u32]; |
| 427 | let mode = context.frame.status & fixture::MACHINE_PRIVILEGE; |
| 428 | assert mode == 0 or mode == fixture::MACHINE_PRIVILEGE; |
| 429 | if mode == fixture::MACHINE_PRIVILEGE { |
| 430 | assert gp == kernelGp(); |
| 431 | assert sp >= context.kernelStack.start |
| 432 | and sp <= context.kernelStack.end; |
| 433 | } else { |
| 434 | assert gp |
| 435 | == instances::address(&state::SHARED.domains.tables[USER.index]); |
| 436 | } |
| 437 | let destination = (hart + notifications - 1) % notifications; |
| 438 | assert context.hart == nil or context.hart == destination; |
| 439 | assert sync::loadAcquire( |
| 440 | memory( |
| 441 | MIGRATIONS |
| 442 | + hart as u64 * 8 |
| 443 | ) as *u64 |
| 444 | ) == destination as u64 |
| 445 | + 1; |
| 446 | assert context.pending == nil; |
| 447 | } |
| 448 | assert state::SHARED.domains.events.queues[USER.index].tail |
| 449 | == notifications + 1; |
| 450 | assert user.memory.ring.data[0].kind == events::INTERRUPT |
| 451 | and user.memory.ring.data[0].code == cases::IRQ_SOURCE; |
| 452 | sync::leave(guard); |
| 453 | for byte in "smp completed transactions:" { |
| 454 | put(byte); |
| 455 | } |
| 456 | for hart in 0 .. limits::HARTS { |
| 457 | if (boot::PLATFORM.harts & (1 << hart)) == 0 { |
| 458 | continue; |
| 459 | } |
| 460 | put(32); |
| 461 | number(counts[hart]); |
| 462 | } |
| 463 | put(10); |
| 464 | for hart in 0 .. limits::HARTS { |
| 465 | if (boot::PLATFORM.harts & (1 << hart)) <> 0 { |
| 466 | assert counts[hart] == cases::TRANSACTIONS as u64; |
| 467 | } |
| 468 | } |
| 469 | } |
| 470 | |
| 471 | /// Post one request per online hart and check acknowledgement ownership. |
| 472 | unsafe fn checkAcknowledgements() { |
| 473 | for target in 0 .. limits::HARTS { |
| 474 | if (boot::PLATFORM.harts & (1 << target)) == 0 { |
| 475 | continue; |
| 476 | } |
| 477 | let posting = sync::enter(); |
| 478 | let id = try! dispatch::request( |
| 479 | &mut state::SHARED.remote, target, remote::Action::Reschedule |
| 480 | ); |
| 481 | sync::leave(posting); |
| 482 | let mut acknowledged = false; |
| 483 | for attempt in 0 .. POLL_ATTEMPTS { |
| 484 | let guard = sync::enter(); |
| 485 | if try! remote::completed(&state::SHARED.remote, 0, id) { |
| 486 | try! remote::release(&mut state::SHARED.remote, 0, id); |
| 487 | set acknowledged = true; |
| 488 | } |
| 489 | sync::leave(guard); |
| 490 | if acknowledged { |
| 491 | break; |
| 492 | } |
| 493 | } |
| 494 | assert acknowledged; |
| 495 | } |
| 496 | for byte in "smp request acknowledgements passed\n" { |
| 497 | put(byte); |
| 498 | } |
| 499 | } |
| 500 | |
| 501 | /// Dispatch on the window's hart retires elapsed authority, so a later call |
| 502 | /// finds a stale handle. |
| 503 | unsafe fn expiredCall() { |
| 504 | let guard = sync::enter(); |
| 505 | let checker = try! domains::get(&state::SHARED.domains, CHECKER); |
| 506 | let mut retired = false; |
| 507 | try budgets::query( |
| 508 | &state::SHARED.budgets, &checker.memory.table, EXPIRED |
| 509 | ) catch error { |
| 510 | assert error == abi::Error::BadHandle; |
| 511 | set retired = true; |
| 512 | }; |
| 513 | assert retired; |
| 514 | sync::leave(guard); |
| 515 | let mut rejected = false; |
| 516 | try calls::synchronized( |
| 517 | CHECKER, abi::Operation::BudgetSplit as u32, &[*EXPIRED, 1, 0, 0], 0 |
| 518 | ) catch error { |
| 519 | assert error == abi::Error::BadHandle; |
| 520 | set rejected = true; |
| 521 | }; |
| 522 | assert rejected; |
| 523 | for byte in "smp resumed call rejected expired authority\n" { |
| 524 | put(byte); |
| 525 | } |
| 526 | } |
| 527 | |
| 528 | /// Transfer a queued IRQ, rearm after consumption, and mask it before |
| 529 | /// capability drop. |
| 530 | unsafe fn interruptOwnership() { |
| 531 | let guard = sync::enter(); |
| 532 | let mut user = try! domains::get(&state::SHARED.domains, USER); |
| 533 | let mut checker = try! domains::get(&state::SHARED.domains, CHECKER); |
| 534 | let device = try! plic::mapping(&boot::PLATFORM) else panic "interruptOwnership: missing PLIC"; |
| 535 | let enabled = PLIC_ENABLES + device.context as u64 * PLIC_ENABLE_STRIDE; |
| 536 | let bit = 1 as u32 << cases::IRQ_SOURCE; |
| 537 | assert (registers::read32(device.registers, enabled) & bit) == 0; |
| 538 | assert state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].state |
| 539 | == interrupts::State::Queued; |
| 540 | let original = |
| 541 | state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].handle; |
| 542 | let target = try! capability::install(&mut user.memory.table, capability::Entry { |
| 543 | kind: abi::Kind::Domain, object: CHECKER, rights: abi::Rights(0) |
| 544 | }); |
| 545 | let back = try! capability::install(&mut checker.memory.table, capability::Entry { |
| 546 | kind: abi::Kind::Domain, object: USER, rights: abi::Rights(0) |
| 547 | }); |
| 548 | let moved = try! calls::invoke( |
| 549 | USER, |
| 550 | abi::Operation::CapabilityTransfer as u32, |
| 551 | &[ |
| 552 | *original, *target, |
| 553 | (abi::READ | abi::TRANSFER) as u64, 0, |
| 554 | ], |
| 555 | 0 |
| 556 | ); |
| 557 | assert state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].owner |
| 558 | == CHECKER; |
| 559 | let first = plic::service( |
| 560 | &mut state::SHARED.interrupts, |
| 561 | &mut state::SHARED.domains, |
| 562 | &state::SHARED.budgets, |
| 563 | dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) |
| 564 | ); |
| 565 | assert state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].state |
| 566 | == interrupts::State::Queued; |
| 567 | assert (registers::read32(device.registers, enabled) & bit) == 0; |
| 568 | let event = try! events::pop(&mut checker.memory.ring) |
| 569 | else panic "interruptOwnership: missing transferred IRQ"; |
| 570 | assert event.kind == events::INTERRUPT and event.code == cases::IRQ_SOURCE; |
| 571 | let before = state::SHARED.domains.events.queues[USER.index].tail; |
| 572 | let returned = try! calls::invoke( |
| 573 | CHECKER, |
| 574 | abi::Operation::CapabilityTransfer as u32, |
| 575 | &[ |
| 576 | moved, *back, |
| 577 | (abi::READ | abi::TRANSFER) as u64, 0, |
| 578 | ], |
| 579 | 0 |
| 580 | ); |
| 581 | let second = plic::service( |
| 582 | &mut state::SHARED.interrupts, |
| 583 | &mut state::SHARED.domains, |
| 584 | &state::SHARED.budgets, |
| 585 | dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) |
| 586 | ); |
| 587 | assert state::SHARED.domains.events.queues[USER.index].tail == before; |
| 588 | assert (registers::read32(device.registers, enabled) & bit) == 0; |
| 589 | let consumed = try! events::pop(&mut user.memory.ring) |
| 590 | else panic "interruptOwnership: missing original IRQ"; |
| 591 | assert consumed.kind == events::INTERRUPT |
| 592 | and consumed.code == cases::IRQ_SOURCE; |
| 593 | let rearmed = plic::service( |
| 594 | &mut state::SHARED.interrupts, |
| 595 | &mut state::SHARED.domains, |
| 596 | &state::SHARED.budgets, |
| 597 | dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) |
| 598 | ); |
| 599 | assert state::SHARED.interrupts.records[cases::IRQ_SOURCE - 1].state |
| 600 | == interrupts::State::Idle; |
| 601 | assert (registers::read32(device.registers, enabled) & bit) <> 0; |
| 602 | let dropped = try! calls::invoke( |
| 603 | USER, abi::Operation::CapabilityDrop as u32, &[returned, 0, 0, 0], 0 |
| 604 | ); |
| 605 | assert dropped == 0 |
| 606 | and state::SHARED.interrupts.slots[cases::IRQ_SOURCE - 1].state |
| 607 | == slots::State::Free; |
| 608 | assert (registers::read32(device.registers, enabled) & bit) == 0; |
| 609 | sync::leave(guard); |
| 610 | for byte in "smp IRQ transfer, rearm, and drop passed\n" { |
| 611 | put(byte); |
| 612 | } |
| 613 | } |
| 614 | |
| 615 | /// Keep a level source asserted across full queues and repeated driver |
| 616 | /// progress. |
| 617 | unsafe fn interruptStorm() { |
| 618 | let guard = sync::enter(); |
| 619 | let mut checker = try! domains::get(&state::SHARED.domains, CHECKER); |
| 620 | let handle = try! interrupts::seed( |
| 621 | &mut state::SHARED.interrupts, &mut checker.memory.table, UART_IRQ |
| 622 | ); |
| 623 | let device = try! plic::mapping(&boot::PLATFORM) else panic "interruptStorm: missing PLIC"; |
| 624 | let enabled = PLIC_ENABLES + device.context as u64 * PLIC_ENABLE_STRIDE; |
| 625 | sync::leave(guard); |
| 626 | for i in 0 .. events::ORDINARY { |
| 627 | let filling = sync::enter(); |
| 628 | assert try! domains::serve(&mut state::SHARED.domains, CHECKER, domains::RingRequest::Send( |
| 629 | events::Event { |
| 630 | kind: events::WAKEUP, reserved: 0, code: i, value: 0 |
| 631 | } |
| 632 | )); |
| 633 | sync::leave(filling); |
| 634 | } |
| 635 | let arming = sync::enter(); |
| 636 | let ready = plic::service( |
| 637 | &mut state::SHARED.interrupts, |
| 638 | &mut state::SHARED.domains, |
| 639 | &state::SHARED.budgets, |
| 640 | dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) |
| 641 | ); |
| 642 | uartInterrupt(2); |
| 643 | sync::leave(arming); |
| 644 | let mut pending = false; |
| 645 | for attempt in 0 .. IRQ_PENDING_ATTEMPTS { |
| 646 | let checking = sync::enter(); |
| 647 | set pending = |
| 648 | state::SHARED.interrupts.records[UART_IRQ - 1].state |
| 649 | == interrupts::State::Pending; |
| 650 | if pending { |
| 651 | assert (registers::read32( |
| 652 | device.registers, enabled |
| 653 | ) & (1 << UART_IRQ)) == 0; |
| 654 | let queue = state::SHARED.domains.events.queues[CHECKER.index]; |
| 655 | assert queue.tail - queue.head == events::ORDINARY; |
| 656 | } |
| 657 | sync::leave(checking); |
| 658 | if pending { |
| 659 | break; |
| 660 | } |
| 661 | } |
| 662 | assert pending; |
| 663 | for i in 0 .. events::ORDINARY { |
| 664 | let draining = sync::enter(); |
| 665 | let mut receiver = try! domains::get(&state::SHARED.domains, CHECKER); |
| 666 | let event = try! events::pop(&mut receiver.memory.ring) |
| 667 | else panic "interruptStorm: missing saturation event"; |
| 668 | assert event.kind == events::WAKEUP and event.code == i; |
| 669 | sync::leave(draining); |
| 670 | } |
| 671 | let posting = sync::enter(); |
| 672 | let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)); |
| 673 | try! timers::request( |
| 674 | &mut state::SHARED.timers, |
| 675 | &state::SHARED.domains, |
| 676 | CHECKER, |
| 677 | 0, |
| 678 | clock, |
| 679 | IRQ_TIMEOUT, |
| 680 | cases::STORM_TOKEN |
| 681 | ); |
| 682 | let request = try! dispatch::request( |
| 683 | &mut state::SHARED.remote, 0, remote::Action::Reschedule |
| 684 | ); |
| 685 | sync::leave(posting); |
| 686 | let acknowledged = try! dispatch::awaitRequest(0, request); |
| 687 | assert acknowledged == 0; |
| 688 | let mut received: u32 = 0; |
| 689 | let mut timed = false; |
| 690 | for attempt in 0 .. IRQ_EVENTS { |
| 691 | let consuming = sync::enter(); |
| 692 | let mut receiver = try! domains::get(&state::SHARED.domains, CHECKER); |
| 693 | if let event = try! events::pop(&mut receiver.memory.ring) { |
| 694 | if event.kind == events::INTERRUPT { |
| 695 | assert event.code == UART_IRQ; |
| 696 | set received += 1; |
| 697 | } else { |
| 698 | assert event.kind == events::TIMEOUT |
| 699 | and event.code == cases::STORM_TOKEN; |
| 700 | assert dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) |
| 701 | >= clock + IRQ_TIMEOUT; |
| 702 | set timed = true; |
| 703 | } |
| 704 | } |
| 705 | let notified = plic::service( |
| 706 | &mut state::SHARED.interrupts, |
| 707 | &mut state::SHARED.domains, |
| 708 | &state::SHARED.budgets, |
| 709 | dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)) |
| 710 | ); |
| 711 | sync::leave(consuming); |
| 712 | if timed and received >= STORM_EVENTS { |
| 713 | break; |
| 714 | } |
| 715 | } |
| 716 | let stopping = sync::enter(); |
| 717 | uartInterrupt(0); |
| 718 | let dropped = try! calls::invoke( |
| 719 | CHECKER, abi::Operation::CapabilityDrop as u32, &[*handle, 0, 0, 0], 0 |
| 720 | ); |
| 721 | assert dropped == 0 and (registers::read32( |
| 722 | device.registers, enabled |
| 723 | ) & (1 << UART_IRQ)) == 0; |
| 724 | sync::leave(stopping); |
| 725 | assert timed and received >= STORM_EVENTS; |
| 726 | for byte in "smp IRQ saturation and storm timeout passed\n" { |
| 727 | put(byte); |
| 728 | } |
| 729 | } |
| 730 | |
| 731 | /// Resume an actual U-mode Wait call through an IPI to an idle remote hart. |
| 732 | unsafe fn idleWakeup() { |
| 733 | let window = IDLE_WINDOW else panic "idleWakeup: hart one window required"; |
| 734 | let guard = sync::enter(); |
| 735 | let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)); |
| 736 | let handle = try! calls::rebind( |
| 737 | &mut state::SHARED, |
| 738 | &mut TABLE, |
| 739 | budgets::Binding { |
| 740 | budget: window, domain: IDLE_HANDLE, context: IDLE_CONTEXT |
| 741 | }, |
| 742 | clock |
| 743 | ); |
| 744 | let request = try! dispatch::request( |
| 745 | &mut state::SHARED.remote, 1, remote::Action::Reschedule |
| 746 | ); |
| 747 | sync::leave(guard); |
| 748 | let acknowledged = try! dispatch::awaitRequest(0, request); |
| 749 | assert acknowledged == 0; |
| 750 | let activated = try! calls::synchronized( |
| 751 | CHECKER, |
| 752 | abi::Operation::DomainActivate as u32, |
| 753 | &[ |
| 754 | *RECEIVER, |
| 755 | IDLE_STACK, 0, 0, |
| 756 | ], |
| 757 | clock |
| 758 | ); |
| 759 | assert activated == 0 and waiting(); |
| 760 | let now = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)); |
| 761 | let result = try! calls::synchronized( |
| 762 | CHECKER, |
| 763 | abi::Operation::EventWakeup as u32, |
| 764 | &[ |
| 765 | *RECEIVER, |
| 766 | cases::WAKE_TOKEN as u64, 0, 0, |
| 767 | ], |
| 768 | now |
| 769 | ); |
| 770 | assert result == 0; |
| 771 | let mut resumed = false; |
| 772 | for attempt in 0 .. POLL_ATTEMPTS { |
| 773 | if sync::loadAcquire(memory(WAKE_STATE) as *u64) |
| 774 | == cases::WakeStage::WakeResumed as u64 |
| 775 | { |
| 776 | set resumed = true; |
| 777 | break; |
| 778 | } |
| 779 | } |
| 780 | assert resumed; |
| 781 | for byte in "smp idle-hart activation and wakeup passed\n" { |
| 782 | put(byte); |
| 783 | } |
| 784 | remoteTimeout(); |
| 785 | } |
| 786 | |
| 787 | /// Deliver a timeout from hart zero to a waiting user context on idle hart one. |
| 788 | unsafe fn remoteTimeout() { |
| 789 | let guard = sync::enter(); |
| 790 | let mut domain = try! domains::get(&state::SHARED.domains, IDLE_DOMAIN); |
| 791 | let event = try! events::pop(&mut domain.memory.ring) |
| 792 | else panic "remoteTimeout: missing wakeup"; |
| 793 | assert event.kind == events::WAKEUP and event.code == cases::WAKE_TOKEN; |
| 794 | sync::storeRelease( |
| 795 | memory(WAKE_STATE) as *mut u64, cases::WakeStage::AllowTimeout as u64 |
| 796 | ); |
| 797 | sync::leave(guard); |
| 798 | assert waiting(); |
| 799 | let posting = sync::enter(); |
| 800 | let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0)); |
| 801 | try! timers::request( |
| 802 | &mut state::SHARED.timers, |
| 803 | &state::SHARED.domains, |
| 804 | IDLE_DOMAIN, |
| 805 | 0, |
| 806 | clock, |
| 807 | REMOTE_TIMEOUT, |
| 808 | cases::TIMEOUT_TOKEN |
| 809 | ); |
| 810 | let request = try! dispatch::request( |
| 811 | &mut state::SHARED.remote, 0, remote::Action::Reschedule |
| 812 | ); |
| 813 | sync::leave(posting); |
| 814 | let acknowledged = try! dispatch::awaitRequest(0, request); |
| 815 | assert acknowledged == 0; |
| 816 | let mut resumed = false; |
| 817 | for attempt in 0 .. POLL_ATTEMPTS { |
| 818 | if sync::loadAcquire(memory(WAKE_STATE) as *u64) |
| 819 | == cases::WakeStage::TimeoutResumed as u64 |
| 820 | { |
| 821 | set resumed = true; |
| 822 | break; |
| 823 | } |
| 824 | } |
| 825 | assert resumed; |
| 826 | for byte in "smp remote timeout passed\n" { |
| 827 | put(byte); |
| 828 | } |
| 829 | } |
| 830 | |
| 831 | /// Wake a parent waiting on hart one through terminal delivery on hart zero. |
| 832 | unsafe fn terminalWakeup() { |
| 833 | let guard = sync::enter(); |
| 834 | let mut receiver = try! domains::get(&state::SHARED.domains, IDLE_DOMAIN); |
| 835 | let timeout = try! events::pop(&mut receiver.memory.ring) |
| 836 | else panic "terminalWakeup: missing idle timeout"; |
| 837 | assert timeout.kind == events::TIMEOUT |
| 838 | and timeout.code == cases::TIMEOUT_TOKEN; |
| 839 | sync::storeRelease( |
| 840 | memory(WAKE_STATE) as *mut u64, cases::WakeStage::AllowTerminalWait as u64 |
| 841 | ); |
| 842 | sync::leave(guard); |
| 843 | assert waiting(); |
| 844 | let stopping = sync::enter(); |
| 845 | let mut checker = try! domains::get(&state::SHARED.domains, CHECKER); |
| 846 | let child = try! capability::install(&mut checker.memory.table, capability::Entry { |
| 847 | kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::DESTROY) |
| 848 | }); |
| 849 | let parent = try! calls::invoke( |
| 850 | CHECKER, abi::Operation::DomainReparent as u32, &[*child, *RECEIVER, 0, 0], 0 |
| 851 | ); |
| 852 | assert parent == 0; |
| 853 | let cancelled = try! dispatch::request(&mut state::SHARED.remote, 1, remote::Action::Budget( |
| 854 | remote::Call { |
| 855 | owner: USER, |
| 856 | context: CONTEXTS[0], |
| 857 | operation: abi::Operation::BudgetSplit as u32, |
| 858 | arguments: [0; 4], |
| 859 | } |
| 860 | )); |
| 861 | set state::SHARED.domains.contexts[CONTEXTS[0].index].pending = cancelled; |
| 862 | let readiness = try! remote::reserve(&mut state::SHARED.remote, 0, 2); |
| 863 | let batch = remote::publish(&mut state::SHARED.remote, readiness, remote::Action::Ready( |
| 864 | CONTEXTS[1] |
| 865 | )); |
| 866 | dispatch::startReady(&mut state::SHARED.domains, CONTEXTS[1], &batch); |
| 867 | let stopped = try! domains::terminateTree( |
| 868 | &mut state::SHARED.domains, USER, events::CHILD_EXIT, 23 |
| 869 | ); |
| 870 | assert stopped == 0; |
| 871 | assert state::SHARED.domains.records[DESCENDANT.index].state |
| 872 | == domains::Lifecycle::Dead; |
| 873 | // The descendant reported to USER, which dies with it, so its event |
| 874 | // moves to USER's parent. |
| 875 | assert state::SHARED.domains.events.terminals[DESCENDANT.index].receiver |
| 876 | == IDLE_DOMAIN; |
| 877 | let request = try! dispatch::request( |
| 878 | &mut state::SHARED.remote, 0, remote::Action::Reschedule |
| 879 | ); |
| 880 | sync::leave(stopping); |
| 881 | let acknowledged = try! dispatch::awaitRequest(0, request); |
| 882 | assert acknowledged == 0; |
| 883 | let mut resumed = false; |
| 884 | for attempt in 0 .. POLL_ATTEMPTS { |
| 885 | if sync::loadAcquire(memory(WAKE_STATE) as *u64) |
| 886 | == cases::WakeStage::TerminalResumed as u64 |
| 887 | { |
| 888 | set resumed = true; |
| 889 | break; |
| 890 | } |
| 891 | } |
| 892 | assert resumed; |
| 893 | let checking = sync::enter(); |
| 894 | assert state::SHARED.remote.slots[cancelled.index].state |
| 895 | == slots::State::Free; |
| 896 | assert state::SHARED.remote.slots[batch.ids[1].index].state |
| 897 | == slots::State::Free; |
| 898 | assert state::SHARED.domains.contexts[CONTEXTS[0].index].pending == nil; |
| 899 | assert state::SHARED.domains.contexts[CONTEXTS[1].index].pending == nil; |
| 900 | let mut domain = try! domains::get(&state::SHARED.domains, IDLE_DOMAIN); |
| 901 | let mut seen: u32 = 0; |
| 902 | for i in 0 .. 2 { |
| 903 | let event = try! events::pop(&mut domain.memory.ring) |
| 904 | else panic "terminalWakeup: missing remote exit"; |
| 905 | if event.value == abi::id(USER) { |
| 906 | assert event.kind == events::CHILD_EXIT and event.code == 23; |
| 907 | set seen |= 1; |
| 908 | } else { |
| 909 | assert event.kind == events::FAULT |
| 910 | and event.code == abi::FAULT_DESTROYED |
| 911 | and event.value == abi::id(DESCENDANT); |
| 912 | set seen |= 2; |
| 913 | } |
| 914 | } |
| 915 | assert seen == 3; |
| 916 | sync::leave(checking); |
| 917 | for byte in "smp cascade cancelled calls and woke remote parent\n" { |
| 918 | put(byte); |
| 919 | } |
| 920 | } |
| 921 | |
| 922 | /// Stop a real page-clear continuation and recover its unpublished frames. |
| 923 | unsafe fn cancelAllocation() { |
| 924 | sync::storeRelease( |
| 925 | memory(WAKE_STATE) as *mut u64, cases::WakeStage::AllowAllocation as u64 |
| 926 | ); |
| 927 | let mut allocation: pages::Allocation = undefined; |
| 928 | let mut request: ?abi::Ref = nil; |
| 929 | for attempt in 0 .. POLL_ATTEMPTS { |
| 930 | let guard = sync::enter(); |
| 931 | let index = IDLE_CONTEXT.index; |
| 932 | let active = state::SHARED.lifecycle.active[index / fixture::CONTEXT_BITS]; |
| 933 | let mask = 1 as u32 << (index % fixture::CONTEXT_BITS); |
| 934 | if (active & mask) <> 0 { |
| 935 | match state::SHARED.lifecycle.records[index].allocation { |
| 936 | case lifecycle::Allocation::Page(value) => { |
| 937 | set allocation = value; |
| 938 | } |
| 939 | else => panic "cancelAllocation: page allocation required", |
| 940 | } |
| 941 | assert allocation.owner == IDLE_DOMAIN and allocation.run.count |
| 942 | == cases::ALLOCATION_PAGES; |
| 943 | sync::leave(guard); |
| 944 | let result = try! calls::synchronized( |
| 945 | CHECKER, |
| 946 | abi::Operation::DomainDestroy as u32, |
| 947 | &[ |
| 948 | *RECEIVER, 0, 0, |
| 949 | 0, |
| 950 | ], |
| 951 | dispatch::clock() |
| 952 | ); |
| 953 | assert result == 0; |
| 954 | let checking = sync::enter(); |
| 955 | assert state::SHARED.domains.records[IDLE_DOMAIN.index].state |
| 956 | == domains::Lifecycle::Dead; |
| 957 | set request = try! dispatch::request( |
| 958 | &mut state::SHARED.remote, 1, remote::Action::Reschedule |
| 959 | ); |
| 960 | sync::leave(checking); |
| 961 | break; |
| 962 | } |
| 963 | sync::leave(guard); |
| 964 | } |
| 965 | let stopped = request else panic "cancelAllocation: page call did not reserve frames"; |
| 966 | let acknowledged = try! dispatch::awaitRequest(0, stopped); |
| 967 | assert acknowledged == 0; |
| 968 | let snapshot = sync::enter(); |
| 969 | let context = try! domains::context( |
| 970 | &state::SHARED.domains, IDLE_DOMAIN, IDLE_CONTEXT |
| 971 | ); |
| 972 | for byte in "smp cancelled page status and progress: " { |
| 973 | put(byte); |
| 974 | } |
| 975 | number(context.frame.status); |
| 976 | put(32); |
| 977 | number(sync::loadAcquire(memory(WAKE_STATE) as *u64)); |
| 978 | put(10); |
| 979 | assert context.hart == nil |
| 980 | and context.state == domains::ContextState::Stopped; |
| 981 | assert (context.frame.status & fixture::MACHINE_PRIVILEGE) |
| 982 | == fixture::MACHINE_PRIVILEGE; |
| 983 | assert sync::loadAcquire(memory(WAKE_STATE) as *u64) |
| 984 | == cases::WakeStage::BeforeAllocation as u64; |
| 985 | sync::leave(snapshot); |
| 986 | let mut reclaimed = false; |
| 987 | for attempt in 0 .. RECLAIM_ATTEMPTS { |
| 988 | let checking = sync::enter(); |
| 989 | set reclaimed = state::SHARED.pages.backings.pool.retiredCount == 0; |
| 990 | if reclaimed { |
| 991 | sync::leave(checking); |
| 992 | break; |
| 993 | } |
| 994 | let progress = try! dispatch::request( |
| 995 | &mut state::SHARED.remote, 0, remote::Action::Reschedule |
| 996 | ); |
| 997 | sync::leave(checking); |
| 998 | let result = try! dispatch::awaitRequest(0, progress); |
| 999 | assert result == 0; |
| 1000 | } |
| 1001 | assert reclaimed; |
| 1002 | let guard = sync::enter(); |
| 1003 | let active = state::SHARED.lifecycle.active[IDLE_CONTEXT.index / fixture::CONTEXT_BITS]; |
| 1004 | let mask = 1 as u32 << (IDLE_CONTEXT.index % fixture::CONTEXT_BITS); |
| 1005 | assert (active & mask) == 0; |
| 1006 | assert state::SHARED.pages.slots[allocation.page.index].state |
| 1007 | == slots::State::Free; |
| 1008 | let backing = allocation.backing.index; |
| 1009 | assert state::SHARED.pages.backings.slots[backing].state |
| 1010 | == slots::State::Free; |
| 1011 | for i in allocation.run.first .. allocation.run.first + allocation.run.count { |
| 1012 | assert frames::available(&state::SHARED.pages.backings.pool, i); |
| 1013 | } |
| 1014 | sync::leave(guard); |
| 1015 | for byte in "smp cancelled preempted page allocation\n" { |
| 1016 | put(byte); |
| 1017 | } |
| 1018 | } |
| 1019 | |
| 1020 | /// Wait for dispatch-driven teardown and verify all dead execution ownership is |
| 1021 | /// gone. |
| 1022 | unsafe fn reclaimedDomains() { |
| 1023 | for attempt in 0 .. POLL_ATTEMPTS { |
| 1024 | let guard = sync::enter(); |
| 1025 | let mut done = state::SHARED.pages.backings.pool.retiredCount == 0; |
| 1026 | for owner in [USER, IDLE_DOMAIN, DESCENDANT] { |
| 1027 | if slots::matches( |
| 1028 | &state::SHARED.domains.slots[..], owner, slots::State::Live |
| 1029 | ) { |
| 1030 | set done = false; |
| 1031 | } |
| 1032 | } |
| 1033 | if done { |
| 1034 | assert state::SHARED.domains.dead == 0; |
| 1035 | for owner in [USER, IDLE_DOMAIN, DESCENDANT] { |
| 1036 | assert state::SHARED.domains.slots[owner.index].state |
| 1037 | == slots::State::Free; |
| 1038 | assert state::SHARED.pages.backings.domains[owner.index] == 0; |
| 1039 | for i in 0 .. state::SHARED.domains.contextSlots.len { |
| 1040 | if state::SHARED.domains.contextSlots[i].state |
| 1041 | == slots::State::Live |
| 1042 | { |
| 1043 | assert state::SHARED.domains.contexts[i].owner <> owner; |
| 1044 | } |
| 1045 | } |
| 1046 | for i in 0 .. state::SHARED.budgets.slots.len { |
| 1047 | if state::SHARED.budgets.slots[i].state |
| 1048 | == slots::State::Live |
| 1049 | { |
| 1050 | assert state::SHARED.budgets.windows[i].owner <> owner; |
| 1051 | } |
| 1052 | } |
| 1053 | for i in 0 .. state::SHARED.interrupts.slots.len { |
| 1054 | if state::SHARED.interrupts.slots[i].state |
| 1055 | == slots::State::Live |
| 1056 | { |
| 1057 | assert state::SHARED.interrupts.records[i].owner |
| 1058 | <> owner; |
| 1059 | } |
| 1060 | } |
| 1061 | } |
| 1062 | sync::leave(guard); |
| 1063 | for byte in "smp reaped dead domains and recovered ownership\n" { |
| 1064 | put(byte); |
| 1065 | } |
| 1066 | return; |
| 1067 | } |
| 1068 | let progress = try! dispatch::request( |
| 1069 | &mut state::SHARED.remote, 0, remote::Action::Reschedule |
| 1070 | ); |
| 1071 | sync::leave(guard); |
| 1072 | let result = try! dispatch::awaitRequest(0, progress); |
| 1073 | assert result == 0; |
| 1074 | } |
| 1075 | panic "reclaimedDomains: domain reaper did not finish"; |
| 1076 | } |
| 1077 | |
| 1078 | /// Confirm the waiting context has released hart ownership before its wakeup. |
| 1079 | unsafe fn waiting() -> bool { |
| 1080 | for attempt in 0 .. POLL_ATTEMPTS { |
| 1081 | let guard = sync::enter(); |
| 1082 | let context = try! domains::context( |
| 1083 | &state::SHARED.domains, IDLE_DOMAIN, IDLE_CONTEXT |
| 1084 | ); |
| 1085 | let stopped = context.state == domains::ContextState::Waiting |
| 1086 | and context.hart == nil; |
| 1087 | sync::leave(guard); |
| 1088 | if stopped { |
| 1089 | return true; |
| 1090 | } |
| 1091 | } |
| 1092 | return false; |
| 1093 | } |
| 1094 | |
| 1095 | /// Print one full-width hexadecimal diagnostic value. |
| 1096 | fn number(value: u64) { |
| 1097 | let mut text: [u8; 16] = [0; 16]; |
| 1098 | for byte in fmt::formatHexU64(value, &mut text[..]) { |
| 1099 | put(byte); |
| 1100 | } |
| 1101 | } |