kernel: Refresh queued terminal acknowledgements
bf3d3cc56d54b3ae4270c496de6ecc86032cd9c8b0df0e2ef8ec0addcbcb9678
1 parent
fa1aa037
kernel/kernel/dispatch.rad
+25 -11
| 51 | 51 | state: State, |
|
| 52 | 52 | /// Validated timer registers for this hart. |
|
| 53 | 53 | timer: Timer, |
|
| 54 | 54 | /// Minimum ticks between scheduled maintenance passes. |
|
| 55 | 55 | maintenance: u64, |
|
| 56 | - | /// Earliest tick for another background maintenance pass. |
|
| 57 | - | nextMaintenance: u64, |
|
| 58 | 56 | } |
|
| 59 | 57 | ||
| 60 | 58 | /// Runtime slots initialized before each hart enables timer interrupts. |
|
| 61 | 59 | unsafe static HARTS: [Runtime; limits::HARTS] = undefined; |
|
| 60 | + | /// Earliest shared maintenance tick, serialized by the metadata lock. |
|
| 61 | + | unsafe static NEXT_MAINTENANCE: u64 = 0; |
|
| 62 | + | /// Hart with a timer armed no later than the shared maintenance deadline. |
|
| 63 | + | unsafe static MAINTENANCE_HART: ?u32 = nil; |
|
| 62 | 64 | ||
| 63 | 65 | /// Read the executing physical hart identifier. |
|
| 64 | 66 | export fn hart() -> u32; |
|
| 65 | 67 | /// Read the current machine stack pointer. |
|
| 66 | 68 | fn stackPointer() -> u64; |
| 81 | 83 | let mut maintenance = machine.timebase as u64 / 100; |
|
| 82 | 84 | if maintenance == 0 { |
|
| 83 | 85 | set maintenance = 1; |
|
| 84 | 86 | } |
|
| 85 | 87 | set HARTS[id] = Runtime { |
|
| 86 | - | state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, nextMaintenance: 0, |
|
| 88 | + | state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, |
|
| 87 | 89 | }; |
|
| 88 | 90 | arm(device, 0xffffffffffffffff); |
|
| 89 | 91 | } |
|
| 90 | 92 | ||
| 91 | 93 | /// Tail-enter dispatch on the firmware stack and resume the selected frame. |
| 97 | 99 | assert id < limits::HARTS; |
|
| 98 | 100 | let stack = HARTS[id].state.stack; |
|
| 99 | 101 | let pointer = stackPointer(); |
|
| 100 | 102 | assert pointer >= stack.start and pointer < stack.end; |
|
| 101 | 103 | let guard = sync::enter(); |
|
| 104 | + | if MAINTENANCE_HART == id { |
|
| 105 | + | set MAINTENANCE_HART = nil; |
|
| 106 | + | } |
|
| 102 | 107 | writeSoftware(HARTS[id].timer.software, 0); |
|
| 103 | 108 | let previous = HARTS[id].state.current; |
|
| 104 | 109 | save(&mut HARTS[id].state, &mut domains::STORE, frame); |
|
| 105 | 110 | if trap::classify(frame.cause) == trap::Cause::External { |
|
| 106 | 111 | plic::claim(id); |
| 122 | 127 | } |
|
| 123 | 128 | set deliveries[count] = delivery; |
|
| 124 | 129 | set count += 1; |
|
| 125 | 130 | } |
|
| 126 | 131 | let clock = now(HARTS[id].timer); |
|
| 132 | + | let maintenanceDue = clock >= NEXT_MAINTENANCE; |
|
| 127 | 133 | let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock); |
|
| 128 | - | let notified = timeout.harts | plic::service(clock) | lifecycle::service(&mut domains::STORE, &budgets::STORE, clock); |
|
| 134 | + | let mut terminals: u32 = 0; |
|
| 135 | + | if domains::STORE.events.pending > 0 or maintenanceDue { |
|
| 136 | + | set terminals = lifecycle::service(&mut domains::STORE, &budgets::STORE, clock); |
|
| 137 | + | } |
|
| 138 | + | let notified = timeout.harts | plic::service(clock) | terminals; |
|
| 129 | 139 | let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock); |
|
| 130 | 140 | let mut deadline = choice.deadline; |
|
| 131 | 141 | if timeout.deadline < deadline { |
|
| 132 | 142 | set deadline = timeout.deadline; |
|
| 133 | 143 | } |
| 149 | 159 | signalContext(context, clock); |
|
| 150 | 160 | } |
|
| 151 | 161 | } |
|
| 152 | 162 | lifecycle::cancelRequests(&mut domains::STORE, &mut remote::STORE); |
|
| 153 | 163 | let cancelled = lifecycle::cancelAllocations(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE, &mut loader::STATE, &mut registry::STORE); |
|
| 154 | - | let maintenanceClock = now(HARTS[id].timer); |
|
| 155 | 164 | let mut maintenance = timeout.pending or cancelled or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0; |
|
| 156 | - | if maintenance and maintenanceClock >= HARTS[id].nextMaintenance { |
|
| 165 | + | if maintenance and maintenanceDue { |
|
| 157 | 166 | let reaped = lifecycle::reapNext(&mut domains::STORE, &mut pages::STORE, &mut budgets::STORE, &mut interrupts::STORE); |
|
| 158 | 167 | let reclaimed = frames::reclaim(&mut pages::STORE.backings.pool, 256); |
|
| 159 | - | set maintenance = timeout.pending or cancelled or reaped or pages::STORE.backings.pool.retiredCount > 0; |
|
| 168 | + | set maintenance = timeout.pending or cancelled or reaped or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0; |
|
| 160 | 169 | let after = now(HARTS[id].timer); |
|
| 161 | 170 | let interval = HARTS[id].maintenance; |
|
| 162 | - | set HARTS[id].nextMaintenance = 0xffffffffffffffff; |
|
| 171 | + | set NEXT_MAINTENANCE = 0xffffffffffffffff; |
|
| 163 | 172 | if interval < 0xffffffffffffffff - after { |
|
| 164 | - | set HARTS[id].nextMaintenance = after + interval; |
|
| 173 | + | set NEXT_MAINTENANCE = after + interval; |
|
| 165 | 174 | } |
|
| 166 | 175 | } |
|
| 167 | - | if maintenance and HARTS[id].nextMaintenance < deadline { |
|
| 168 | - | set deadline = HARTS[id].nextMaintenance; |
|
| 176 | + | if maintenance and (timeout.pending or MAINTENANCE_HART == nil) { |
|
| 177 | + | if MAINTENANCE_HART == nil { |
|
| 178 | + | set MAINTENANCE_HART = id; |
|
| 179 | + | } |
|
| 180 | + | if NEXT_MAINTENANCE < deadline { |
|
| 181 | + | set deadline = NEXT_MAINTENANCE; |
|
| 182 | + | } |
|
| 169 | 183 | } |
|
| 170 | 184 | arm(HARTS[id].timer, deadline); |
|
| 171 | 185 | sync::leave(guard); |
|
| 172 | 186 | } |
|
| 173 | 187 |
kernel/kernel/events.rad
+10 -0
| 85 | 85 | ||
| 86 | 86 | /// Fixed event metadata. Callers serialize all kernel-side mutations. |
|
| 87 | 87 | export record Store: Copy { |
|
| 88 | 88 | /// Terminal events retained for publication into receiver rings. |
|
| 89 | 89 | pending: u32, |
|
| 90 | + | /// Published terminal events that await consumer acknowledgement. |
|
| 91 | + | queued: u32, |
|
| 90 | 92 | /// Producer metadata indexed by receiver domain. |
|
| 91 | 93 | queues: [Queue; limits::DOMAINS], |
|
| 92 | 94 | /// Terminal storage indexed by the terminating domain. |
|
| 93 | 95 | terminals: [Terminal; limits::DOMAINS], |
|
| 94 | 96 | } |
| 110 | 112 | } |
|
| 111 | 113 | ||
| 112 | 114 | /// Initialize metadata before any queue or terminal record exists. |
|
| 113 | 115 | export fn initialize(store: &mut Store) { |
|
| 114 | 116 | set store.pending = 0; |
|
| 117 | + | set store.queued = 0; |
|
| 115 | 118 | for i in 0..limits::DOMAINS { |
|
| 116 | 119 | set store.queues[i].generation = 0; |
|
| 117 | 120 | set store.terminals[i].state = State::Free; |
|
| 118 | 121 | } |
|
| 119 | 122 | } |
| 158 | 161 | let terminal = &mut store.terminals[subject.index]; |
|
| 159 | 162 | assert terminal.subject == subject |
|
| 160 | 163 | and terminal.receiver == receiver |
|
| 161 | 164 | and terminal.state == State::Queued; |
|
| 162 | 165 | set terminal.state = State::Acknowledged; |
|
| 166 | + | assert store.queued > 0; |
|
| 167 | + | set store.queued -= 1; |
|
| 163 | 168 | } |
|
| 164 | 169 | set queue.head = next(queue.head); |
|
| 165 | 170 | } |
|
| 166 | 171 | } |
|
| 167 | 172 |
| 287 | 292 | let queue = &mut store.queues[terminal.receiver.index]; |
|
| 288 | 293 | if distance(queue.tail, queue.head) == CAPACITY { |
|
| 289 | 294 | return false; |
|
| 290 | 295 | } |
|
| 291 | 296 | set store.terminals[subject.index].state = State::Queued; |
|
| 297 | + | set store.queued += 1; |
|
| 292 | 298 | assert store.pending > 0; |
|
| 293 | 299 | set store.pending -= 1; |
|
| 294 | 300 | publish(queue, ring, terminal.event, subject); |
|
| 295 | 301 | return true; |
|
| 296 | 302 | } |
| 368 | 374 | if generations[receiver.index] == receiver.generation { |
|
| 369 | 375 | if store.terminals[i].state == State::Pending { |
|
| 370 | 376 | assert store.pending > 0; |
|
| 371 | 377 | set store.pending -= 1; |
|
| 372 | 378 | } |
|
| 379 | + | else { |
|
| 380 | + | assert store.queued > 0; |
|
| 381 | + | set store.queued -= 1; |
|
| 382 | + | } |
|
| 373 | 383 | set store.terminals[i].state = State::Acknowledged; |
|
| 374 | 384 | } |
|
| 375 | 385 | } |
|
| 376 | 386 | } |
kernel/kernel/lifecycle.rad
+11 -4
| 494 | 494 | } |
|
| 495 | 495 | try! domains::retireStorage(store, &mut memory.backings, owner); |
|
| 496 | 496 | return true; |
|
| 497 | 497 | } |
|
| 498 | 498 | ||
| 499 | - | /// Publish retained terminal events and report receiver harts under the metadata lock. |
|
| 499 | + | /// Refresh terminal acknowledgements, publish pending events, and report receiver harts. |
|
| 500 | + | /// The caller holds the metadata lock throughout service. |
|
| 500 | 501 | export unsafe fn service(store: &mut domains::Store, windows: &budgets::Store, now: u64) -> u32 { |
|
| 501 | - | if store.events.pending == 0 { |
|
| 502 | + | if store.events.pending == 0 and store.events.queued == 0 { |
|
| 502 | 503 | return 0; |
|
| 503 | 504 | } |
|
| 504 | 505 | let mut notified: [bool; limits::DOMAINS] = [false; limits::DOMAINS]; |
|
| 505 | 506 | let mut changed = false; |
|
| 506 | 507 | for i in 0..limits::DOMAINS { |
|
| 507 | - | let terminal = store.events.terminals[i]; |
|
| 508 | - | if terminal.state <> events::State::Pending { |
|
| 508 | + | let state = store.events.terminals[i].state; |
|
| 509 | + | if state <> events::State::Pending and state <> events::State::Queued { |
|
| 509 | 510 | continue; |
|
| 510 | 511 | } |
|
| 512 | + | let terminal = store.events.terminals[i]; |
|
| 511 | 513 | let mut receiver = try domains::get(store, terminal.receiver) catch { |
|
| 512 | 514 | continue; |
|
| 513 | 515 | }; |
|
| 514 | 516 | if receiver.state == domains::Lifecycle::Dead { |
|
| 515 | 517 | continue; |
|
| 516 | 518 | } |
|
| 519 | + | if terminal.state == events::State::Queued { |
|
| 520 | + | try events::refresh(&mut store.events, terminal.receiver, &receiver.memory.ring) catch { |
|
| 521 | + | }; |
|
| 522 | + | continue; |
|
| 523 | + | } |
|
| 517 | 524 | let delivered = try events::deliver(&mut store.events, terminal.subject, &mut receiver.memory.ring) catch { |
|
| 518 | 525 | false |
|
| 519 | 526 | }; |
|
| 520 | 527 | if delivered { |
|
| 521 | 528 | set notified[terminal.receiver.index] = true; |
kernel/kernel/tests/lifecycle.rad
+32 -0
| 12 | 12 | use kernel::remote; |
|
| 13 | 13 | use kernel::pages; |
|
| 14 | 14 | use kernel::backing; |
|
| 15 | 15 | use kernel::capability; |
|
| 16 | 16 | use kernel::frames; |
|
| 17 | + | use kernel::interrupts; |
|
| 17 | 18 | ||
| 18 | 19 | /// Receiver generations, contexts, and terminal events. |
|
| 19 | 20 | unsafe static STORE: domains::Store = undefined; |
|
| 20 | 21 | /// CPU authority retained by receivers. |
|
| 21 | 22 | unsafe static WINDOWS: budgets::Store = undefined; |
|
| 22 | 23 | /// Queued calls and acknowledgements retained by stopped contexts. |
|
| 23 | 24 | unsafe static REQUESTS: remote::Store = undefined; |
|
| 24 | 25 | /// Physical allocation metadata used for interrupted page calls. |
|
| 25 | 26 | unsafe static MEMORY: pages::Store = undefined; |
|
| 27 | + | /// Interrupt ownership passed to terminal identity reaping. |
|
| 28 | + | unsafe static IRQS: interrupts::Store = undefined; |
|
| 26 | 29 | /// Mapped frames available to interrupted page calls. |
|
| 27 | 30 | static RAM: [u64; 4096] = [0; 4096]; |
|
| 28 | 31 | /// Reservations retained across private initialization. |
|
| 29 | 32 | unsafe static CALLS: lifecycle::Calls = undefined; |
|
| 30 | 33 | /// Shared queue storage for the first receiver. |
| 297 | 300 | retain(4, 0); |
|
| 298 | 301 | assert lifecycle::service(&mut STORE, &WINDOWS, 100) == 0; |
|
| 299 | 302 | assert STORE.events.terminals[4].state == events::State::Queued; |
|
| 300 | 303 | } |
|
| 301 | 304 | ||
| 305 | + | /// Maintenance acknowledges the last queued exit while its receiver keeps running. |
|
| 306 | + | @test unsafe fn queuedAcknowledgement() throws (testing::TestError) { |
|
| 307 | + | initialize(); |
|
| 308 | + | let subject = abi::Ref { index: 2, generation: 1 }; |
|
| 309 | + | set STORE.slots[2] = slots::Slot { generation: 1, state: slots::State::Live }; |
|
| 310 | + | set STORE.records[2].state = domains::Lifecycle::Dead; |
|
| 311 | + | set STORE.reclaimed[2] = true; |
|
| 312 | + | set STORE.dead = 1; |
|
| 313 | + | retain(2, 0); |
|
| 314 | + | assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 2; |
|
| 315 | + | assert STORE.events.pending == 0 and STORE.events.terminals[2].state == events::State::Queued; |
|
| 316 | + | assert STORE.events.queued == 1; |
|
| 317 | + | assert not lifecycle::reapNext(&mut STORE, &mut MEMORY, &mut WINDOWS, &mut IRQS); |
|
| 318 | + | assert lifecycle::service(&mut STORE, &WINDOWS, 21) == 0; |
|
| 319 | + | assert STORE.events.terminals[2].state == events::State::Queued; |
|
| 320 | + | let event = try! events::pop(&mut FIRST.ring) else panic "terminal event"; |
|
| 321 | + | assert event.value == abi::id(subject); |
|
| 322 | + | assert STORE.contexts[0].state == domains::ContextState::Ready; |
|
| 323 | + | assert lifecycle::service(&mut STORE, &WINDOWS, 22) == 0; |
|
| 324 | + | assert try! events::acknowledged(&STORE.events, subject); |
|
| 325 | + | assert STORE.events.queued == 0; |
|
| 326 | + | assert lifecycle::reapNext(&mut STORE, &mut MEMORY, &mut WINDOWS, &mut IRQS); |
|
| 327 | + | assert STORE.dead == 0 and STORE.slots[2].state == slots::State::Free; |
|
| 328 | + | assert STORE.events.terminals[2].state == events::State::Free; |
|
| 329 | + | } |
|
| 330 | + | ||
| 302 | 331 | /// Full receivers retain terminal events while other receivers make progress. |
|
| 303 | 332 | @test unsafe fn saturation() throws (testing::TestError) { |
|
| 304 | 333 | initialize(); |
|
| 305 | 334 | let receiver = abi::Ref { index: 0, generation: 1 }; |
|
| 306 | 335 | for i in 0..events::ORDINARY { |
| 324 | 353 | assert STORE.events.queues[0].tail == events::CAPACITY + 1; |
|
| 325 | 354 | retain(4, 0); |
|
| 326 | 355 | assert STORE.events.pending == 1; |
|
| 327 | 356 | try! events::close(&mut STORE.events, receiver); |
|
| 328 | 357 | assert STORE.events.pending == 0; |
|
| 358 | + | assert STORE.events.queued == 1; |
|
| 359 | + | try! events::close(&mut STORE.events, abi::Ref { index: 1, generation: 1 }); |
|
| 360 | + | assert STORE.events.queued == 0; |
|
| 329 | 361 | } |