kernel: Refresh queued terminal acknowledgements
f5d37a5e340557b2ef04ffbc056b2da17580c7bcb4c296eeda0203b7b62a479e
1 parent
920f8fc3
kernel/kernel/dispatch.rad
+20 -10
| 51 | 51 | state: State, |
|
| 52 | 52 | /// Validated timer registers for this hart. |
|
| 53 | 53 | timer: Timer, |
|
| 54 | 54 | /// Minimum ticks between scheduled maintenance passes. |
|
| 55 | 55 | maintenance: u64, |
|
| 56 | - | /// Earliest tick for another background maintenance pass. |
|
| 57 | - | nextMaintenance: u64, |
|
| 58 | 56 | } |
|
| 59 | 57 | ||
| 60 | 58 | /// Runtime slots initialized before each hart enables timer interrupts. |
|
| 61 | 59 | unsafe static HARTS: [Runtime; limits::HARTS] = undefined; |
|
| 60 | + | /// Earliest shared maintenance tick, serialized by the metadata lock. |
|
| 61 | + | unsafe static NEXT_MAINTENANCE: u64 = 0; |
|
| 62 | + | /// Hart with a timer armed no later than the shared maintenance deadline. |
|
| 63 | + | unsafe static MAINTENANCE_HART: ?u32 = nil; |
|
| 62 | 64 | ||
| 63 | 65 | /// Read the executing physical hart identifier. |
|
| 64 | 66 | export fn hart() -> u32; |
|
| 65 | 67 | /// Read the current machine stack pointer. |
|
| 66 | 68 | fn stackPointer() -> u64; |
| 79 | 81 | set idle.registers[2] = stack.end; |
|
| 80 | 82 | set idle.registers[3] = kernelGp; |
|
| 81 | 83 | let mut maintenance = machine.timebase as u64 / 100; |
|
| 82 | 84 | if maintenance == 0 { set maintenance = 1; } |
|
| 83 | 85 | set HARTS[id] = Runtime { |
|
| 84 | - | state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, nextMaintenance: 0, |
|
| 86 | + | state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, |
|
| 85 | 87 | }; |
|
| 86 | 88 | arm(device, 0xffffffffffffffff); |
|
| 87 | 89 | } |
|
| 88 | 90 | ||
| 89 | 91 | /// Tail-enter dispatch on the firmware stack and resume the selected frame. |
| 95 | 97 | assert id < limits::HARTS; |
|
| 96 | 98 | let stack = HARTS[id].state.stack; |
|
| 97 | 99 | let pointer = stackPointer(); |
|
| 98 | 100 | assert pointer >= stack.start and pointer < stack.end; |
|
| 99 | 101 | let guard = sync::enter(); |
|
| 102 | + | if MAINTENANCE_HART == id { set MAINTENANCE_HART = nil; } |
|
| 100 | 103 | writeSoftware(HARTS[id].timer.software, 0); |
|
| 101 | 104 | let previous = HARTS[id].state.current; |
|
| 102 | 105 | save(&mut HARTS[id].state, &mut domains::STORE, frame); |
|
| 103 | 106 | if trap::classify(frame.cause) == trap::Cause::External { plic::claim(id); } |
|
| 104 | 107 | let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined; |
| 115 | 118 | } |
|
| 116 | 119 | set deliveries[count] = delivery; |
|
| 117 | 120 | set count += 1; |
|
| 118 | 121 | } |
|
| 119 | 122 | let clock = now(HARTS[id].timer); |
|
| 123 | + | let maintenanceDue = clock >= NEXT_MAINTENANCE; |
|
| 120 | 124 | let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock); |
|
| 121 | - | let notified = timeout.harts | plic::service(clock) | lifecycle::service(&mut domains::STORE, &budgets::STORE, clock); |
|
| 125 | + | let mut terminals: u32 = 0; |
|
| 126 | + | if domains::STORE.events.pending > 0 or maintenanceDue { |
|
| 127 | + | set terminals = lifecycle::service(&mut domains::STORE, &budgets::STORE, clock); |
|
| 128 | + | } |
|
| 129 | + | let notified = timeout.harts | plic::service(clock) | terminals; |
|
| 122 | 130 | let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock); |
|
| 123 | 131 | let mut deadline = choice.deadline; |
|
| 124 | 132 | if timeout.deadline < deadline { set deadline = timeout.deadline; } |
|
| 125 | 133 | reschedule(notified); |
|
| 126 | 134 | if previous <> choice.context { |
| 132 | 140 | if let case remote::Action::Budget(call) = delivery.request.action { signalContext(call.context, clock); } |
|
| 133 | 141 | if let case remote::Action::Ready(context) = delivery.request.action { signalContext(context, clock); } |
|
| 134 | 142 | } |
|
| 135 | 143 | lifecycle::cancelRequests(&mut domains::STORE, &mut remote::STORE); |
|
| 136 | 144 | let cancelled = lifecycle::cancelAllocations(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE, &mut loader::STATE, &mut registry::STORE); |
|
| 137 | - | let maintenanceClock = now(HARTS[id].timer); |
|
| 138 | 145 | let mut maintenance = timeout.pending or cancelled or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0; |
|
| 139 | - | if maintenance and maintenanceClock >= HARTS[id].nextMaintenance { |
|
| 146 | + | if maintenance and maintenanceDue { |
|
| 140 | 147 | let reaped = lifecycle::reapNext(&mut domains::STORE, &mut pages::STORE, &mut budgets::STORE, &mut interrupts::STORE); |
|
| 141 | 148 | let reclaimed = frames::reclaim(&mut pages::STORE.backings.pool, 256); |
|
| 142 | - | set maintenance = timeout.pending or cancelled or reaped or pages::STORE.backings.pool.retiredCount > 0; |
|
| 149 | + | set maintenance = timeout.pending or cancelled or reaped or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0; |
|
| 143 | 150 | let after = now(HARTS[id].timer); |
|
| 144 | 151 | let interval = HARTS[id].maintenance; |
|
| 145 | - | set HARTS[id].nextMaintenance = 0xffffffffffffffff; |
|
| 146 | - | if interval < 0xffffffffffffffff - after { set HARTS[id].nextMaintenance = after + interval; } |
|
| 152 | + | set NEXT_MAINTENANCE = 0xffffffffffffffff; |
|
| 153 | + | if interval < 0xffffffffffffffff - after { set NEXT_MAINTENANCE = after + interval; } |
|
| 154 | + | } |
|
| 155 | + | if maintenance and (timeout.pending or MAINTENANCE_HART == nil) { |
|
| 156 | + | if MAINTENANCE_HART == nil { set MAINTENANCE_HART = id; } |
|
| 157 | + | if NEXT_MAINTENANCE < deadline { set deadline = NEXT_MAINTENANCE; } |
|
| 147 | 158 | } |
|
| 148 | - | if maintenance and HARTS[id].nextMaintenance < deadline { set deadline = HARTS[id].nextMaintenance; } |
|
| 149 | 159 | arm(HARTS[id].timer, deadline); |
|
| 150 | 160 | sync::leave(guard); |
|
| 151 | 161 | } |
|
| 152 | 162 | ||
| 153 | 163 | /// Signal an initialized online hart while request metadata is serialized. |
kernel/kernel/events.rad
+7 -0
| 85 | 85 | ||
| 86 | 86 | /// Fixed event metadata. Callers serialize all kernel-side mutations. |
|
| 87 | 87 | export record Store: Copy { |
|
| 88 | 88 | /// Terminal events retained for publication into receiver rings. |
|
| 89 | 89 | pending: u32, |
|
| 90 | + | /// Published terminal events that await consumer acknowledgement. |
|
| 91 | + | queued: u32, |
|
| 90 | 92 | /// Producer metadata indexed by receiver domain. |
|
| 91 | 93 | queues: [Queue; limits::DOMAINS], |
|
| 92 | 94 | /// Terminal storage indexed by the terminating domain. |
|
| 93 | 95 | terminals: [Terminal; limits::DOMAINS], |
|
| 94 | 96 | } |
| 108 | 110 | fn next(value: u32) -> u32 { return ((value as u64 + 1) & 0xffffffff) as u32; } |
|
| 109 | 111 | ||
| 110 | 112 | /// Initialize metadata before any queue or terminal record exists. |
|
| 111 | 113 | export fn initialize(store: &mut Store) { |
|
| 112 | 114 | set store.pending = 0; |
|
| 115 | + | set store.queued = 0; |
|
| 113 | 116 | for i in 0..limits::DOMAINS { |
|
| 114 | 117 | set store.queues[i].generation = 0; |
|
| 115 | 118 | set store.terminals[i].state = State::Free; |
|
| 116 | 119 | } |
|
| 117 | 120 | } |
| 148 | 151 | let terminal = &mut store.terminals[subject.index]; |
|
| 149 | 152 | assert terminal.subject == subject |
|
| 150 | 153 | and terminal.receiver == receiver |
|
| 151 | 154 | and terminal.state == State::Queued; |
|
| 152 | 155 | set terminal.state = State::Acknowledged; |
|
| 156 | + | assert store.queued > 0; |
|
| 157 | + | set store.queued -= 1; |
|
| 153 | 158 | } |
|
| 154 | 159 | set queue.head = next(queue.head); |
|
| 155 | 160 | } |
|
| 156 | 161 | } |
|
| 157 | 162 |
| 246 | 251 | if terminal.state <> State::Pending { return false; } |
|
| 247 | 252 | try refresh(store, terminal.receiver, ring); |
|
| 248 | 253 | let queue = &mut store.queues[terminal.receiver.index]; |
|
| 249 | 254 | if distance(queue.tail, queue.head) == CAPACITY { return false; } |
|
| 250 | 255 | set store.terminals[subject.index].state = State::Queued; |
|
| 256 | + | set store.queued += 1; |
|
| 251 | 257 | assert store.pending > 0; |
|
| 252 | 258 | set store.pending -= 1; |
|
| 253 | 259 | publish(queue, ring, terminal.event, subject); |
|
| 254 | 260 | return true; |
|
| 255 | 261 | } |
| 302 | 308 | let terminal = store.terminals[i]; |
|
| 303 | 309 | if terminal.state <> State::Pending and terminal.state <> State::Queued { continue; } |
|
| 304 | 310 | let receiver = terminal.receiver; |
|
| 305 | 311 | if generations[receiver.index] == receiver.generation { |
|
| 306 | 312 | if store.terminals[i].state == State::Pending { assert store.pending > 0; set store.pending -= 1; } |
|
| 313 | + | else { assert store.queued > 0; set store.queued -= 1; } |
|
| 307 | 314 | set store.terminals[i].state = State::Acknowledged; |
|
| 308 | 315 | } |
|
| 309 | 316 | } |
|
| 310 | 317 | } |
kernel/kernel/lifecycle.rad
+9 -3
| 429 | 429 | if try! domains::retireContext(store, &mut memory.backings.pool, owner) { return true; } |
|
| 430 | 430 | try! domains::retireStorage(store, &mut memory.backings, owner); |
|
| 431 | 431 | return true; |
|
| 432 | 432 | } |
|
| 433 | 433 | ||
| 434 | - | /// Publish retained terminal events and report receiver harts under the metadata lock. |
|
| 434 | + | /// Refresh terminal acknowledgements, publish pending events, and report receiver harts. |
|
| 435 | + | /// The caller holds the metadata lock throughout service. |
|
| 435 | 436 | export unsafe fn service(store: &mut domains::Store, windows: &budgets::Store, now: u64) -> u32 { |
|
| 436 | - | if store.events.pending == 0 { return 0; } |
|
| 437 | + | if store.events.pending == 0 and store.events.queued == 0 { return 0; } |
|
| 437 | 438 | let mut notified: [bool; limits::DOMAINS] = [false; limits::DOMAINS]; |
|
| 438 | 439 | let mut changed = false; |
|
| 439 | 440 | for i in 0..limits::DOMAINS { |
|
| 441 | + | let state = store.events.terminals[i].state; |
|
| 442 | + | if state <> events::State::Pending and state <> events::State::Queued { continue; } |
|
| 440 | 443 | let terminal = store.events.terminals[i]; |
|
| 441 | - | if terminal.state <> events::State::Pending { continue; } |
|
| 442 | 444 | let mut receiver = try domains::get(store, terminal.receiver) catch { continue; }; |
|
| 443 | 445 | if receiver.state == domains::Lifecycle::Dead { continue; } |
|
| 446 | + | if terminal.state == events::State::Queued { |
|
| 447 | + | try events::refresh(&mut store.events, terminal.receiver, &receiver.memory.ring) catch {}; |
|
| 448 | + | continue; |
|
| 449 | + | } |
|
| 444 | 450 | let delivered = try events::deliver(&mut store.events, terminal.subject, &mut receiver.memory.ring) catch { false }; |
|
| 445 | 451 | if delivered { set notified[terminal.receiver.index] = true; set changed = true; } |
|
| 446 | 452 | } |
|
| 447 | 453 | if not changed { return 0; } |
|
| 448 | 454 | for i in 0..limits::CONTEXTS { |
kernel/kernel/tests/lifecycle.rad
+32 -0
| 12 | 12 | use kernel::remote; |
|
| 13 | 13 | use kernel::pages; |
|
| 14 | 14 | use kernel::backing; |
|
| 15 | 15 | use kernel::capability; |
|
| 16 | 16 | use kernel::frames; |
|
| 17 | + | use kernel::interrupts; |
|
| 17 | 18 | ||
| 18 | 19 | /// Receiver generations, contexts, and terminal events. |
|
| 19 | 20 | unsafe static STORE: domains::Store = undefined; |
|
| 20 | 21 | /// CPU authority retained by receivers. |
|
| 21 | 22 | unsafe static WINDOWS: budgets::Store = undefined; |
|
| 22 | 23 | /// Queued calls and acknowledgements retained by stopped contexts. |
|
| 23 | 24 | unsafe static REQUESTS: remote::Store = undefined; |
|
| 24 | 25 | /// Physical allocation metadata used for interrupted page calls. |
|
| 25 | 26 | unsafe static MEMORY: pages::Store = undefined; |
|
| 27 | + | /// Interrupt ownership passed to terminal identity reaping. |
|
| 28 | + | unsafe static IRQS: interrupts::Store = undefined; |
|
| 26 | 29 | /// Mapped frames available to interrupted page calls. |
|
| 27 | 30 | static RAM: [u64; 4096] = [0; 4096]; |
|
| 28 | 31 | /// Reservations retained across private initialization. |
|
| 29 | 32 | unsafe static CALLS: lifecycle::Calls = undefined; |
|
| 30 | 33 | /// Shared queue storage for the first receiver. |
| 277 | 280 | retain(4, 0); |
|
| 278 | 281 | assert lifecycle::service(&mut STORE, &WINDOWS, 100) == 0; |
|
| 279 | 282 | assert STORE.events.terminals[4].state == events::State::Queued; |
|
| 280 | 283 | } |
|
| 281 | 284 | ||
| 285 | + | /// Maintenance acknowledges the last queued exit while its receiver keeps running. |
|
| 286 | + | @test unsafe fn queuedAcknowledgement() throws (testing::TestError) { |
|
| 287 | + | initialize(); |
|
| 288 | + | let subject = abi::Ref { index: 2, generation: 1 }; |
|
| 289 | + | set STORE.slots[2] = slots::Slot { generation: 1, state: slots::State::Live }; |
|
| 290 | + | set STORE.records[2].state = domains::Lifecycle::Dead; |
|
| 291 | + | set STORE.reclaimed[2] = true; |
|
| 292 | + | set STORE.dead = 1; |
|
| 293 | + | retain(2, 0); |
|
| 294 | + | assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 2; |
|
| 295 | + | assert STORE.events.pending == 0 and STORE.events.terminals[2].state == events::State::Queued; |
|
| 296 | + | assert STORE.events.queued == 1; |
|
| 297 | + | assert not lifecycle::reapNext(&mut STORE, &mut MEMORY, &mut WINDOWS, &mut IRQS); |
|
| 298 | + | assert lifecycle::service(&mut STORE, &WINDOWS, 21) == 0; |
|
| 299 | + | assert STORE.events.terminals[2].state == events::State::Queued; |
|
| 300 | + | let event = try! events::pop(&mut FIRST.ring) else panic "terminal event"; |
|
| 301 | + | assert event.value == abi::id(subject); |
|
| 302 | + | assert STORE.contexts[0].state == domains::ContextState::Ready; |
|
| 303 | + | assert lifecycle::service(&mut STORE, &WINDOWS, 22) == 0; |
|
| 304 | + | assert try! events::acknowledged(&STORE.events, subject); |
|
| 305 | + | assert STORE.events.queued == 0; |
|
| 306 | + | assert lifecycle::reapNext(&mut STORE, &mut MEMORY, &mut WINDOWS, &mut IRQS); |
|
| 307 | + | assert STORE.dead == 0 and STORE.slots[2].state == slots::State::Free; |
|
| 308 | + | assert STORE.events.terminals[2].state == events::State::Free; |
|
| 309 | + | } |
|
| 310 | + | ||
| 282 | 311 | /// Full receivers retain terminal events while other receivers make progress. |
|
| 283 | 312 | @test unsafe fn saturation() throws (testing::TestError) { |
|
| 284 | 313 | initialize(); |
|
| 285 | 314 | let receiver = abi::Ref { index: 0, generation: 1 }; |
|
| 286 | 315 | for i in 0..events::ORDINARY { |
| 302 | 331 | assert STORE.events.queues[0].tail == events::CAPACITY + 1; |
|
| 303 | 332 | retain(4, 0); |
|
| 304 | 333 | assert STORE.events.pending == 1; |
|
| 305 | 334 | try! events::close(&mut STORE.events, receiver); |
|
| 306 | 335 | assert STORE.events.pending == 0; |
|
| 336 | + | assert STORE.events.queued == 1; |
|
| 337 | + | try! events::close(&mut STORE.events, abi::Ref { index: 1, generation: 1 }); |
|
| 338 | + | assert STORE.events.queued == 0; |
|
| 307 | 339 | } |