kernel: Refresh queued terminal acknowledgements

f5d37a5e340557b2ef04ffbc056b2da17580c7bcb4c296eeda0203b7b62a479e
Alexis Sellier committed ago 1 parent 920f8fc3
kernel/kernel/dispatch.rad +20 -10
51 51
    state: State,
52 52
    /// Validated timer registers for this hart.
53 53
    timer: Timer,
54 54
    /// Minimum ticks between scheduled maintenance passes.
55 55
    maintenance: u64,
56 -
    /// Earliest tick for another background maintenance pass.
57 -
    nextMaintenance: u64,
58 56
}
59 57
60 58
/// Runtime slots initialized before each hart enables timer interrupts.
61 59
unsafe static HARTS: [Runtime; limits::HARTS] = undefined;
60 +
/// Earliest shared maintenance tick, serialized by the metadata lock.
61 +
unsafe static NEXT_MAINTENANCE: u64 = 0;
62 +
/// Hart with a timer armed no later than the shared maintenance deadline.
63 +
unsafe static MAINTENANCE_HART: ?u32 = nil;
62 64
63 65
/// Read the executing physical hart identifier.
64 66
export fn hart() -> u32;
65 67
/// Read the current machine stack pointer.
66 68
fn stackPointer() -> u64;
79 81
    set idle.registers[2] = stack.end;
80 82
    set idle.registers[3] = kernelGp;
81 83
    let mut maintenance = machine.timebase as u64 / 100;
82 84
    if maintenance == 0 { set maintenance = 1; }
83 85
    set HARTS[id] = Runtime {
84 -
        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, nextMaintenance: 0,
86 +
        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance,
85 87
    };
86 88
    arm(device, 0xffffffffffffffff);
87 89
}
88 90
89 91
/// Tail-enter dispatch on the firmware stack and resume the selected frame.
95 97
    assert id < limits::HARTS;
96 98
    let stack = HARTS[id].state.stack;
97 99
    let pointer = stackPointer();
98 100
    assert pointer >= stack.start and pointer < stack.end;
99 101
    let guard = sync::enter();
102 +
    if MAINTENANCE_HART == id { set MAINTENANCE_HART = nil; }
100 103
    writeSoftware(HARTS[id].timer.software, 0);
101 104
    let previous = HARTS[id].state.current;
102 105
    save(&mut HARTS[id].state, &mut domains::STORE, frame);
103 106
    if trap::classify(frame.cause) == trap::Cause::External { plic::claim(id); }
104 107
    let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
115 118
        }
116 119
        set deliveries[count] = delivery;
117 120
        set count += 1;
118 121
    }
119 122
    let clock = now(HARTS[id].timer);
123 +
    let maintenanceDue = clock >= NEXT_MAINTENANCE;
120 124
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
121 -
    let notified = timeout.harts | plic::service(clock) | lifecycle::service(&mut domains::STORE, &budgets::STORE, clock);
125 +
    let mut terminals: u32 = 0;
126 +
    if domains::STORE.events.pending > 0 or maintenanceDue {
127 +
        set terminals = lifecycle::service(&mut domains::STORE, &budgets::STORE, clock);
128 +
    }
129 +
    let notified = timeout.harts | plic::service(clock) | terminals;
122 130
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
123 131
    let mut deadline = choice.deadline;
124 132
    if timeout.deadline < deadline { set deadline = timeout.deadline; }
125 133
    reschedule(notified);
126 134
    if previous <> choice.context {
132 140
        if let case remote::Action::Budget(call) = delivery.request.action { signalContext(call.context, clock); }
133 141
        if let case remote::Action::Ready(context) = delivery.request.action { signalContext(context, clock); }
134 142
    }
135 143
    lifecycle::cancelRequests(&mut domains::STORE, &mut remote::STORE);
136 144
    let cancelled = lifecycle::cancelAllocations(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE, &mut loader::STATE, &mut registry::STORE);
137 -
    let maintenanceClock = now(HARTS[id].timer);
138 145
    let mut maintenance = timeout.pending or cancelled or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0;
139 -
    if maintenance and maintenanceClock >= HARTS[id].nextMaintenance {
146 +
    if maintenance and maintenanceDue {
140 147
        let reaped = lifecycle::reapNext(&mut domains::STORE, &mut pages::STORE, &mut budgets::STORE, &mut interrupts::STORE);
141 148
        let reclaimed = frames::reclaim(&mut pages::STORE.backings.pool, 256);
142 -
        set maintenance = timeout.pending or cancelled or reaped or pages::STORE.backings.pool.retiredCount > 0;
149 +
        set maintenance = timeout.pending or cancelled or reaped or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0;
143 150
        let after = now(HARTS[id].timer);
144 151
        let interval = HARTS[id].maintenance;
145 -
        set HARTS[id].nextMaintenance = 0xffffffffffffffff;
146 -
        if interval < 0xffffffffffffffff - after { set HARTS[id].nextMaintenance = after + interval; }
152 +
        set NEXT_MAINTENANCE = 0xffffffffffffffff;
153 +
        if interval < 0xffffffffffffffff - after { set NEXT_MAINTENANCE = after + interval; }
154 +
    }
155 +
    if maintenance and (timeout.pending or MAINTENANCE_HART == nil) {
156 +
        if MAINTENANCE_HART == nil { set MAINTENANCE_HART = id; }
157 +
        if NEXT_MAINTENANCE < deadline { set deadline = NEXT_MAINTENANCE; }
147 158
    }
148 -
    if maintenance and HARTS[id].nextMaintenance < deadline { set deadline = HARTS[id].nextMaintenance; }
149 159
    arm(HARTS[id].timer, deadline);
150 160
    sync::leave(guard);
151 161
}
152 162
153 163
/// Signal an initialized online hart while request metadata is serialized.
kernel/kernel/events.rad +7 -0
85 85
86 86
/// Fixed event metadata. Callers serialize all kernel-side mutations.
87 87
export record Store: Copy {
88 88
    /// Terminal events retained for publication into receiver rings.
89 89
    pending: u32,
90 +
    /// Published terminal events that await consumer acknowledgement.
91 +
    queued: u32,
90 92
    /// Producer metadata indexed by receiver domain.
91 93
    queues: [Queue; limits::DOMAINS],
92 94
    /// Terminal storage indexed by the terminating domain.
93 95
    terminals: [Terminal; limits::DOMAINS],
94 96
}
108 110
fn next(value: u32) -> u32 { return ((value as u64 + 1) & 0xffffffff) as u32; }
109 111
110 112
/// Initialize metadata before any queue or terminal record exists.
111 113
export fn initialize(store: &mut Store) {
112 114
    set store.pending = 0;
115 +
    set store.queued = 0;
113 116
    for i in 0..limits::DOMAINS {
114 117
        set store.queues[i].generation = 0;
115 118
        set store.terminals[i].state = State::Free;
116 119
    }
117 120
}
148 151
            let terminal = &mut store.terminals[subject.index];
149 152
            assert terminal.subject == subject
150 153
                and terminal.receiver == receiver
151 154
                and terminal.state == State::Queued;
152 155
            set terminal.state = State::Acknowledged;
156 +
            assert store.queued > 0;
157 +
            set store.queued -= 1;
153 158
        }
154 159
        set queue.head = next(queue.head);
155 160
    }
156 161
}
157 162
246 251
    if terminal.state <> State::Pending { return false; }
247 252
    try refresh(store, terminal.receiver, ring);
248 253
    let queue = &mut store.queues[terminal.receiver.index];
249 254
    if distance(queue.tail, queue.head) == CAPACITY { return false; }
250 255
    set store.terminals[subject.index].state = State::Queued;
256 +
    set store.queued += 1;
251 257
    assert store.pending > 0;
252 258
    set store.pending -= 1;
253 259
    publish(queue, ring, terminal.event, subject);
254 260
    return true;
255 261
}
302 308
        let terminal = store.terminals[i];
303 309
        if terminal.state <> State::Pending and terminal.state <> State::Queued { continue; }
304 310
        let receiver = terminal.receiver;
305 311
        if generations[receiver.index] == receiver.generation {
306 312
            if store.terminals[i].state == State::Pending { assert store.pending > 0; set store.pending -= 1; }
313 +
            else { assert store.queued > 0; set store.queued -= 1; }
307 314
            set store.terminals[i].state = State::Acknowledged;
308 315
        }
309 316
    }
310 317
}
kernel/kernel/lifecycle.rad +9 -3
429 429
    if try! domains::retireContext(store, &mut memory.backings.pool, owner) { return true; }
430 430
    try! domains::retireStorage(store, &mut memory.backings, owner);
431 431
    return true;
432 432
}
433 433
434 -
/// Publish retained terminal events and report receiver harts under the metadata lock.
434 +
/// Refresh terminal acknowledgements, publish pending events, and report receiver harts.
435 +
/// The caller holds the metadata lock throughout service.
435 436
export unsafe fn service(store: &mut domains::Store, windows: &budgets::Store, now: u64) -> u32 {
436 -
    if store.events.pending == 0 { return 0; }
437 +
    if store.events.pending == 0 and store.events.queued == 0 { return 0; }
437 438
    let mut notified: [bool; limits::DOMAINS] = [false; limits::DOMAINS];
438 439
    let mut changed = false;
439 440
    for i in 0..limits::DOMAINS {
441 +
        let state = store.events.terminals[i].state;
442 +
        if state <> events::State::Pending and state <> events::State::Queued { continue; }
440 443
        let terminal = store.events.terminals[i];
441 -
        if terminal.state <> events::State::Pending { continue; }
442 444
        let mut receiver = try domains::get(store, terminal.receiver) catch { continue; };
443 445
        if receiver.state == domains::Lifecycle::Dead { continue; }
446 +
        if terminal.state == events::State::Queued {
447 +
            try events::refresh(&mut store.events, terminal.receiver, &receiver.memory.ring) catch {};
448 +
            continue;
449 +
        }
444 450
        let delivered = try events::deliver(&mut store.events, terminal.subject, &mut receiver.memory.ring) catch { false };
445 451
        if delivered { set notified[terminal.receiver.index] = true; set changed = true; }
446 452
    }
447 453
    if not changed { return 0; }
448 454
    for i in 0..limits::CONTEXTS {
kernel/kernel/tests/lifecycle.rad +32 -0
12 12
use kernel::remote;
13 13
use kernel::pages;
14 14
use kernel::backing;
15 15
use kernel::capability;
16 16
use kernel::frames;
17 +
use kernel::interrupts;
17 18
18 19
/// Receiver generations, contexts, and terminal events.
19 20
unsafe static STORE: domains::Store = undefined;
20 21
/// CPU authority retained by receivers.
21 22
unsafe static WINDOWS: budgets::Store = undefined;
22 23
/// Queued calls and acknowledgements retained by stopped contexts.
23 24
unsafe static REQUESTS: remote::Store = undefined;
24 25
/// Physical allocation metadata used for interrupted page calls.
25 26
unsafe static MEMORY: pages::Store = undefined;
27 +
/// Interrupt ownership passed to terminal identity reaping.
28 +
unsafe static IRQS: interrupts::Store = undefined;
26 29
/// Mapped frames available to interrupted page calls.
27 30
static RAM: [u64; 4096] = [0; 4096];
28 31
/// Reservations retained across private initialization.
29 32
unsafe static CALLS: lifecycle::Calls = undefined;
30 33
/// Shared queue storage for the first receiver.
277 280
    retain(4, 0);
278 281
    assert lifecycle::service(&mut STORE, &WINDOWS, 100) == 0;
279 282
    assert STORE.events.terminals[4].state == events::State::Queued;
280 283
}
281 284
285 +
/// Maintenance acknowledges the last queued exit while its receiver keeps running.
286 +
@test unsafe fn queuedAcknowledgement() throws (testing::TestError) {
287 +
    initialize();
288 +
    let subject = abi::Ref { index: 2, generation: 1 };
289 +
    set STORE.slots[2] = slots::Slot { generation: 1, state: slots::State::Live };
290 +
    set STORE.records[2].state = domains::Lifecycle::Dead;
291 +
    set STORE.reclaimed[2] = true;
292 +
    set STORE.dead = 1;
293 +
    retain(2, 0);
294 +
    assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 2;
295 +
    assert STORE.events.pending == 0 and STORE.events.terminals[2].state == events::State::Queued;
296 +
    assert STORE.events.queued == 1;
297 +
    assert not lifecycle::reapNext(&mut STORE, &mut MEMORY, &mut WINDOWS, &mut IRQS);
298 +
    assert lifecycle::service(&mut STORE, &WINDOWS, 21) == 0;
299 +
    assert STORE.events.terminals[2].state == events::State::Queued;
300 +
    let event = try! events::pop(&mut FIRST.ring) else panic "terminal event";
301 +
    assert event.value == abi::id(subject);
302 +
    assert STORE.contexts[0].state == domains::ContextState::Ready;
303 +
    assert lifecycle::service(&mut STORE, &WINDOWS, 22) == 0;
304 +
    assert try! events::acknowledged(&STORE.events, subject);
305 +
    assert STORE.events.queued == 0;
306 +
    assert lifecycle::reapNext(&mut STORE, &mut MEMORY, &mut WINDOWS, &mut IRQS);
307 +
    assert STORE.dead == 0 and STORE.slots[2].state == slots::State::Free;
308 +
    assert STORE.events.terminals[2].state == events::State::Free;
309 +
}
310 +
282 311
/// Full receivers retain terminal events while other receivers make progress.
283 312
@test unsafe fn saturation() throws (testing::TestError) {
284 313
    initialize();
285 314
    let receiver = abi::Ref { index: 0, generation: 1 };
286 315
    for i in 0..events::ORDINARY {
302 331
    assert STORE.events.queues[0].tail == events::CAPACITY + 1;
303 332
    retain(4, 0);
304 333
    assert STORE.events.pending == 1;
305 334
    try! events::close(&mut STORE.events, receiver);
306 335
    assert STORE.events.pending == 0;
336 +
    assert STORE.events.queued == 1;
337 +
    try! events::close(&mut STORE.events, abi::Ref { index: 1, generation: 1 });
338 +
    assert STORE.events.queued == 0;
307 339
}