kernel: Refresh queued terminal acknowledgements

bf3d3cc56d54b3ae4270c496de6ecc86032cd9c8b0df0e2ef8ec0addcbcb9678
Alexis Sellier committed ago 1 parent fa1aa037
kernel/kernel/dispatch.rad +25 -11
51 51
    state: State,
52 52
    /// Validated timer registers for this hart.
53 53
    timer: Timer,
54 54
    /// Minimum ticks between scheduled maintenance passes.
55 55
    maintenance: u64,
56 -
    /// Earliest tick for another background maintenance pass.
57 -
    nextMaintenance: u64,
58 56
}
59 57
60 58
/// Runtime slots initialized before each hart enables timer interrupts.
61 59
unsafe static HARTS: [Runtime; limits::HARTS] = undefined;
60 +
/// Earliest shared maintenance tick, serialized by the metadata lock.
61 +
unsafe static NEXT_MAINTENANCE: u64 = 0;
62 +
/// Hart with a timer armed no later than the shared maintenance deadline.
63 +
unsafe static MAINTENANCE_HART: ?u32 = nil;
62 64
63 65
/// Read the executing physical hart identifier.
64 66
export fn hart() -> u32;
65 67
/// Read the current machine stack pointer.
66 68
fn stackPointer() -> u64;
81 83
    let mut maintenance = machine.timebase as u64 / 100;
82 84
    if maintenance == 0 {
83 85
        set maintenance = 1;
84 86
    }
85 87
    set HARTS[id] = Runtime {
86 -
        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, nextMaintenance: 0,
88 +
        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance,
87 89
    };
88 90
    arm(device, 0xffffffffffffffff);
89 91
}
90 92
91 93
/// Tail-enter dispatch on the firmware stack and resume the selected frame.
97 99
    assert id < limits::HARTS;
98 100
    let stack = HARTS[id].state.stack;
99 101
    let pointer = stackPointer();
100 102
    assert pointer >= stack.start and pointer < stack.end;
101 103
    let guard = sync::enter();
104 +
    if MAINTENANCE_HART == id {
105 +
        set MAINTENANCE_HART = nil;
106 +
    }
102 107
    writeSoftware(HARTS[id].timer.software, 0);
103 108
    let previous = HARTS[id].state.current;
104 109
    save(&mut HARTS[id].state, &mut domains::STORE, frame);
105 110
    if trap::classify(frame.cause) == trap::Cause::External {
106 111
        plic::claim(id);
122 127
        }
123 128
        set deliveries[count] = delivery;
124 129
        set count += 1;
125 130
    }
126 131
    let clock = now(HARTS[id].timer);
132 +
    let maintenanceDue = clock >= NEXT_MAINTENANCE;
127 133
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
128 -
    let notified = timeout.harts | plic::service(clock) | lifecycle::service(&mut domains::STORE, &budgets::STORE, clock);
134 +
    let mut terminals: u32 = 0;
135 +
    if domains::STORE.events.pending > 0 or maintenanceDue {
136 +
        set terminals = lifecycle::service(&mut domains::STORE, &budgets::STORE, clock);
137 +
    }
138 +
    let notified = timeout.harts | plic::service(clock) | terminals;
129 139
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
130 140
    let mut deadline = choice.deadline;
131 141
    if timeout.deadline < deadline {
132 142
        set deadline = timeout.deadline;
133 143
    }
149 159
            signalContext(context, clock);
150 160
        }
151 161
    }
152 162
    lifecycle::cancelRequests(&mut domains::STORE, &mut remote::STORE);
153 163
    let cancelled = lifecycle::cancelAllocations(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE, &mut loader::STATE, &mut registry::STORE);
154 -
    let maintenanceClock = now(HARTS[id].timer);
155 164
    let mut maintenance = timeout.pending or cancelled or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0;
156 -
    if maintenance and maintenanceClock >= HARTS[id].nextMaintenance {
165 +
    if maintenance and maintenanceDue {
157 166
        let reaped = lifecycle::reapNext(&mut domains::STORE, &mut pages::STORE, &mut budgets::STORE, &mut interrupts::STORE);
158 167
        let reclaimed = frames::reclaim(&mut pages::STORE.backings.pool, 256);
159 -
        set maintenance = timeout.pending or cancelled or reaped or pages::STORE.backings.pool.retiredCount > 0;
168 +
        set maintenance = timeout.pending or cancelled or reaped or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0;
160 169
        let after = now(HARTS[id].timer);
161 170
        let interval = HARTS[id].maintenance;
162 -
        set HARTS[id].nextMaintenance = 0xffffffffffffffff;
171 +
        set NEXT_MAINTENANCE = 0xffffffffffffffff;
163 172
        if interval < 0xffffffffffffffff - after {
164 -
            set HARTS[id].nextMaintenance = after + interval;
173 +
            set NEXT_MAINTENANCE = after + interval;
165 174
        }
166 175
    }
167 -
    if maintenance and HARTS[id].nextMaintenance < deadline {
168 -
        set deadline = HARTS[id].nextMaintenance;
176 +
    if maintenance and (timeout.pending or MAINTENANCE_HART == nil) {
177 +
        if MAINTENANCE_HART == nil {
178 +
            set MAINTENANCE_HART = id;
179 +
        }
180 +
        if NEXT_MAINTENANCE < deadline {
181 +
            set deadline = NEXT_MAINTENANCE;
182 +
        }
169 183
    }
170 184
    arm(HARTS[id].timer, deadline);
171 185
    sync::leave(guard);
172 186
}
173 187
kernel/kernel/events.rad +10 -0
85 85
86 86
/// Fixed event metadata. Callers serialize all kernel-side mutations.
87 87
export record Store: Copy {
88 88
    /// Terminal events retained for publication into receiver rings.
89 89
    pending: u32,
90 +
    /// Published terminal events that await consumer acknowledgement.
91 +
    queued: u32,
90 92
    /// Producer metadata indexed by receiver domain.
91 93
    queues: [Queue; limits::DOMAINS],
92 94
    /// Terminal storage indexed by the terminating domain.
93 95
    terminals: [Terminal; limits::DOMAINS],
94 96
}
110 112
}
111 113
112 114
/// Initialize metadata before any queue or terminal record exists.
113 115
export fn initialize(store: &mut Store) {
114 116
    set store.pending = 0;
117 +
    set store.queued = 0;
115 118
    for i in 0..limits::DOMAINS {
116 119
        set store.queues[i].generation = 0;
117 120
        set store.terminals[i].state = State::Free;
118 121
    }
119 122
}
158 161
            let terminal = &mut store.terminals[subject.index];
159 162
            assert terminal.subject == subject
160 163
                and terminal.receiver == receiver
161 164
                and terminal.state == State::Queued;
162 165
            set terminal.state = State::Acknowledged;
166 +
            assert store.queued > 0;
167 +
            set store.queued -= 1;
163 168
        }
164 169
        set queue.head = next(queue.head);
165 170
    }
166 171
}
167 172
287 292
    let queue = &mut store.queues[terminal.receiver.index];
288 293
    if distance(queue.tail, queue.head) == CAPACITY {
289 294
        return false;
290 295
    }
291 296
    set store.terminals[subject.index].state = State::Queued;
297 +
    set store.queued += 1;
292 298
    assert store.pending > 0;
293 299
    set store.pending -= 1;
294 300
    publish(queue, ring, terminal.event, subject);
295 301
    return true;
296 302
}
368 374
        if generations[receiver.index] == receiver.generation {
369 375
            if store.terminals[i].state == State::Pending {
370 376
                assert store.pending > 0;
371 377
                set store.pending -= 1;
372 378
            }
379 +
            else {
380 +
                assert store.queued > 0;
381 +
                set store.queued -= 1;
382 +
            }
373 383
            set store.terminals[i].state = State::Acknowledged;
374 384
        }
375 385
    }
376 386
}
kernel/kernel/lifecycle.rad +11 -4
494 494
    }
495 495
    try! domains::retireStorage(store, &mut memory.backings, owner);
496 496
    return true;
497 497
}
498 498
499 -
/// Publish retained terminal events and report receiver harts under the metadata lock.
499 +
/// Refresh terminal acknowledgements, publish pending events, and report receiver harts.
500 +
/// The caller holds the metadata lock throughout service.
500 501
export unsafe fn service(store: &mut domains::Store, windows: &budgets::Store, now: u64) -> u32 {
501 -
    if store.events.pending == 0 {
502 +
    if store.events.pending == 0 and store.events.queued == 0 {
502 503
        return 0;
503 504
    }
504 505
    let mut notified: [bool; limits::DOMAINS] = [false; limits::DOMAINS];
505 506
    let mut changed = false;
506 507
    for i in 0..limits::DOMAINS {
507 -
        let terminal = store.events.terminals[i];
508 -
        if terminal.state <> events::State::Pending {
508 +
        let state = store.events.terminals[i].state;
509 +
        if state <> events::State::Pending and state <> events::State::Queued {
509 510
            continue;
510 511
        }
512 +
        let terminal = store.events.terminals[i];
511 513
        let mut receiver = try domains::get(store, terminal.receiver) catch {
512 514
            continue;
513 515
        };
514 516
        if receiver.state == domains::Lifecycle::Dead {
515 517
            continue;
516 518
        }
519 +
        if terminal.state == events::State::Queued {
520 +
            try events::refresh(&mut store.events, terminal.receiver, &receiver.memory.ring) catch {
521 +
            };
522 +
            continue;
523 +
        }
517 524
        let delivered = try events::deliver(&mut store.events, terminal.subject, &mut receiver.memory.ring) catch {
518 525
            false
519 526
        };
520 527
        if delivered {
521 528
            set notified[terminal.receiver.index] = true;
kernel/kernel/tests/lifecycle.rad +32 -0
12 12
use kernel::remote;
13 13
use kernel::pages;
14 14
use kernel::backing;
15 15
use kernel::capability;
16 16
use kernel::frames;
17 +
use kernel::interrupts;
17 18
18 19
/// Receiver generations, contexts, and terminal events.
19 20
unsafe static STORE: domains::Store = undefined;
20 21
/// CPU authority retained by receivers.
21 22
unsafe static WINDOWS: budgets::Store = undefined;
22 23
/// Queued calls and acknowledgements retained by stopped contexts.
23 24
unsafe static REQUESTS: remote::Store = undefined;
24 25
/// Physical allocation metadata used for interrupted page calls.
25 26
unsafe static MEMORY: pages::Store = undefined;
27 +
/// Interrupt ownership passed to terminal identity reaping.
28 +
unsafe static IRQS: interrupts::Store = undefined;
26 29
/// Mapped frames available to interrupted page calls.
27 30
static RAM: [u64; 4096] = [0; 4096];
28 31
/// Reservations retained across private initialization.
29 32
unsafe static CALLS: lifecycle::Calls = undefined;
30 33
/// Shared queue storage for the first receiver.
297 300
    retain(4, 0);
298 301
    assert lifecycle::service(&mut STORE, &WINDOWS, 100) == 0;
299 302
    assert STORE.events.terminals[4].state == events::State::Queued;
300 303
}
301 304
305 +
/// Maintenance acknowledges the last queued exit while its receiver keeps running.
306 +
@test unsafe fn queuedAcknowledgement() throws (testing::TestError) {
307 +
    initialize();
308 +
    let subject = abi::Ref { index: 2, generation: 1 };
309 +
    set STORE.slots[2] = slots::Slot { generation: 1, state: slots::State::Live };
310 +
    set STORE.records[2].state = domains::Lifecycle::Dead;
311 +
    set STORE.reclaimed[2] = true;
312 +
    set STORE.dead = 1;
313 +
    retain(2, 0);
314 +
    assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 2;
315 +
    assert STORE.events.pending == 0 and STORE.events.terminals[2].state == events::State::Queued;
316 +
    assert STORE.events.queued == 1;
317 +
    assert not lifecycle::reapNext(&mut STORE, &mut MEMORY, &mut WINDOWS, &mut IRQS);
318 +
    assert lifecycle::service(&mut STORE, &WINDOWS, 21) == 0;
319 +
    assert STORE.events.terminals[2].state == events::State::Queued;
320 +
    let event = try! events::pop(&mut FIRST.ring) else panic "terminal event";
321 +
    assert event.value == abi::id(subject);
322 +
    assert STORE.contexts[0].state == domains::ContextState::Ready;
323 +
    assert lifecycle::service(&mut STORE, &WINDOWS, 22) == 0;
324 +
    assert try! events::acknowledged(&STORE.events, subject);
325 +
    assert STORE.events.queued == 0;
326 +
    assert lifecycle::reapNext(&mut STORE, &mut MEMORY, &mut WINDOWS, &mut IRQS);
327 +
    assert STORE.dead == 0 and STORE.slots[2].state == slots::State::Free;
328 +
    assert STORE.events.terminals[2].state == events::State::Free;
329 +
}
330 +
302 331
/// Full receivers retain terminal events while other receivers make progress.
303 332
@test unsafe fn saturation() throws (testing::TestError) {
304 333
    initialize();
305 334
    let receiver = abi::Ref { index: 0, generation: 1 };
306 335
    for i in 0..events::ORDINARY {
324 353
    assert STORE.events.queues[0].tail == events::CAPACITY + 1;
325 354
    retain(4, 0);
326 355
    assert STORE.events.pending == 1;
327 356
    try! events::close(&mut STORE.events, receiver);
328 357
    assert STORE.events.pending == 0;
358 +
    assert STORE.events.queued == 1;
359 +
    try! events::close(&mut STORE.events, abi::Ref { index: 1, generation: 1 });
360 +
    assert STORE.events.queued == 0;
329 361
}