test/cycles/kernel/dispatchcheck.rad 11.2 KiB raw
1
//! Concurrent private package graphs and complete worker resource recovery.
2
export mod inputs;
3
export mod invariants;
4
5
use kernel::calls;
6
use kernel::fixture;
7
use kernel::limits;
8
use kernel::state;
9
use kernel::abi;
10
use kernel::frames;
11
use kernel::slots;
12
use kernel::pages;
13
use kernel::capability;
14
use kernel::registry;
15
use kernel::domains;
16
use kernel::budgets;
17
use kernel::dispatch;
18
use kernel::boot;
19
use kernel::sync;
20
use kernel::remote;
21
use kernel::events;
22
use kernel::devices;
23
use kernel::platform;
24
25
/// Delay before controllers start, scaled by online hart count.
26
constant START_DELAY: u64 = 1000000;
27
28
/// Duration reserved per worker, scaled by online hart count.
29
constant WORKER_TICKS: u64 = 10000000;
30
31
/// Duration reserved per controller, scaled by online hart count.
32
constant CONTROLLER_TICKS: u64 = 800000000;
33
34
/// Maximum maintenance requests before recovery must finish.
35
constant RECLAIM_ATTEMPTS: u32 = 100000;
36
37
/// Maximum frame or slot entries inspected in one metadata section.
38
constant FRAME_BATCH: u32 = 128;
39
40
// Setup initializes retained state for online harts before releasing READY.
41
// Other harts must acquire READY before they read this state.
42
/// Bootstrap resource capabilities.
43
unsafe static TABLE: capability::Table = undefined;
44
/// Surviving event consumer.
45
unsafe static PARENT: abi::Ref = undefined;
46
/// Controller identities indexed by hart.
47
unsafe static CHILDREN: [fixture::Actor; limits::HARTS] = undefined;
48
/// Bootstrap-owned argument page addresses indexed by hart.
49
unsafe static ARGUMENTS: [u64; limits::HARTS] = [0; limits::HARTS];
50
/// Free frames before controller creation.
51
unsafe static FREE: u32 = 0;
52
/// Reusable page, backing, domain, and context slots before controller
53
/// creation.
54
unsafe static SLOTS: [u32; 4] = [0; 4];
55
/// Fixture publication barrier.
56
static READY: u64 = 0;
57
/// Validated physical page mapping.
58
fn memory(address: u64) -> *mut u8;
59
/// Current kernel package-state table.
60
fn kernelGp() -> u64;
61
/// Native checker entry.
62
fn completion() -> u64;
63
/// Write one diagnostic byte.
64
fn put(byte: u8);
65
/// Finish the native fixture.
66
fn finish();
67
/// Sleep until this hart's software interrupt is pending. Interrupts stay
68
/// globally masked, so the interrupt wakes the hart without trapping.
69
fn park();
70
71
/// Count reusable frames in short metadata sections.
72
unsafe fn freeFrames() -> u32 {
73
    let mut count: u32 = 0;
74
    let mut first: u32 = 0;
75
    while first < state::SHARED.pages.backings.pool.count {
76
        let mut end = first + FRAME_BATCH;
77
        if end > state::SHARED.pages.backings.pool.count {
78
            set end = state::SHARED.pages.backings.pool.count;
79
        }
80
        let guard = sync::enter();
81
        for i in first .. end {
82
            if frames::available(&state::SHARED.pages.backings.pool, i) {
83
                set count += 1;
84
            }
85
        }
86
        sync::leave(guard);
87
        set first = end;
88
    }
89
    return count;
90
}
91
92
/// Count reusable object slots after all allocation transactions have stopped.
93
fn freeSlots(entries: &[slots::Slot]) -> u32 {
94
    let mut count: u32 = 0;
95
    let mut first: u32 = 0;
96
    while first < entries.len {
97
        let mut end = first + FRAME_BATCH;
98
        if end > entries.len {
99
            set end = entries.len;
100
        }
101
        let guard = sync::enter();
102
        for i in first .. end {
103
            assert entries[i].state <> slots::State::Reserved;
104
            if entries[i].state == slots::State::Free {
105
                set count += 1;
106
            }
107
        }
108
        sync::leave(guard);
109
        set first = end;
110
    }
111
    return count;
112
}
113
114
/// Read object capacity at the quiescent boundaries of the workload.
115
unsafe fn capacity() -> [u32; 4] {
116
    return [
117
        freeSlots(&state::SHARED.pages.slots[..]),
118
        freeSlots(&state::SHARED.pages.backings.slots[..]),
119
        freeSlots(&state::SHARED.domains.slots[..]),
120
        freeSlots(&state::SHARED.domains.contextSlots[..]),
121
    ];
122
}
123
124
/// Publish fixture state before any hart enters dispatch.
125
export unsafe fn start(last: bool) {
126
    if last {
127
        setup();
128
        sync::storeRelease(&mut READY, 1);
129
        // Every online hart finished dispatch::initialize before arriving.
130
        for hart in 0 .. limits::HARTS {
131
            if hart <> dispatch::hart() and (boot::PLATFORM.harts & (1 << hart))
132
                <> 0
133
            {
134
                dispatch::signal(hart);
135
            }
136
        }
137
    }
138
    while sync::loadAcquire(&READY) == 0 {
139
        park();
140
    }
141
}
142
143
/// Prepare one controller per hart and a later surviving checker.
144
unsafe fn setup() {
145
    let authority = fixture::bootstrap(&mut TABLE);
146
    let image = fixture::load(&mut TABLE, authority, &inputs::spin[..]);
147
    let source = fixture::input(&mut TABLE, authority, &inputs::job[..]);
148
    let parentHandle = try! domains::create(
149
        &mut state::SHARED.domains,
150
        &mut state::SHARED.pages.backings,
151
        &state::SHARED.registry,
152
        &mut TABLE,
153
        authority,
154
        image
155
    );
156
    set PARENT = (try! capability::get(&TABLE, parentHandle)).object;
157
    let parent = try! domains::get(&state::SHARED.domains, PARENT);
158
    fixture::machine(PARENT, completion(), kernelGp());
159
    let mut arguments: [abi::Handle; limits::HARTS] = undefined;
160
    let mut count: u64 = 0;
161
    for hart in 0 .. limits::HARTS {
162
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
163
            continue;
164
        }
165
        set count += 1;
166
        set arguments[hart] = try! pages::allocate(
167
            &mut state::SHARED.pages, &mut TABLE, authority, 1
168
        );
169
        set ARGUMENTS[hart] =
170
            (try! pages::get(
171
                &state::SHARED.pages, (try! capability::get(&TABLE, arguments[hart])).object
172
            )).base;
173
    }
174
    set FREE = freeFrames();
175
    set SLOTS = capacity();
176
    for hart in 0 .. limits::HARTS {
177
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
178
            continue;
179
        }
180
        let actor = fixture::create(&mut TABLE, authority, image);
181
        let handle = actor.handle;
182
        let object = actor.domain;
183
        let mut child = try! domains::get(&state::SHARED.domains, object);
184
        set CHILDREN[hart] = actor;
185
        try! domains::reparent(
186
            &mut state::SHARED.domains, &TABLE, handle, parentHandle
187
        );
188
        let self = try! capability::install(&mut child.memory.table, capability::Entry {
189
            kind: abi::Kind::Domain, object, rights: abi::Rights(abi::CREATE | abi::ALLOCATE)
190
        });
191
        let stack = try! pages::allocate(
192
            &mut state::SHARED.pages, &mut child.memory.table, self, 4
193
        );
194
        let args = @sliceOf(memory(ARGUMENTS[hart]) as *mut u64, 6);
195
        set args[0] = *try! pages::grant(
196
            &mut state::SHARED.pages, &TABLE, &mut child.memory.table, source, abi::READ as u64
197
        );
198
        set args[1] = inputs::job.len as u64;
199
        for i in 0 .. state::SHARED.devices.count {
200
            if state::SHARED.devices.regions[i].kind == platform::Kind::Clint {
201
                set args[2] = *try! devices::install(
202
                    &state::SHARED.devices, &mut child.memory.table, i
203
                );
204
            }
205
        }
206
        assert args[2] <> 0;
207
        let page = try! pages::grant(
208
            &mut state::SHARED.pages,
209
            &TABLE,
210
            &mut child.memory.table,
211
            arguments[hart],
212
            (abi::READ | abi::WRITE | abi::GRANT) as u64
213
        );
214
        set args[3] = *page;
215
        set args[4] = WORKER_TICKS * count;
216
        try! fixture::activate(&TABLE, handle, stack, page, 56);
217
    }
218
    for byte in "cycles controllers ready\n" {
219
        put(byte);
220
    }
221
    let now = dispatch::clock();
222
    let start = now + START_DELAY * count;
223
    for hart in 0 .. limits::HARTS {
224
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
225
            continue;
226
        }
227
        let window = try! budgets::seed(
228
            &mut state::SHARED.budgets, &mut TABLE, hart, start, 0xffffffffffffffff
229
        );
230
        let tail = try! budgets::split(
231
            &mut state::SHARED.budgets, &mut TABLE, window, start + CONTROLLER_TICKS * count, now
232
        );
233
        try! calls::rebind(
234
            &mut state::SHARED,
235
            &mut TABLE,
236
            budgets::Binding {
237
                budget: window, domain: CHILDREN[hart].handle, context: CHILDREN[hart].context
238
            },
239
            now
240
        );
241
        if hart == 0 {
242
            let final = try! calls::rebind(
243
                &mut state::SHARED,
244
                &mut TABLE,
245
                budgets::Binding {
246
                    budget: tail, domain: parentHandle, context: parent.initial
247
                },
248
                now
249
            );
250
        }
251
    }
252
}
253
254
/// Check all terminal records, shared code identity, and transient recovery.
255
/// Only the surviving checker runs these phases after the worker windows end.
256
export unsafe fn verify() {
257
    let retained = checkWorkers();
258
    checkRecovery(retained);
259
    invariants::check(&TABLE);
260
    for byte in "cycles final recovery and invariants passed\n" {
261
        put(byte);
262
    }
263
    finish();
264
}
265
266
/// Check terminal records and package reuse under the metadata lock.
267
unsafe fn checkWorkers() -> u32 {
268
    for byte in "cycles checking completion\n" {
269
        put(byte);
270
    }
271
    let guard = sync::enter();
272
    let mut parent = try! domains::get(&state::SHARED.domains, PARENT);
273
    let job = registry::find(
274
        &registry::capture(&state::SHARED.registry), &"job"[..]
275
    ) else panic "checkWorkers: worker package required";
276
    let package = try! registry::get(&state::SHARED.registry, job);
277
    let mut seen: u32 = 0;
278
    while let event = try! events::pop(&mut parent.memory.ring) {
279
        assert event.kind == events::CHILD_EXIT and event.code == 68;
280
        let mut found = false;
281
        for hart in 0 .. limits::HARTS {
282
            if (boot::PLATFORM.harts & (1 << hart)) == 0 {
283
                continue;
284
            }
285
            if event.value == abi::id(CHILDREN[hart].domain) {
286
                assert (seen & (1 << hart)) == 0;
287
                set seen |= 1 << hart;
288
                set found = true;
289
                let args = @sliceOf(memory(ARGUMENTS[hart]) as *mut u64, 7);
290
                assert args[5] == package.codeAddress and args[6] == 4;
291
            }
292
        }
293
        assert found;
294
    }
295
    assert seen == boot::PLATFORM.harts;
296
    try! events::refresh(
297
        &mut state::SHARED.domains.events, PARENT, &parent.memory.ring
298
    );
299
    assert state::SHARED.loader.live[job.index] and state::SHARED.loader.owner
300
        == nil;
301
    let retained = state::SHARED.loader.resident[job.index].code.count
302
        + state::SHARED.loader.resident[job.index].metadata.count;
303
    sync::leave(guard);
304
    return retained;
305
}
306
307
/// Reap stopped domains, then compare frame and object capacity at quiescence.
308
unsafe fn checkRecovery(retained: u32) {
309
    let mut done = false;
310
    for attempt in 0 .. RECLAIM_ATTEMPTS {
311
        let guard = sync::enter();
312
        set done = state::SHARED.domains.dead == 0
313
            and state::SHARED.pages.backings.pool.retiredCount == 0;
314
        if done {
315
            sync::leave(guard);
316
            break;
317
        }
318
        let request = try! dispatch::request(
319
            &mut state::SHARED.remote, 0, remote::Action::Reschedule
320
        );
321
        sync::leave(guard);
322
        assert try! dispatch::awaitRequest(0, request) == 0;
323
    }
324
    assert done;
325
    assert freeFrames() + retained == FREE;
326
    let recovered = capacity();
327
    for i in 0 .. 4 {
328
        assert recovered[i] == SLOTS[i];
329
    }
330
}