.githooks/
compiler/
kernel/
lib/
scripts/
seed/
sublime/
test/
boot/
bootstrap/
compiler/
cycles/
kernel/
dispatchcheck.rad
11.2 KiB
job.rad
351 B
machine.ras
919 B
spin.rad
3.9 KiB
support.rad
256 B
dispatch/
format/
host/
integration/
kernel/
loader/
manifests/
mmio/
native/
packages/
runtime/
scheduling/
shared/
slots/
smp/
std/
sync/
termination/
tests/
trap/
verify/
Makefile
9.2 KiB
kernel.rad
776 B
runner.rad
11.6 KiB
vim/
.gitignore
333 B
.gitsigners
224 B
AGENTS.md
8.3 KiB
CONTRIBUTING
2.1 KiB
LICENSE
1.1 KiB
Makefile
2.5 KiB
README
3.4 KiB
STYLE
3.2 KiB
std.lib
2.1 KiB
std.lib.test
1.1 KiB
test/cycles/kernel/dispatchcheck.rad
raw
| 1 | //! Concurrent private package graphs and complete worker resource recovery. |
| 2 | export mod inputs; |
| 3 | export mod invariants; |
| 4 | |
| 5 | use kernel::calls; |
| 6 | use kernel::fixture; |
| 7 | use kernel::limits; |
| 8 | use kernel::state; |
| 9 | use kernel::abi; |
| 10 | use kernel::frames; |
| 11 | use kernel::slots; |
| 12 | use kernel::pages; |
| 13 | use kernel::capability; |
| 14 | use kernel::registry; |
| 15 | use kernel::domains; |
| 16 | use kernel::budgets; |
| 17 | use kernel::dispatch; |
| 18 | use kernel::boot; |
| 19 | use kernel::sync; |
| 20 | use kernel::remote; |
| 21 | use kernel::events; |
| 22 | use kernel::devices; |
| 23 | use kernel::platform; |
| 24 | |
| 25 | /// Delay before controllers start, scaled by online hart count. |
| 26 | constant START_DELAY: u64 = 1000000; |
| 27 | |
| 28 | /// Duration reserved per worker, scaled by online hart count. |
| 29 | constant WORKER_TICKS: u64 = 10000000; |
| 30 | |
| 31 | /// Duration reserved per controller, scaled by online hart count. |
| 32 | constant CONTROLLER_TICKS: u64 = 800000000; |
| 33 | |
| 34 | /// Maximum maintenance requests before recovery must finish. |
| 35 | constant RECLAIM_ATTEMPTS: u32 = 100000; |
| 36 | |
| 37 | /// Maximum frame or slot entries inspected in one metadata section. |
| 38 | constant FRAME_BATCH: u32 = 128; |
| 39 | |
| 40 | // Setup initializes retained state for online harts before releasing READY. |
| 41 | // Other harts must acquire READY before they read this state. |
| 42 | /// Bootstrap resource capabilities. |
| 43 | unsafe static TABLE: capability::Table = undefined; |
| 44 | /// Surviving event consumer. |
| 45 | unsafe static PARENT: abi::Ref = undefined; |
| 46 | /// Controller identities indexed by hart. |
| 47 | unsafe static CHILDREN: [fixture::Actor; limits::HARTS] = undefined; |
| 48 | /// Bootstrap-owned argument page addresses indexed by hart. |
| 49 | unsafe static ARGUMENTS: [u64; limits::HARTS] = [0; limits::HARTS]; |
| 50 | /// Free frames before controller creation. |
| 51 | unsafe static FREE: u32 = 0; |
| 52 | /// Reusable page, backing, domain, and context slots before controller |
| 53 | /// creation. |
| 54 | unsafe static SLOTS: [u32; 4] = [0; 4]; |
| 55 | /// Fixture publication barrier. |
| 56 | static READY: u64 = 0; |
| 57 | /// Validated physical page mapping. |
| 58 | fn memory(address: u64) -> *mut u8; |
| 59 | /// Current kernel package-state table. |
| 60 | fn kernelGp() -> u64; |
| 61 | /// Native checker entry. |
| 62 | fn completion() -> u64; |
| 63 | /// Write one diagnostic byte. |
| 64 | fn put(byte: u8); |
| 65 | /// Finish the native fixture. |
| 66 | fn finish(); |
| 67 | /// Sleep until this hart's software interrupt is pending. Interrupts stay |
| 68 | /// globally masked, so the interrupt wakes the hart without trapping. |
| 69 | fn park(); |
| 70 | |
| 71 | /// Count reusable frames in short metadata sections. |
| 72 | unsafe fn freeFrames() -> u32 { |
| 73 | let mut count: u32 = 0; |
| 74 | let mut first: u32 = 0; |
| 75 | while first < state::SHARED.pages.backings.pool.count { |
| 76 | let mut end = first + FRAME_BATCH; |
| 77 | if end > state::SHARED.pages.backings.pool.count { |
| 78 | set end = state::SHARED.pages.backings.pool.count; |
| 79 | } |
| 80 | let guard = sync::enter(); |
| 81 | for i in first .. end { |
| 82 | if frames::available(&state::SHARED.pages.backings.pool, i) { |
| 83 | set count += 1; |
| 84 | } |
| 85 | } |
| 86 | sync::leave(guard); |
| 87 | set first = end; |
| 88 | } |
| 89 | return count; |
| 90 | } |
| 91 | |
| 92 | /// Count reusable object slots after all allocation transactions have stopped. |
| 93 | fn freeSlots(entries: &[slots::Slot]) -> u32 { |
| 94 | let mut count: u32 = 0; |
| 95 | let mut first: u32 = 0; |
| 96 | while first < entries.len { |
| 97 | let mut end = first + FRAME_BATCH; |
| 98 | if end > entries.len { |
| 99 | set end = entries.len; |
| 100 | } |
| 101 | let guard = sync::enter(); |
| 102 | for i in first .. end { |
| 103 | assert entries[i].state <> slots::State::Reserved; |
| 104 | if entries[i].state == slots::State::Free { |
| 105 | set count += 1; |
| 106 | } |
| 107 | } |
| 108 | sync::leave(guard); |
| 109 | set first = end; |
| 110 | } |
| 111 | return count; |
| 112 | } |
| 113 | |
| 114 | /// Read object capacity at the quiescent boundaries of the workload. |
| 115 | unsafe fn capacity() -> [u32; 4] { |
| 116 | return [ |
| 117 | freeSlots(&state::SHARED.pages.slots[..]), |
| 118 | freeSlots(&state::SHARED.pages.backings.slots[..]), |
| 119 | freeSlots(&state::SHARED.domains.slots[..]), |
| 120 | freeSlots(&state::SHARED.domains.contextSlots[..]), |
| 121 | ]; |
| 122 | } |
| 123 | |
| 124 | /// Publish fixture state before any hart enters dispatch. |
| 125 | export unsafe fn start(last: bool) { |
| 126 | if last { |
| 127 | setup(); |
| 128 | sync::storeRelease(&mut READY, 1); |
| 129 | // Every online hart finished dispatch::initialize before arriving. |
| 130 | for hart in 0 .. limits::HARTS { |
| 131 | if hart <> dispatch::hart() and (boot::PLATFORM.harts & (1 << hart)) |
| 132 | <> 0 |
| 133 | { |
| 134 | dispatch::signal(hart); |
| 135 | } |
| 136 | } |
| 137 | } |
| 138 | while sync::loadAcquire(&READY) == 0 { |
| 139 | park(); |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | /// Prepare one controller per hart and a later surviving checker. |
| 144 | unsafe fn setup() { |
| 145 | let authority = fixture::bootstrap(&mut TABLE); |
| 146 | let image = fixture::load(&mut TABLE, authority, &inputs::spin[..]); |
| 147 | let source = fixture::input(&mut TABLE, authority, &inputs::job[..]); |
| 148 | let parentHandle = try! domains::create( |
| 149 | &mut state::SHARED.domains, |
| 150 | &mut state::SHARED.pages.backings, |
| 151 | &state::SHARED.registry, |
| 152 | &mut TABLE, |
| 153 | authority, |
| 154 | image |
| 155 | ); |
| 156 | set PARENT = (try! capability::get(&TABLE, parentHandle)).object; |
| 157 | let parent = try! domains::get(&state::SHARED.domains, PARENT); |
| 158 | fixture::machine(PARENT, completion(), kernelGp()); |
| 159 | let mut arguments: [abi::Handle; limits::HARTS] = undefined; |
| 160 | let mut count: u64 = 0; |
| 161 | for hart in 0 .. limits::HARTS { |
| 162 | if (boot::PLATFORM.harts & (1 << hart)) == 0 { |
| 163 | continue; |
| 164 | } |
| 165 | set count += 1; |
| 166 | set arguments[hart] = try! pages::allocate( |
| 167 | &mut state::SHARED.pages, &mut TABLE, authority, 1 |
| 168 | ); |
| 169 | set ARGUMENTS[hart] = |
| 170 | (try! pages::get( |
| 171 | &state::SHARED.pages, (try! capability::get(&TABLE, arguments[hart])).object |
| 172 | )).base; |
| 173 | } |
| 174 | set FREE = freeFrames(); |
| 175 | set SLOTS = capacity(); |
| 176 | for hart in 0 .. limits::HARTS { |
| 177 | if (boot::PLATFORM.harts & (1 << hart)) == 0 { |
| 178 | continue; |
| 179 | } |
| 180 | let actor = fixture::create(&mut TABLE, authority, image); |
| 181 | let handle = actor.handle; |
| 182 | let object = actor.domain; |
| 183 | let mut child = try! domains::get(&state::SHARED.domains, object); |
| 184 | set CHILDREN[hart] = actor; |
| 185 | try! domains::reparent( |
| 186 | &mut state::SHARED.domains, &TABLE, handle, parentHandle |
| 187 | ); |
| 188 | let self = try! capability::install(&mut child.memory.table, capability::Entry { |
| 189 | kind: abi::Kind::Domain, object, rights: abi::Rights(abi::CREATE | abi::ALLOCATE) |
| 190 | }); |
| 191 | let stack = try! pages::allocate( |
| 192 | &mut state::SHARED.pages, &mut child.memory.table, self, 4 |
| 193 | ); |
| 194 | let args = @sliceOf(memory(ARGUMENTS[hart]) as *mut u64, 6); |
| 195 | set args[0] = *try! pages::grant( |
| 196 | &mut state::SHARED.pages, &TABLE, &mut child.memory.table, source, abi::READ as u64 |
| 197 | ); |
| 198 | set args[1] = inputs::job.len as u64; |
| 199 | for i in 0 .. state::SHARED.devices.count { |
| 200 | if state::SHARED.devices.regions[i].kind == platform::Kind::Clint { |
| 201 | set args[2] = *try! devices::install( |
| 202 | &state::SHARED.devices, &mut child.memory.table, i |
| 203 | ); |
| 204 | } |
| 205 | } |
| 206 | assert args[2] <> 0; |
| 207 | let page = try! pages::grant( |
| 208 | &mut state::SHARED.pages, |
| 209 | &TABLE, |
| 210 | &mut child.memory.table, |
| 211 | arguments[hart], |
| 212 | (abi::READ | abi::WRITE | abi::GRANT) as u64 |
| 213 | ); |
| 214 | set args[3] = *page; |
| 215 | set args[4] = WORKER_TICKS * count; |
| 216 | try! fixture::activate(&TABLE, handle, stack, page, 56); |
| 217 | } |
| 218 | for byte in "cycles controllers ready\n" { |
| 219 | put(byte); |
| 220 | } |
| 221 | let now = dispatch::clock(); |
| 222 | let start = now + START_DELAY * count; |
| 223 | for hart in 0 .. limits::HARTS { |
| 224 | if (boot::PLATFORM.harts & (1 << hart)) == 0 { |
| 225 | continue; |
| 226 | } |
| 227 | let window = try! budgets::seed( |
| 228 | &mut state::SHARED.budgets, &mut TABLE, hart, start, 0xffffffffffffffff |
| 229 | ); |
| 230 | let tail = try! budgets::split( |
| 231 | &mut state::SHARED.budgets, &mut TABLE, window, start + CONTROLLER_TICKS * count, now |
| 232 | ); |
| 233 | try! calls::rebind( |
| 234 | &mut state::SHARED, |
| 235 | &mut TABLE, |
| 236 | budgets::Binding { |
| 237 | budget: window, domain: CHILDREN[hart].handle, context: CHILDREN[hart].context |
| 238 | }, |
| 239 | now |
| 240 | ); |
| 241 | if hart == 0 { |
| 242 | let final = try! calls::rebind( |
| 243 | &mut state::SHARED, |
| 244 | &mut TABLE, |
| 245 | budgets::Binding { |
| 246 | budget: tail, domain: parentHandle, context: parent.initial |
| 247 | }, |
| 248 | now |
| 249 | ); |
| 250 | } |
| 251 | } |
| 252 | } |
| 253 | |
| 254 | /// Check all terminal records, shared code identity, and transient recovery. |
| 255 | /// Only the surviving checker runs these phases after the worker windows end. |
| 256 | export unsafe fn verify() { |
| 257 | let retained = checkWorkers(); |
| 258 | checkRecovery(retained); |
| 259 | invariants::check(&TABLE); |
| 260 | for byte in "cycles final recovery and invariants passed\n" { |
| 261 | put(byte); |
| 262 | } |
| 263 | finish(); |
| 264 | } |
| 265 | |
| 266 | /// Check terminal records and package reuse under the metadata lock. |
| 267 | unsafe fn checkWorkers() -> u32 { |
| 268 | for byte in "cycles checking completion\n" { |
| 269 | put(byte); |
| 270 | } |
| 271 | let guard = sync::enter(); |
| 272 | let mut parent = try! domains::get(&state::SHARED.domains, PARENT); |
| 273 | let job = registry::find( |
| 274 | ®istry::capture(&state::SHARED.registry), &"job"[..] |
| 275 | ) else panic "checkWorkers: worker package required"; |
| 276 | let package = try! registry::get(&state::SHARED.registry, job); |
| 277 | let mut seen: u32 = 0; |
| 278 | while let event = try! events::pop(&mut parent.memory.ring) { |
| 279 | assert event.kind == events::CHILD_EXIT and event.code == 68; |
| 280 | let mut found = false; |
| 281 | for hart in 0 .. limits::HARTS { |
| 282 | if (boot::PLATFORM.harts & (1 << hart)) == 0 { |
| 283 | continue; |
| 284 | } |
| 285 | if event.value == abi::id(CHILDREN[hart].domain) { |
| 286 | assert (seen & (1 << hart)) == 0; |
| 287 | set seen |= 1 << hart; |
| 288 | set found = true; |
| 289 | let args = @sliceOf(memory(ARGUMENTS[hart]) as *mut u64, 7); |
| 290 | assert args[5] == package.codeAddress and args[6] == 4; |
| 291 | } |
| 292 | } |
| 293 | assert found; |
| 294 | } |
| 295 | assert seen == boot::PLATFORM.harts; |
| 296 | try! events::refresh( |
| 297 | &mut state::SHARED.domains.events, PARENT, &parent.memory.ring |
| 298 | ); |
| 299 | assert state::SHARED.loader.live[job.index] and state::SHARED.loader.owner |
| 300 | == nil; |
| 301 | let retained = state::SHARED.loader.resident[job.index].code.count |
| 302 | + state::SHARED.loader.resident[job.index].metadata.count; |
| 303 | sync::leave(guard); |
| 304 | return retained; |
| 305 | } |
| 306 | |
| 307 | /// Reap stopped domains, then compare frame and object capacity at quiescence. |
| 308 | unsafe fn checkRecovery(retained: u32) { |
| 309 | let mut done = false; |
| 310 | for attempt in 0 .. RECLAIM_ATTEMPTS { |
| 311 | let guard = sync::enter(); |
| 312 | set done = state::SHARED.domains.dead == 0 |
| 313 | and state::SHARED.pages.backings.pool.retiredCount == 0; |
| 314 | if done { |
| 315 | sync::leave(guard); |
| 316 | break; |
| 317 | } |
| 318 | let request = try! dispatch::request( |
| 319 | &mut state::SHARED.remote, 0, remote::Action::Reschedule |
| 320 | ); |
| 321 | sync::leave(guard); |
| 322 | assert try! dispatch::awaitRequest(0, request) == 0; |
| 323 | } |
| 324 | assert done; |
| 325 | assert freeFrames() + retained == FREE; |
| 326 | let recovered = capacity(); |
| 327 | for i in 0 .. 4 { |
| 328 | assert recovered[i] == SLOTS[i]; |
| 329 | } |
| 330 | } |