kernel: Preserve vector execution contexts
466ae14e5f857810e7cc6a1c0df94372764b5aaae10bffc1a32aad8659646b9d
Save all 32 vector registers and vl, vtype, vstart, and vcsr in each trap frame. Restore that state across nested interrupts, dispatch, and user context switches. Clear vstart before whole-register transfers and restore it after the vector configuration. Enable vector execution for kernel handlers and initialize new contexts with zero vector state. Check the 256-bit native vector ABI during trap installation. Cover register tails, nested timers, context switches, and both context creation paths. Assisted-by: Codex:gpt-6 Change-Id: I34ec4c799361ed37b994154ef94310e7ffb16831
1 parent
493954d4
kernel/kernel/boot.ras
+4 -1
| 1 | 1 | //! RV64 machine startup. Firmware supplies a0=hart, a1=FDT, and aligned sp. |
|
| 2 | + | // Initial vector state value in mstatus. |
|
| 3 | + | .constant BOOT_VS_INITIAL 512; |
|
| 2 | 4 | .text; |
|
| 3 | 5 | .export @kernel::boot::start; |
|
| 4 | 6 | .export @kernel::boot::initialize; |
|
| 5 | 7 | ||
| 6 | 8 | @kernel::boot::start |
| 9 | 11 | ebreak; |
|
| 10 | 12 | ||
| 11 | 13 | // Initialize machine CSRs and preserve the firmware stack across Radiance entry. |
|
| 12 | 14 | @kernel::boot::initialize |
|
| 13 | 15 | csrw mie %zero; |
|
| 14 | - | csrw mstatus %zero; |
|
| 16 | + | li %t0 BOOT_VS_INITIAL; |
|
| 17 | + | csrw mstatus %t0; |
|
| 15 | 18 | csrw mscratch %sp; |
|
| 16 | 19 | mv %a2 %sp; |
|
| 17 | 20 | mv %a5 %gp; |
|
| 18 | 21 | addi %sp %sp -16; |
|
| 19 | 22 | sd %ra 0(%sp); |
kernel/kernel/dispatch.rad
+18 -16
| 1 | 1 | //! Hart-local selection of runnable CPU authority and timer boundaries. |
|
| 2 | 2 | ||
| 3 | - | use super::state; |
|
| 3 | + | use std::arch::rv64; |
|
| 4 | + | use std::mem; |
|
| 4 | 5 | use super::abi; |
|
| 5 | - | use super::limits; |
|
| 6 | - | use super::slots; |
|
| 7 | 6 | use super::budgets; |
|
| 7 | + | use super::calls; |
|
| 8 | 8 | use super::domains; |
|
| 9 | - | use super::lifecycle; |
|
| 10 | - | use super::loader; |
|
| 11 | - | use super::registry; |
|
| 9 | + | use super::events; |
|
| 12 | 10 | use super::frames; |
|
| 13 | 11 | use super::interrupts; |
|
| 14 | - | use super::events; |
|
| 12 | + | use super::lifecycle; |
|
| 13 | + | use super::limits; |
|
| 14 | + | use super::loader; |
|
| 15 | 15 | use super::pages; |
|
| 16 | 16 | use super::platform; |
|
| 17 | - | use super::trap; |
|
| 18 | - | use super::range; |
|
| 19 | - | use super::calls; |
|
| 20 | - | use super::timers; |
|
| 21 | - | use super::sync; |
|
| 22 | - | use super::remote; |
|
| 23 | 17 | use super::plic; |
|
| 18 | + | use super::range; |
|
| 24 | 19 | use super::registers; |
|
| 25 | - | use std::mem; |
|
| 26 | - | use std::arch::rv64; |
|
| 20 | + | use super::registry; |
|
| 21 | + | use super::remote; |
|
| 22 | + | use super::slots; |
|
| 23 | + | use super::state; |
|
| 24 | + | use super::sync; |
|
| 25 | + | use super::timers; |
|
| 26 | + | use super::trap; |
|
| 27 | 27 | ||
| 28 | 28 | /// Base maintenance passes per second. |
|
| 29 | 29 | constant MAINTENANCE_HZ: u64 = 100; |
|
| 30 | 30 | /// Maximum frames reclaimed in one maintenance pass. |
|
| 31 | 31 | constant RECLAIM_BATCH: u32 = 256; |
| 117 | 117 | let stack = machine.stacks[id]; |
|
| 118 | 118 | let mut idle = trap::Frame { |
|
| 119 | 119 | registers: [0; rv64::NUM_REGISTERS], |
|
| 120 | 120 | pc: idleAddress(), |
|
| 121 | 121 | status: rv64::MSTATUS_MPP_MACHINE |
|
| 122 | - | | rv64::MSTATUS_MPIE, |
|
| 122 | + | | rv64::MSTATUS_MPIE |
|
| 123 | + | | rv64::MSTATUS_VS_INITIAL, |
|
| 123 | 124 | cause: 0, |
|
| 124 | 125 | value: 0, |
|
| 126 | + | vector: trap::EMPTY_VECTOR, |
|
| 125 | 127 | }; |
|
| 126 | 128 | set idle.registers[*rv64::SP as u32] = stack.end; |
|
| 127 | 129 | set idle.registers[*rv64::GP as u32] = kernelGp; |
|
| 128 | 130 | let mut maintenance = machine.timebase as u64 / MAINTENANCE_HZ; |
|
| 129 | 131 | if maintenance == 0 { |
kernel/kernel/dispatch.ras
+1 -1
| 2 | 2 | // Machine interrupt-enable bit in mstatus. |
|
| 3 | 3 | .constant DISPATCH_MIE 8; |
|
| 4 | 4 | // Machine software, timer, and external interrupt enable bits. |
|
| 5 | 5 | .constant DISPATCH_SOURCES 2184; |
|
| 6 | 6 | // Bytes in the saved Frame record. |
|
| 7 | - | .constant DISPATCH_FRAME_SIZE 288; |
|
| 7 | + | .constant DISPATCH_FRAME_SIZE 1344; |
|
| 8 | 8 | // Bytes in one saved frame word. |
|
| 9 | 9 | .constant DISPATCH_WORD_SIZE 8; |
|
| 10 | 10 | // Words copied into the retained dispatch frame. |
|
| 11 | 11 | .constant DISPATCH_FRAME_WORDS DISPATCH_FRAME_SIZE / DISPATCH_WORD_SIZE; |
|
| 12 | 12 | // Hart dispatchFrame field offset. |
kernel/kernel/domains.rad
+4 -2
| 735 | 735 | let kernelStack = try! frames::extent(&memory.backings.pool, kernelFrames); |
|
| 736 | 736 | let userStack = try! pages::consumeStack(memory, targetTable, abi::Handle(start.stack)); |
|
| 737 | 737 | let mut frame = trap::Frame { |
|
| 738 | 738 | registers: [0; rv64::NUM_REGISTERS], |
|
| 739 | 739 | pc: start.entry, |
|
| 740 | - | status: rv64::MSTATUS_MPIE, |
|
| 740 | + | status: rv64::MSTATUS_MPIE | rv64::MSTATUS_VS_INITIAL, |
|
| 741 | 741 | cause: 0, |
|
| 742 | 742 | value: 0, |
|
| 743 | + | vector: trap::EMPTY_VECTOR, |
|
| 743 | 744 | }; |
|
| 744 | 745 | set frame.registers[*rv64::RA as u32] = returnAddress(); |
|
| 745 | 746 | set frame.registers[*rv64::SP as u32] = userStack.end; |
|
| 746 | 747 | set frame.registers[*rv64::GP as u32] = |
|
| 747 | 748 | instances::address(&store.tables[domain.index]); |
| 1178 | 1179 | )); |
|
| 1179 | 1180 | // MPIE enables interrupts after the first return into U-mode. |
|
| 1180 | 1181 | let mut frame = trap::Frame { |
|
| 1181 | 1182 | registers: [0; rv64::NUM_REGISTERS], |
|
| 1182 | 1183 | pc: creation.entry, |
|
| 1183 | - | status: rv64::MSTATUS_MPIE, |
|
| 1184 | + | status: rv64::MSTATUS_MPIE | rv64::MSTATUS_VS_INITIAL, |
|
| 1184 | 1185 | cause: 0, |
|
| 1185 | 1186 | value: 0, |
|
| 1187 | + | vector: trap::EMPTY_VECTOR, |
|
| 1186 | 1188 | }; |
|
| 1187 | 1189 | set frame.registers[*rv64::RA as u32] = returnAddress(); |
|
| 1188 | 1190 | let graphFrames = instances::publish(graph, &mut store.tables[object.index]); |
|
| 1189 | 1191 | set frame.registers[*rv64::GP as u32] = |
|
| 1190 | 1192 | instances::address(&store.tables[object.index]); |
kernel/kernel/tests/budgets.rad
+6 -4
| 1 | 1 | //! Conservation and expiration of exclusive CPU windows. |
|
| 2 | 2 | ||
| 3 | 3 | use std::testing; |
|
| 4 | 4 | use kernel::abi; |
|
| 5 | - | use kernel::capability; |
|
| 6 | 5 | use kernel::budgets; |
|
| 7 | - | use kernel::slots; |
|
| 6 | + | use kernel::capability; |
|
| 8 | 7 | use kernel::domains; |
|
| 9 | 8 | use kernel::frames; |
|
| 10 | 9 | use kernel::instances; |
|
| 11 | 10 | use kernel::physical; |
|
| 12 | - | use kernel::trap; |
|
| 13 | 11 | use kernel::range; |
|
| 12 | + | use kernel::slots; |
|
| 13 | + | use kernel::trap; |
|
| 14 | 14 | ||
| 15 | 15 | /// Exclusive CPU-window metadata. |
|
| 16 | 16 | unsafe static STORE: budgets::Store = undefined; |
|
| 17 | 17 | /// Capability owner for the test windows. |
|
| 18 | 18 | unsafe static TABLE: capability::Table = undefined; |
| 295 | 295 | kernelFrames: frames::Run { first: 0, count: 0 }, |
|
| 296 | 296 | kernelStack: range::Range { |
|
| 297 | 297 | start: 0, end: 0 |
|
| 298 | 298 | }, |
|
| 299 | 299 | userStack: range::Range { start: 0, end: 0 }, |
|
| 300 | - | frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 }, |
|
| 300 | + | frame: trap::Frame { |
|
| 301 | + | registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0, vector: trap::EMPTY_VECTOR |
|
| 302 | + | }, |
|
| 301 | 303 | }; |
|
| 302 | 304 | set DOMAINS.records[child.index] = domains::Domain { |
|
| 303 | 305 | state: domains::Lifecycle::Pending, |
|
| 304 | 306 | creator: root, |
|
| 305 | 307 | parent: root, |
kernel/kernel/tests/calls.rad
+39 -19
| 1 | 1 | //! Direct-call scalar results, checked record copies, and architectural replies. |
|
| 2 | 2 | ||
| 3 | 3 | use std::testing; |
|
| 4 | - | use kernel::state; |
|
| 5 | - | use kernel::sync; |
|
| 6 | 4 | use kernel::abi; |
|
| 7 | - | use kernel::frames; |
|
| 8 | - | use kernel::physical; |
|
| 9 | 5 | use kernel::access; |
|
| 6 | + | use kernel::backing; |
|
| 7 | + | use kernel::budgets; |
|
| 10 | 8 | use kernel::calls; |
|
| 9 | + | use kernel::capability; |
|
| 10 | + | use kernel::devices; |
|
| 11 | 11 | use kernel::domains; |
|
| 12 | + | use kernel::events; |
|
| 13 | + | use kernel::frames; |
|
| 14 | + | use kernel::interrupts; |
|
| 15 | + | use kernel::lifecycle; |
|
| 16 | + | use kernel::limits; |
|
| 12 | 17 | use kernel::pages; |
|
| 13 | - | use kernel::backing; |
|
| 14 | - | use kernel::capability; |
|
| 15 | - | use kernel::slots; |
|
| 18 | + | use kernel::physical; |
|
| 19 | + | use kernel::platform; |
|
| 16 | 20 | use kernel::registry; |
|
| 17 | - | use kernel::limits; |
|
| 18 | - | use kernel::trap; |
|
| 19 | - | use kernel::budgets; |
|
| 20 | 21 | use kernel::remote; |
|
| 21 | - | use kernel::interrupts; |
|
| 22 | - | use kernel::events; |
|
| 23 | - | use kernel::devices; |
|
| 24 | - | use kernel::platform; |
|
| 25 | - | use kernel::lifecycle; |
|
| 22 | + | use kernel::slots; |
|
| 23 | + | use kernel::state; |
|
| 24 | + | use kernel::sync; |
|
| 25 | + | use kernel::trap; |
|
| 26 | 26 | ||
| 27 | 27 | /// Event storage for administrative destruction targets. |
|
| 28 | 28 | unsafe static TARGETS: [domains::Memory; 2] = undefined; |
|
| 29 | 29 | ||
| 30 | 30 | /// Retained dead-domain records cannot be used or delegated through handles. |
| 873 | 873 | @test unsafe fn invalidQueryAborts() throws (testing::TestError) { |
|
| 874 | 874 | for operation in [80 as u64, 81, 82, 83, 84] { |
|
| 875 | 875 | let owner = initialize(); |
|
| 876 | 876 | try! events::open(&mut state::SHARED.domains.events, owner, &mut MEMORY.ring); |
|
| 877 | 877 | let mut frame = trap::Frame { |
|
| 878 | - | registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 |
|
| 878 | + | registers: [0; 32], |
|
| 879 | + | pc: 0x80000000, |
|
| 880 | + | status: 0x80, |
|
| 881 | + | cause: 8, |
|
| 882 | + | value: 0, |
|
| 883 | + | vector: trap::EMPTY_VECTOR, |
|
| 879 | 884 | }; |
|
| 880 | 885 | set frame.registers[17] = operation; |
|
| 881 | 886 | calls::handle(owner, &mut frame, 0, 0, unlocked); |
|
| 882 | 887 | assert state::SHARED.domains.records[owner.index].state == domains::Lifecycle::Dead; |
|
| 883 | 888 | assert state::SHARED.domains.events.terminals[owner.index].event.code == abi::FAULT_ABORT; |
| 887 | 892 | ||
| 888 | 893 | /// An unknown user operation advances PC and returns only a negative a0 error. |
|
| 889 | 894 | @test unsafe fn trapReply() throws (testing::TestError) { |
|
| 890 | 895 | let owner = initialize(); |
|
| 891 | 896 | let mut frame = trap::Frame { |
|
| 892 | - | registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 |
|
| 897 | + | registers: [123; 32], |
|
| 898 | + | pc: 0x80000000, |
|
| 899 | + | status: 0x80, |
|
| 900 | + | cause: 8, |
|
| 901 | + | value: 0, |
|
| 902 | + | vector: trap::EMPTY_VECTOR, |
|
| 893 | 903 | }; |
|
| 894 | 904 | set frame.registers[17] = 0xffffffffffffffff; |
|
| 895 | 905 | calls::handle(owner, &mut frame, 0, 0, unlocked); |
|
| 896 | 906 | try testing::expect(frame.pc == 0x80000004 and frame.registers[10] == 0xfffffffffffffffc); |
|
| 897 | 907 | for i in 0 .. 32 { |
| 957 | 967 | let owner = initialize(); |
|
| 958 | 968 | let handle = try! allocate(owner, 1); |
|
| 959 | 969 | let base = &RAM[OFFSET] as u64; |
|
| 960 | 970 | set RAM[OFFSET + 2] = 0x12345678; |
|
| 961 | 971 | let mut frame = trap::Frame { |
|
| 962 | - | registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 |
|
| 972 | + | registers: [0; 32], |
|
| 973 | + | pc: 0x80000000, |
|
| 974 | + | status: 0x80, |
|
| 975 | + | cause: 8, |
|
| 976 | + | value: 0, |
|
| 977 | + | vector: trap::EMPTY_VECTOR, |
|
| 963 | 978 | }; |
|
| 964 | 979 | set frame.registers[17] = 80; |
|
| 965 | 980 | set frame.registers[10] = handle; |
|
| 966 | 981 | set frame.registers[14] = base; |
|
| 967 | 982 | set frame.registers[15] = 16; |
| 1006 | 1021 | set state::SHARED.pages.records[object.index].base = 0x7fff00000000000; |
|
| 1007 | 1022 | set state::SHARED.pages.records[object.index].count = 0x3000000; |
|
| 1008 | 1023 | let region = try! calls::invokeReply(owner, 32, &[handle, 0, 0, 0], 0); |
|
| 1009 | 1024 | try testing::expect(region.value == 0x7fff00000000000 and region.extra == 0x3000000 * 4096); |
|
| 1010 | 1025 | let mut frame = trap::Frame { |
|
| 1011 | - | registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 |
|
| 1026 | + | registers: [0; 32], |
|
| 1027 | + | pc: 0x80000000, |
|
| 1028 | + | status: 0x80, |
|
| 1029 | + | cause: 8, |
|
| 1030 | + | value: 0, |
|
| 1031 | + | vector: trap::EMPTY_VECTOR, |
|
| 1012 | 1032 | }; |
|
| 1013 | 1033 | set frame.registers[17] = 32; |
|
| 1014 | 1034 | set frame.registers[10] = handle; |
|
| 1015 | 1035 | calls::handle(owner, &mut frame, 0, 0, unlocked); |
|
| 1016 | 1036 | try testing::expect( |
kernel/kernel/tests/dispatch.rad
+5 -3
| 1 | 1 | //! Dispatch selection at CPU-window boundaries and idle gaps. |
|
| 2 | 2 | ||
| 3 | 3 | use std::testing; |
|
| 4 | 4 | use kernel::abi; |
|
| 5 | - | use kernel::slots; |
|
| 6 | - | use kernel::domains; |
|
| 7 | 5 | use kernel::budgets; |
|
| 8 | 6 | use kernel::capability; |
|
| 9 | 7 | use kernel::dispatch; |
|
| 8 | + | use kernel::domains; |
|
| 10 | 9 | use kernel::platform; |
|
| 11 | 10 | use kernel::range; |
|
| 12 | 11 | use kernel::registers; |
|
| 13 | 12 | use kernel::remote; |
|
| 13 | + | use kernel::slots; |
|
| 14 | 14 | use kernel::trap; |
|
| 15 | 15 | ||
| 16 | 16 | /// Platform resources used to validate CLINT address selection. |
|
| 17 | 17 | unsafe static PLATFORM: platform::Platform = undefined; |
|
| 18 | 18 |
| 159 | 159 | fn handler(frame: &mut trap::Frame, hart: &mut trap::Hart) {} |
|
| 160 | 160 | ||
| 161 | 161 | /// Exchange preserves an M-mode continuation while another context uses its own stack. |
|
| 162 | 162 | @test unsafe fn continuations() throws (testing::TestError) { |
|
| 163 | 163 | initialize(); |
|
| 164 | - | let mut frame = trap::Frame { registers: [0; 32], pc: 0x40, status: 0x80, cause: 0, value: 0 }; |
|
| 164 | + | let mut frame = trap::Frame { |
|
| 165 | + | registers: [0; 32], pc: 0x40, status: 0x80, cause: 0, value: 0, vector: trap::EMPTY_VECTOR |
|
| 166 | + | }; |
|
| 165 | 167 | set DOMAINS.contexts[0].frame = frame; |
|
| 166 | 168 | set DOMAINS.contexts[0].kernelStack = range::Range { start: 0x8000, end: 0xc000 }; |
|
| 167 | 169 | set DOMAINS.contextSlots[1] = slots::Slot { generation: 1, state: slots::State::Live }; |
|
| 168 | 170 | set DOMAINS.contexts[1].owner = TABLE.owner; |
|
| 169 | 171 | set DOMAINS.contexts[1].state = domains::ContextState::Ready; |
kernel/kernel/tests/domains.rad
+15 -1
| 1 | 1 | //! Pending domain creation, initial context state, and creation authority. |
|
| 2 | 2 | ||
| 3 | + | use std::arch::rv64; |
|
| 3 | 4 | use std::arch::rv64::shared; |
|
| 4 | 5 | use std::testing; |
|
| 5 | 6 | use super::support; |
|
| 6 | 7 | use kernel::abi; |
|
| 7 | 8 | use kernel::backing; |
| 19 | 20 | use kernel::physical; |
|
| 20 | 21 | use kernel::registry; |
|
| 21 | 22 | use kernel::slots; |
|
| 22 | 23 | use kernel::state; |
|
| 23 | 24 | use kernel::transactions; |
|
| 25 | + | use kernel::trap; |
|
| 24 | 26 | ||
| 25 | 27 | /// Exclusive resources released by the domain reaper. |
|
| 26 | 28 | unsafe static WINDOWS: budgets::Store = undefined; |
|
| 27 | 29 | /// Interrupt ownership released after hardware masking. |
|
| 28 | 30 | unsafe static IRQS: interrupts::Store = undefined; |
| 995 | 997 | try testing::expect( |
|
| 996 | 998 | created.kernelStack.start >= initial.kernelStack.end |
|
| 997 | 999 | or created.kernelStack.end <= initial.kernelStack.start |
|
| 998 | 1000 | ); |
|
| 999 | 1001 | try testing::expect(created.frame.pc == later.entry); |
|
| 1002 | + | freshVector(&created.frame); |
|
| 1000 | 1003 | try testing::expect( |
|
| 1001 | 1004 | created.frame.registers[2] == secondPage.base + 8192 |
|
| 1002 | 1005 | and created.frame.registers[4] == secondPage.base |
|
| 1003 | 1006 | ); |
|
| 1004 | 1007 | try testing::expect( |
| 1368 | 1371 | try testing::expect( |
|
| 1369 | 1372 | context.frame.registers[3] == instances::address( |
|
| 1370 | 1373 | &DOMAINS.tables[authority.object.index] |
|
| 1371 | 1374 | ) |
|
| 1372 | 1375 | ); |
|
| 1373 | - | try testing::expect(context.frame.registers[2] == 0 and context.frame.status == 0x80); |
|
| 1376 | + | try testing::expect(context.frame.registers[2] == 0); |
|
| 1377 | + | freshVector(&context.frame); |
|
| 1374 | 1378 | try testing::expect(context.kernelFrames.count == domains::KERNEL_STACK_PAGES); |
|
| 1375 | 1379 | try testing::expect( |
|
| 1376 | 1380 | context.kernelStack.end - context.kernelStack.start |
|
| 1377 | 1381 | == domains::KERNEL_STACK_PAGES as u64 * 4096 |
|
| 1378 | 1382 | ); |
| 1523 | 1527 | try testing::expect(DOMAINS.contextSlots[0].state == slots::State::Free); |
|
| 1524 | 1528 | for i in 0 .. BACKINGS.pool.count { |
|
| 1525 | 1529 | assert frames::available(&BACKINGS.pool, i); |
|
| 1526 | 1530 | } |
|
| 1527 | 1531 | } |
|
| 1532 | + | ||
| 1533 | + | /// New user contexts enable vectors with zero registers and no active elements. |
|
| 1534 | + | fn freshVector(frame: &trap::Frame) { |
|
| 1535 | + | assert frame.status == (rv64::MSTATUS_MPIE | rv64::MSTATUS_VS_INITIAL); |
|
| 1536 | + | assert frame.vector.length == 0 and frame.vector.type == 0; |
|
| 1537 | + | assert frame.vector.start == 0 and frame.vector.control == 0; |
|
| 1538 | + | for word in &frame.vector.registers[..] { |
|
| 1539 | + | assert word == 0; |
|
| 1540 | + | } |
|
| 1541 | + | } |
kernel/kernel/tests/trap.rad
+3 -3
| 1 | 1 | //! Trap classification, frame layout, and resume-address boundaries. |
|
| 2 | 2 | ||
| 3 | 3 | use std::testing; |
|
| 4 | 4 | use kernel::trap; |
|
| 5 | 5 | ||
| 6 | - | /// The assembly frame includes x0-x31 and all four trap CSRs. |
|
| 6 | + | /// The assembly frame includes integer and vector registers and their CSRs. |
|
| 7 | 7 | @test fn layout() throws (testing::TestError) { |
|
| 8 | - | try testing::expect(@sizeOf(trap::Frame) == 288); |
|
| 9 | - | assert @sizeOf(trap::Hart) == 352; |
|
| 8 | + | try testing::expect(@sizeOf(trap::Frame) == 1344); |
|
| 9 | + | assert @sizeOf(trap::Hart) == 1408; |
|
| 10 | 10 | try testing::expect(trap::FRAME_SIZE % 16 == 0); |
|
| 11 | 11 | } |
|
| 12 | 12 | ||
| 13 | 13 | /// Interrupts retain their source identity; exceptions retain their fault class. |
|
| 14 | 14 | @test fn classification() throws (testing::TestError) { |
kernel/kernel/trap.rad
+30 -4
| 1 | - | //! Integer trap contexts and exception classification. |
|
| 1 | + | //! Integer and vector trap contexts and exception classification. |
|
| 2 | 2 | ||
| 3 | - | use std::mem; |
|
| 4 | 3 | use std::arch::rv64; |
|
| 4 | + | use std::arch::rv64::vector; |
|
| 5 | + | use std::mem; |
|
| 6 | + | ||
| 5 | 7 | /// Frame bytes reserved by trap entry. This is a multiple of stack alignment. |
|
| 6 | - | export constant FRAME_SIZE: u32 = 288; |
|
| 8 | + | export constant FRAME_SIZE: u32 = 1344; |
|
| 9 | + | ||
| 10 | + | /// Number of 64-bit words in one register of the native vector ABI. |
|
| 11 | + | constant VECTOR_WORDS_PER_REGISTER: u32 = 4; |
|
| 12 | + | ||
| 13 | + | /// Saved vector registers and CSRs for the 256-bit native vector ABI. |
|
| 14 | + | export record VectorState: Copy { |
|
| 15 | + | /// Thirty-two vector registers, with four 64-bit words per register. |
|
| 16 | + | registers: [u64; vector::REGISTER_COUNT * VECTOR_WORDS_PER_REGISTER], |
|
| 17 | + | /// Number of active vector elements. |
|
| 18 | + | length: u64, |
|
| 19 | + | /// Vector element width, grouping and inactive element policies. |
|
| 20 | + | type: u64, |
|
| 21 | + | /// Element index at which vector execution resumes. |
|
| 22 | + | start: u64, |
|
| 23 | + | /// Fixed-point rounding mode and saturation flag. |
|
| 24 | + | control: u64, |
|
| 25 | + | } |
|
| 26 | + | ||
| 27 | + | /// Zero vector registers with no active elements. |
|
| 28 | + | export constant EMPTY_VECTOR: VectorState = VectorState { |
|
| 29 | + | registers: [0; vector::REGISTER_COUNT * VECTOR_WORDS_PER_REGISTER], length: 0, type: 0, start: 0, control: 0 |
|
| 30 | + | }; |
|
| 7 | 31 | ||
| 8 | 32 | /// Saved architectural state. Assembly offsets follow field declaration order. |
|
| 9 | 33 | export record Frame: Copy { |
|
| 10 | 34 | /// Integer registers indexed by architectural register number. x0 is zero. |
|
| 11 | 35 | registers: [u64; rv64::NUM_REGISTERS], |
| 15 | 39 | status: u64, |
|
| 16 | 40 | /// Interrupt bit and exception code captured from mcause. |
|
| 17 | 41 | cause: u64, |
|
| 18 | 42 | /// Fault address or instruction bits captured from mtval. |
|
| 19 | 43 | value: u64, |
|
| 44 | + | /// Complete vector state captured before the trap handler runs. |
|
| 45 | + | vector: VectorState, |
|
| 20 | 46 | } |
|
| 21 | 47 | ||
| 22 | 48 | /// Per-hart trap entry state named by mscratch. Entry scratch fields are private |
|
| 23 | 49 | /// to the assembly prologue while machine interrupts are disabled. |
|
| 24 | 50 | export record Hart: Copy { |
| 111 | 137 | } |
|
| 112 | 138 | ||
| 113 | 139 | /// Install trap entry for the current hart with machine interrupts disabled. |
|
| 114 | 140 | /// The Hart must stay valid for the program and belong to the executing hart. |
|
| 115 | 141 | export unsafe fn install(hart: *mut Hart); |
|
| 116 | - | /// Restore a complete integer context through mret. The current mscratch must |
|
| 142 | + | /// Restore a complete integer and vector context through mret. The current mscratch must |
|
| 117 | 143 | /// name an installed Hart, and the frame must have machine interrupts disabled. |
|
| 118 | 144 | export unsafe fn resume(frame: *Frame) -> !; |
|
| 119 | 145 | ||
| 120 | 146 | /// Disable machine interrupts and stop the current hart. |
|
| 121 | 147 | export fn halt() -> !; |
kernel/kernel/trap.ras
+57 -2
| 1 | - | //! RV64 integer trap entry. mscratch names the current hart's kernel state. |
|
| 1 | + | //! RV64 integer and vector trap entry. mscratch names the current hart's kernel state. |
|
| 2 | 2 | // Machine interrupt-enable bit in mstatus. |
|
| 3 | 3 | .constant TRAP_MIE 8; |
|
| 4 | 4 | // Mask that clears the machine interrupt-enable bit. |
|
| 5 | 5 | .constant TRAP_MIE_CLEAR -TRAP_MIE - 1; |
|
| 6 | 6 | // Bit offset of the previous privilege field. |
| 10 | 10 | // Previous privilege value for machine mode. |
|
| 11 | 11 | .constant TRAP_MPP_MACHINE 3; |
|
| 12 | 12 | // Mask for the required 16-byte stack alignment. |
|
| 13 | 13 | .constant TRAP_STACK_MASK 15; |
|
| 14 | 14 | // Bytes in the saved Frame record. |
|
| 15 | - | .constant TRAP_FRAME_SIZE 288; |
|
| 15 | + | .constant TRAP_FRAME_SIZE 1344; |
|
| 16 | 16 | // Hart stackTop field offset. |
|
| 17 | 17 | .constant TRAP_HART_STACK_TOP 0; |
|
| 18 | 18 | // Hart stackBottom field offset. |
|
| 19 | 19 | .constant TRAP_HART_STACK_BOTTOM 8; |
|
| 20 | 20 | // Hart kernelGp field offset. |
| 34 | 34 | // Frame cause field offset. |
|
| 35 | 35 | .constant TRAP_FRAME_CAUSE 272; |
|
| 36 | 36 | // Frame value field offset. |
|
| 37 | 37 | .constant TRAP_FRAME_VALUE 280; |
|
| 38 | 38 | ||
| 39 | + | // Initial vector state value in mstatus. |
|
| 40 | + | .constant TRAP_VS_INITIAL 512; |
|
| 41 | + | // Bytes per vector register in the native ABI. |
|
| 42 | + | .constant TRAP_VECTOR_BYTES 32; |
|
| 43 | + | // Bytes in a group of eight vector registers. |
|
| 44 | + | .constant TRAP_VECTOR_GROUP 256; |
|
| 45 | + | // Frame vector register and CSR field offsets. |
|
| 46 | + | .constant TRAP_FRAME_VECTORS 288; |
|
| 47 | + | .constant TRAP_FRAME_VL 1312; |
|
| 48 | + | .constant TRAP_FRAME_VTYPE 1320; |
|
| 49 | + | .constant TRAP_FRAME_VSTART 1328; |
|
| 50 | + | .constant TRAP_FRAME_VCSR 1336; |
|
| 51 | + | ||
| 39 | 52 | .text; |
|
| 40 | 53 | .export @kernel::trap::install; |
|
| 41 | 54 | .export @kernel::trap::resume; |
|
| 42 | 55 | .export @kernel::trap::vector; |
|
| 43 | 56 | .export @kernel::trap::halt; |
|
| 44 | 57 | ||
| 45 | 58 | // Install a direct trap vector and its per-hart stack anchor. |
|
| 46 | 59 | @kernel::trap::install |
|
| 47 | 60 | csrw mie %zero; |
|
| 48 | 61 | csrci mstatus TRAP_MIE; |
|
| 62 | + | li %t0 TRAP_VS_INITIAL; |
|
| 63 | + | csrs mstatus %t0; |
|
| 64 | + | csrr %t0 vlenb; |
|
| 65 | + | li %t1 TRAP_VECTOR_BYTES; |
|
| 66 | + | bne %t0 %t1 @stackFault; |
|
| 49 | 67 | csrw mscratch %a0; |
|
| 50 | 68 | la %t0 @kernel::trap::vector; |
|
| 51 | 69 | csrw mtvec %t0; |
|
| 52 | 70 | ret; |
|
| 53 | 71 |
| 117 | 135 | sd %t0 TRAP_FRAME_STATUS(%sp); |
|
| 118 | 136 | csrr %t0 mcause; |
|
| 119 | 137 | sd %t0 TRAP_FRAME_CAUSE(%sp); |
|
| 120 | 138 | csrr %t0 mtval; |
|
| 121 | 139 | sd %t0 TRAP_FRAME_VALUE(%sp); |
|
| 140 | + | li %t0 TRAP_VS_INITIAL; |
|
| 141 | + | csrs mstatus %t0; |
|
| 142 | + | csrr %t0 vstart; |
|
| 143 | + | sd %t0 TRAP_FRAME_VSTART(%sp); |
|
| 144 | + | csrw vstart %zero; |
|
| 145 | + | csrr %t0 vl; |
|
| 146 | + | sd %t0 TRAP_FRAME_VL(%sp); |
|
| 147 | + | csrr %t0 vtype; |
|
| 148 | + | sd %t0 TRAP_FRAME_VTYPE(%sp); |
|
| 149 | + | csrr %t0 vcsr; |
|
| 150 | + | sd %t0 TRAP_FRAME_VCSR(%sp); |
|
| 151 | + | addi %t0 %sp TRAP_FRAME_VECTORS; |
|
| 152 | + | vs8r.v %v0 (%t0); |
|
| 153 | + | addi %t0 %t0 TRAP_VECTOR_GROUP; |
|
| 154 | + | vs8r.v %v8 (%t0); |
|
| 155 | + | addi %t0 %t0 TRAP_VECTOR_GROUP; |
|
| 156 | + | vs8r.v %v16 (%t0); |
|
| 157 | + | addi %t0 %t0 TRAP_VECTOR_GROUP; |
|
| 158 | + | vs8r.v %v24 (%t0); |
|
| 122 | 159 | csrw mscratch %tp; |
|
| 123 | 160 | ld %gp TRAP_HART_GP(%tp); |
|
| 124 | 161 | mv %a0 %sp; |
|
| 125 | 162 | mv %a1 %tp; |
|
| 126 | 163 | ld %t0 TRAP_HART_HANDLER(%tp); |
| 131 | 168 | @kernel::trap::resume |
|
| 132 | 169 | csrci mstatus TRAP_MIE; |
|
| 133 | 170 | mv %sp %a0; |
|
| 134 | 171 | // A resumed frame must not use a reservation from another execution context. |
|
| 135 | 172 | sc.d %zero %zero (%sp); |
|
| 173 | + | li %t0 TRAP_VS_INITIAL; |
|
| 174 | + | csrs mstatus %t0; |
|
| 175 | + | csrw vstart %zero; |
|
| 176 | + | addi %t0 %sp TRAP_FRAME_VECTORS; |
|
| 177 | + | vl8re8.v %v0 (%t0); |
|
| 178 | + | addi %t0 %t0 TRAP_VECTOR_GROUP; |
|
| 179 | + | vl8re8.v %v8 (%t0); |
|
| 180 | + | addi %t0 %t0 TRAP_VECTOR_GROUP; |
|
| 181 | + | vl8re8.v %v16 (%t0); |
|
| 182 | + | addi %t0 %t0 TRAP_VECTOR_GROUP; |
|
| 183 | + | vl8re8.v %v24 (%t0); |
|
| 184 | + | ld %t0 TRAP_FRAME_VL(%sp); |
|
| 185 | + | ld %t1 TRAP_FRAME_VTYPE(%sp); |
|
| 186 | + | vsetvl %zero %t0 %t1; |
|
| 187 | + | ld %t0 TRAP_FRAME_VCSR(%sp); |
|
| 188 | + | csrw vcsr %t0; |
|
| 189 | + | ld %t0 TRAP_FRAME_VSTART(%sp); |
|
| 190 | + | csrw vstart %t0; |
|
| 136 | 191 | ld %t0 TRAP_FRAME_PC(%sp); |
|
| 137 | 192 | csrw mepc %t0; |
|
| 138 | 193 | ld %t0 TRAP_FRAME_STATUS(%sp); |
|
| 139 | 194 | andi %t0 %t0 TRAP_MIE_CLEAR; |
|
| 140 | 195 | csrw mstatus %t0; |
lib/std/arch/rv64.rad
+2 -0
| 79 | 79 | /// Machine external interrupt cause. |
|
| 80 | 80 | export constant MCAUSE_MACHINE_EXTERNAL: u64 = MCAUSE_INTERRUPT |
|
| 81 | 81 | | IRQ_MACHINE_EXTERNAL; |
|
| 82 | 82 | /// Machine previous interrupt-enable bit in mstatus. |
|
| 83 | 83 | export constant MSTATUS_MPIE: u64 = 1 << 7; |
|
| 84 | + | /// Initial vector state field value in mstatus. |
|
| 85 | + | export constant MSTATUS_VS_INITIAL: u64 = 1 << 9; |
|
| 84 | 86 | /// Previous privilege field mask in mstatus. |
|
| 85 | 87 | export constant MSTATUS_MPP_MASK: u64 = 3 << 11; |
|
| 86 | 88 | /// Previous privilege field value for machine mode. |
|
| 87 | 89 | export constant MSTATUS_MPP_MACHINE: u64 = 3 << 11; |
|
| 88 | 90 |
test/integration/machine.rad
+2 -1
| 71 | 71 | ||
| 72 | 72 | /// Run the trap fixtures on one hart. |
|
| 73 | 73 | export fn trap(h: &mut Harness, args: *[*[u8]]) throws (sys::Error) { |
|
| 74 | 74 | try h.begin("trap"); |
|
| 75 | 75 | let mark = h.failed; |
|
| 76 | - | for fixture in &["machine", "reservations"] { |
|
| 76 | + | for fixture in &["machine", "reservations", "vector"] { |
|
| 77 | 77 | let sources = [h.join(&["test/trap/", fixture, ".ras"]), "kernel/kernel/trap.ras"]; |
|
| 78 | 78 | let what = h.join(&["trap fixture ", fixture]); |
|
| 79 | 79 | try assembleAndRun(h, what, h.join(&["trap.", fixture]), &sources[..], 1); |
|
| 80 | 80 | } |
|
| 81 | 81 | h.done( |
|
| 82 | 82 | mark, |
|
| 83 | 83 | "trap contexts: machine/user registers, nested timer, illegal instruction, and stacks passed" |
|
| 84 | 84 | ); |
|
| 85 | 85 | h.done(mark, "trap reservations: entry, resume, and user context isolation passed"); |
|
| 86 | + | h.done(mark, "trap vectors: complete state, nested timer and user context switches passed"); |
|
| 86 | 87 | } |
|
| 87 | 88 | ||
| 88 | 89 | /// Compile two packages once and run them with two private state graphs, then |
|
| 89 | 90 | /// link a package built alone with an entry built beside an unrelated package, |
|
| 90 | 91 | /// whose typed error tags must agree. |
test/trap/machine.ras
+4 -2
| 1 | 1 | //! Machine and user trap entry with a nested machine timer interrupt. |
|
| 2 | + | // Bytes in the complete integer and vector trap frame. |
|
| 3 | + | .constant TRAP_TEST_FRAME_SIZE 1344; |
|
| 2 | 4 | // Machine completion device address. |
|
| 3 | 5 | .constant TRAP_TEST_FINISH_ADDRESS 0x10001000; |
|
| 4 | 6 | // Successful machine completion value. |
|
| 5 | 7 | .constant TRAP_TEST_FINISH_PASS 0x5555; |
|
| 6 | 8 | // Failed machine completion value. |
| 302 | 304 | ld %t0 264(%a0); |
|
| 303 | 305 | li %t1 TRAP_TEST_MPP_MACHINE; |
|
| 304 | 306 | and %t0 %t0 %t1; |
|
| 305 | 307 | bne %t0 %t1 @fail; |
|
| 306 | 308 | ld %t0 16(%a0); |
|
| 307 | - | addi %t1 %a0 288; |
|
| 309 | + | addi %t1 %a0 TRAP_TEST_FRAME_SIZE; |
|
| 308 | 310 | bne %t0 %t1 @fail; |
|
| 309 | 311 | li %t0 TRAP_TEST_TIMER_COMPARE; |
|
| 310 | 312 | li %t1 -1; |
|
| 311 | 313 | sd %t1 0(%t0); |
|
| 312 | 314 | li %t0 2; |
| 334 | 336 | ld %t0 264(%a0); |
|
| 335 | 337 | li %t1 TRAP_TEST_MPP_MACHINE; |
|
| 336 | 338 | and %t0 %t0 %t1; |
|
| 337 | 339 | bnez %t0 @fail; |
|
| 338 | 340 | ld %t0 0(%tp); |
|
| 339 | - | addi %t0 %t0 -288; |
|
| 341 | + | addi %t0 %t0 -TRAP_TEST_FRAME_SIZE; |
|
| 340 | 342 | bne %a0 %t0 @fail; |
|
| 341 | 343 | @advance |
|
| 342 | 344 | ld %t0 256(%a0); |
|
| 343 | 345 | addi %t0 %t0 4; |
|
| 344 | 346 | sd %t0 256(%a0); |
test/trap/vector.ras
added
+221 -0
| 1 | + | //! Complete vector state across machine traps, nested timers and user switches. |
|
| 2 | + | // Machine completion device address and result values. |
|
| 3 | + | .constant VECTOR_TEST_FINISH 0x10001000; |
|
| 4 | + | .constant VECTOR_TEST_PASS 0x5555; |
|
| 5 | + | .constant VECTOR_TEST_FAIL 0x13333; |
|
| 6 | + | // Vector state enable bits in mstatus. |
|
| 7 | + | .constant VECTOR_TEST_VS 0x600; |
|
| 8 | + | // Bytes in the complete integer and vector trap frame. |
|
| 9 | + | .constant VECTOR_TEST_FRAME 1344; |
|
| 10 | + | // Hart zero timer compare register. |
|
| 11 | + | .constant VECTOR_TEST_COMPARE 0x02004000; |
|
| 12 | + | ||
| 13 | + | .text; |
|
| 14 | + | li %s0 0x80020000; |
|
| 15 | + | sd %sp 0(%s0); |
|
| 16 | + | li %t0 16384; |
|
| 17 | + | sub %t0 %sp %t0; |
|
| 18 | + | sd %t0 8(%s0); |
|
| 19 | + | sd %zero 16(%s0); |
|
| 20 | + | la %t0 @handler; |
|
| 21 | + | sd %t0 24(%s0); |
|
| 22 | + | sd %zero 64(%s0); |
|
| 23 | + | mv %a0 %s0; |
|
| 24 | + | call @kernel::trap::install; |
|
| 25 | + | li %t0 VECTOR_TEST_VS; |
|
| 26 | + | csrs mstatus %t0; |
|
| 27 | + | call @pattern; |
|
| 28 | + | li %t0 0x80026000; |
|
| 29 | + | call @store; |
|
| 30 | + | call @configure; |
|
| 31 | + | ecall; |
|
| 32 | + | call @check; |
|
| 33 | + | // Enter a user context with the same known vector contents. |
|
| 34 | + | call @configure; |
|
| 35 | + | la %t0 @user; |
|
| 36 | + | csrw mepc %t0; |
|
| 37 | + | li %t0 VECTOR_TEST_VS; |
|
| 38 | + | csrw mstatus %t0; |
|
| 39 | + | li %sp 0x80023000; |
|
| 40 | + | mret; |
|
| 41 | + | @user |
|
| 42 | + | ecall; |
|
| 43 | + | call @check; |
|
| 44 | + | li %a7 1; |
|
| 45 | + | ecall; |
|
| 46 | + | j @fail; |
|
| 47 | + | @peer |
|
| 48 | + | call @clobber; |
|
| 49 | + | ecall; |
|
| 50 | + | j @fail; |
|
| 51 | + | ||
| 52 | + | // Fill all vector lanes and registers with group-specific values. |
|
| 53 | + | @pattern |
|
| 54 | + | li %t0 64; |
|
| 55 | + | vsetvli %zero %t0 e32 m8 ta ma; |
|
| 56 | + | vid.v %v0; |
|
| 57 | + | vadd.vi %v8 %v0 1; |
|
| 58 | + | vadd.vi %v16 %v0 2; |
|
| 59 | + | vadd.vi %v24 %v0 3; |
|
| 60 | + | ret; |
|
| 61 | + | ||
| 62 | + | // Store every vector register at t0 with vstart cleared. |
|
| 63 | + | @store |
|
| 64 | + | csrw vstart %zero; |
|
| 65 | + | vs8r.v %v0 (%t0); |
|
| 66 | + | addi %t0 %t0 256; |
|
| 67 | + | vs8r.v %v8 (%t0); |
|
| 68 | + | addi %t0 %t0 256; |
|
| 69 | + | vs8r.v %v16 (%t0); |
|
| 70 | + | addi %t0 %t0 256; |
|
| 71 | + | vs8r.v %v24 (%t0); |
|
| 72 | + | ret; |
|
| 73 | + | ||
| 74 | + | // Configure a partial vector, a restart index and fixed-point state. |
|
| 75 | + | @configure |
|
| 76 | + | vsetivli %zero 3 e32 m2 ta ma; |
|
| 77 | + | li %t0 5; |
|
| 78 | + | csrw vcsr %t0; |
|
| 79 | + | li %t0 2; |
|
| 80 | + | csrw vstart %t0; |
|
| 81 | + | ret; |
|
| 82 | + | ||
| 83 | + | // Check CSRs and every byte, including vector tails. |
|
| 84 | + | @check |
|
| 85 | + | csrr %t0 vstart; |
|
| 86 | + | li %t1 2; |
|
| 87 | + | bne %t0 %t1 @fail; |
|
| 88 | + | csrw vstart %zero; |
|
| 89 | + | csrr %t0 vl; |
|
| 90 | + | li %t1 3; |
|
| 91 | + | bne %t0 %t1 @fail; |
|
| 92 | + | csrr %t0 vtype; |
|
| 93 | + | li %t1 209; |
|
| 94 | + | bne %t0 %t1 @fail; |
|
| 95 | + | csrr %t0 vcsr; |
|
| 96 | + | li %t1 5; |
|
| 97 | + | bne %t0 %t1 @fail; |
|
| 98 | + | mv %t6 %ra; |
|
| 99 | + | li %t0 0x80027000; |
|
| 100 | + | call @store; |
|
| 101 | + | mv %ra %t6; |
|
| 102 | + | li %t0 0x80026000; |
|
| 103 | + | li %t1 0x80027000; |
|
| 104 | + | li %t2 128; |
|
| 105 | + | @compare |
|
| 106 | + | ld %t3 0(%t0); |
|
| 107 | + | ld %t4 0(%t1); |
|
| 108 | + | bne %t3 %t4 @fail; |
|
| 109 | + | addi %t0 %t0 8; |
|
| 110 | + | addi %t1 %t1 8; |
|
| 111 | + | addi %t2 %t2 -1; |
|
| 112 | + | bnez %t2 @compare; |
|
| 113 | + | ret; |
|
| 114 | + | ||
| 115 | + | // Replace all live vector data and configuration. |
|
| 116 | + | @clobber |
|
| 117 | + | csrw vstart %zero; |
|
| 118 | + | li %t0 64; |
|
| 119 | + | vsetvli %zero %t0 e32 m8 ta ma; |
|
| 120 | + | vmv.v.i %v0 -1; |
|
| 121 | + | vmv.v.i %v8 -1; |
|
| 122 | + | vmv.v.i %v16 -1; |
|
| 123 | + | vmv.v.i %v24 -1; |
|
| 124 | + | csrw vcsr %zero; |
|
| 125 | + | ret; |
|
| 126 | + | ||
| 127 | + | // A timer interrupts an outer handler while vector state is live. |
|
| 128 | + | @handler |
|
| 129 | + | ld %t0 272(%a0); |
|
| 130 | + | blt %t0 %zero @timer; |
|
| 131 | + | ld %t0 64(%s0); |
|
| 132 | + | beqz %t0 @machine; |
|
| 133 | + | li %t1 2; |
|
| 134 | + | beq %t0 %t1 @switchPeer; |
|
| 135 | + | li %t1 3; |
|
| 136 | + | beq %t0 %t1 @switchUser; |
|
| 137 | + | li %t0 VECTOR_TEST_FINISH; |
|
| 138 | + | li %t1 VECTOR_TEST_PASS; |
|
| 139 | + | sw %t1 0(%t0); |
|
| 140 | + | @machine |
|
| 141 | + | li %t0 1; |
|
| 142 | + | sd %t0 64(%s0); |
|
| 143 | + | addi %sp %sp -16; |
|
| 144 | + | sd %ra 0(%sp); |
|
| 145 | + | sd %a0 8(%sp); |
|
| 146 | + | call @pattern; |
|
| 147 | + | call @configure; |
|
| 148 | + | li %t0 VECTOR_TEST_COMPARE; |
|
| 149 | + | sd %zero 0(%t0); |
|
| 150 | + | li %t0 128; |
|
| 151 | + | csrw mie %t0; |
|
| 152 | + | csrsi mstatus 8; |
|
| 153 | + | @await |
|
| 154 | + | ld %t0 64(%s0); |
|
| 155 | + | li %t1 2; |
|
| 156 | + | bne %t0 %t1 @await; |
|
| 157 | + | csrci mstatus 8; |
|
| 158 | + | csrw mie %zero; |
|
| 159 | + | call @check; |
|
| 160 | + | call @clobber; |
|
| 161 | + | ld %ra 0(%sp); |
|
| 162 | + | ld %a0 8(%sp); |
|
| 163 | + | addi %sp %sp 16; |
|
| 164 | + | j @advance; |
|
| 165 | + | @timer |
|
| 166 | + | li %t0 VECTOR_TEST_COMPARE; |
|
| 167 | + | li %t1 -1; |
|
| 168 | + | sd %t1 0(%t0); |
|
| 169 | + | mv %t5 %ra; |
|
| 170 | + | call @clobber; |
|
| 171 | + | mv %ra %t5; |
|
| 172 | + | li %t0 2; |
|
| 173 | + | sd %t0 64(%s0); |
|
| 174 | + | ret; |
|
| 175 | + | @switchPeer |
|
| 176 | + | // Save the first context and initialize a separate zero vector context. |
|
| 177 | + | li %t0 0x80024000; |
|
| 178 | + | mv %t1 %a0; |
|
| 179 | + | li %t2 VECTOR_TEST_FRAME / 8; |
|
| 180 | + | @saveFrame |
|
| 181 | + | ld %t3 0(%t1); |
|
| 182 | + | sd %t3 0(%t0); |
|
| 183 | + | addi %t0 %t0 8; |
|
| 184 | + | addi %t1 %t1 8; |
|
| 185 | + | addi %t2 %t2 -1; |
|
| 186 | + | bnez %t2 @saveFrame; |
|
| 187 | + | addi %t0 %a0 288; |
|
| 188 | + | li %t1 132; |
|
| 189 | + | @clearVector |
|
| 190 | + | sd %zero 0(%t0); |
|
| 191 | + | addi %t0 %t0 8; |
|
| 192 | + | addi %t1 %t1 -1; |
|
| 193 | + | bnez %t1 @clearVector; |
|
| 194 | + | la %t0 @peer; |
|
| 195 | + | sd %t0 256(%a0); |
|
| 196 | + | li %t0 3; |
|
| 197 | + | sd %t0 64(%s0); |
|
| 198 | + | ret; |
|
| 199 | + | @switchUser |
|
| 200 | + | li %t0 0x80024000; |
|
| 201 | + | mv %t1 %a0; |
|
| 202 | + | li %t2 VECTOR_TEST_FRAME / 8; |
|
| 203 | + | @restoreFrame |
|
| 204 | + | ld %t3 0(%t0); |
|
| 205 | + | sd %t3 0(%t1); |
|
| 206 | + | addi %t0 %t0 8; |
|
| 207 | + | addi %t1 %t1 8; |
|
| 208 | + | addi %t2 %t2 -1; |
|
| 209 | + | bnez %t2 @restoreFrame; |
|
| 210 | + | li %t0 4; |
|
| 211 | + | sd %t0 64(%s0); |
|
| 212 | + | @advance |
|
| 213 | + | ld %t0 256(%a0); |
|
| 214 | + | addi %t0 %t0 4; |
|
| 215 | + | sd %t0 256(%a0); |
|
| 216 | + | ret; |
|
| 217 | + | @fail |
|
| 218 | + | li %t0 VECTOR_TEST_FINISH; |
|
| 219 | + | li %t1 VECTOR_TEST_FAIL; |
|
| 220 | + | sw %t1 0(%t0); |
|
| 221 | + | ebreak; |