kernel: Preserve vector execution contexts

466ae14e5f857810e7cc6a1c0df94372764b5aaae10bffc1a32aad8659646b9d
Save all 32 vector registers and vl, vtype, vstart, and vcsr in each
trap frame. Restore that state across nested interrupts, dispatch, and
user context switches. Clear vstart before whole-register transfers
and restore it after the vector configuration.

Enable vector execution for kernel handlers and initialize new
contexts with zero vector state. Check the 256-bit native vector ABI
during trap installation. Cover register tails, nested timers, context
switches, and both context creation paths.

Assisted-by: Codex:gpt-6
Change-Id: I34ec4c799361ed37b994154ef94310e7ffb16831
Alexis Sellier committed ago 1 parent 493954d4
kernel/kernel/boot.ras +4 -1
1 1
//! RV64 machine startup. Firmware supplies a0=hart, a1=FDT, and aligned sp.
2 +
// Initial vector state value in mstatus.
3 +
.constant BOOT_VS_INITIAL 512;
2 4
.text;
3 5
.export @kernel::boot::start;
4 6
.export @kernel::boot::initialize;
5 7
6 8
@kernel::boot::start
9 11
    ebreak;
10 12
11 13
// Initialize machine CSRs and preserve the firmware stack across Radiance entry.
12 14
@kernel::boot::initialize
13 15
    csrw mie %zero;
14 -
    csrw mstatus %zero;
16 +
    li %t0 BOOT_VS_INITIAL;
17 +
    csrw mstatus %t0;
15 18
    csrw mscratch %sp;
16 19
    mv %a2 %sp;
17 20
    mv %a5 %gp;
18 21
    addi %sp %sp -16;
19 22
    sd %ra 0(%sp);
kernel/kernel/dispatch.rad +18 -16
1 1
//! Hart-local selection of runnable CPU authority and timer boundaries.
2 2
3 -
use super::state;
3 +
use std::arch::rv64;
4 +
use std::mem;
4 5
use super::abi;
5 -
use super::limits;
6 -
use super::slots;
7 6
use super::budgets;
7 +
use super::calls;
8 8
use super::domains;
9 -
use super::lifecycle;
10 -
use super::loader;
11 -
use super::registry;
9 +
use super::events;
12 10
use super::frames;
13 11
use super::interrupts;
14 -
use super::events;
12 +
use super::lifecycle;
13 +
use super::limits;
14 +
use super::loader;
15 15
use super::pages;
16 16
use super::platform;
17 -
use super::trap;
18 -
use super::range;
19 -
use super::calls;
20 -
use super::timers;
21 -
use super::sync;
22 -
use super::remote;
23 17
use super::plic;
18 +
use super::range;
24 19
use super::registers;
25 -
use std::mem;
26 -
use std::arch::rv64;
20 +
use super::registry;
21 +
use super::remote;
22 +
use super::slots;
23 +
use super::state;
24 +
use super::sync;
25 +
use super::timers;
26 +
use super::trap;
27 27
28 28
/// Base maintenance passes per second.
29 29
constant MAINTENANCE_HZ: u64 = 100;
30 30
/// Maximum frames reclaimed in one maintenance pass.
31 31
constant RECLAIM_BATCH: u32 = 256;
117 117
    let stack = machine.stacks[id];
118 118
    let mut idle = trap::Frame {
119 119
        registers: [0; rv64::NUM_REGISTERS],
120 120
        pc: idleAddress(),
121 121
        status: rv64::MSTATUS_MPP_MACHINE
122 -
            | rv64::MSTATUS_MPIE,
122 +
            | rv64::MSTATUS_MPIE
123 +
            | rv64::MSTATUS_VS_INITIAL,
123 124
        cause: 0,
124 125
        value: 0,
126 +
        vector: trap::EMPTY_VECTOR,
125 127
    };
126 128
    set idle.registers[*rv64::SP as u32] = stack.end;
127 129
    set idle.registers[*rv64::GP as u32] = kernelGp;
128 130
    let mut maintenance = machine.timebase as u64 / MAINTENANCE_HZ;
129 131
    if maintenance == 0 {
kernel/kernel/dispatch.ras +1 -1
2 2
// Machine interrupt-enable bit in mstatus.
3 3
.constant DISPATCH_MIE 8;
4 4
// Machine software, timer, and external interrupt enable bits.
5 5
.constant DISPATCH_SOURCES 2184;
6 6
// Bytes in the saved Frame record.
7 -
.constant DISPATCH_FRAME_SIZE 288;
7 +
.constant DISPATCH_FRAME_SIZE 1344;
8 8
// Bytes in one saved frame word.
9 9
.constant DISPATCH_WORD_SIZE 8;
10 10
// Words copied into the retained dispatch frame.
11 11
.constant DISPATCH_FRAME_WORDS DISPATCH_FRAME_SIZE / DISPATCH_WORD_SIZE;
12 12
// Hart dispatchFrame field offset.
kernel/kernel/domains.rad +4 -2
735 735
    let kernelStack = try! frames::extent(&memory.backings.pool, kernelFrames);
736 736
    let userStack = try! pages::consumeStack(memory, targetTable, abi::Handle(start.stack));
737 737
    let mut frame = trap::Frame {
738 738
        registers: [0; rv64::NUM_REGISTERS],
739 739
        pc: start.entry,
740 -
        status: rv64::MSTATUS_MPIE,
740 +
        status: rv64::MSTATUS_MPIE | rv64::MSTATUS_VS_INITIAL,
741 741
        cause: 0,
742 742
        value: 0,
743 +
        vector: trap::EMPTY_VECTOR,
743 744
    };
744 745
    set frame.registers[*rv64::RA as u32] = returnAddress();
745 746
    set frame.registers[*rv64::SP as u32] = userStack.end;
746 747
    set frame.registers[*rv64::GP as u32] =
747 748
        instances::address(&store.tables[domain.index]);
1178 1179
            ));
1179 1180
            // MPIE enables interrupts after the first return into U-mode.
1180 1181
            let mut frame = trap::Frame {
1181 1182
                registers: [0; rv64::NUM_REGISTERS],
1182 1183
                pc: creation.entry,
1183 -
                status: rv64::MSTATUS_MPIE,
1184 +
                status: rv64::MSTATUS_MPIE | rv64::MSTATUS_VS_INITIAL,
1184 1185
                cause: 0,
1185 1186
                value: 0,
1187 +
                vector: trap::EMPTY_VECTOR,
1186 1188
            };
1187 1189
            set frame.registers[*rv64::RA as u32] = returnAddress();
1188 1190
            let graphFrames = instances::publish(graph, &mut store.tables[object.index]);
1189 1191
            set frame.registers[*rv64::GP as u32] =
1190 1192
                instances::address(&store.tables[object.index]);
kernel/kernel/tests/budgets.rad +6 -4
1 1
//! Conservation and expiration of exclusive CPU windows.
2 2
3 3
use std::testing;
4 4
use kernel::abi;
5 -
use kernel::capability;
6 5
use kernel::budgets;
7 -
use kernel::slots;
6 +
use kernel::capability;
8 7
use kernel::domains;
9 8
use kernel::frames;
10 9
use kernel::instances;
11 10
use kernel::physical;
12 -
use kernel::trap;
13 11
use kernel::range;
12 +
use kernel::slots;
13 +
use kernel::trap;
14 14
15 15
/// Exclusive CPU-window metadata.
16 16
unsafe static STORE: budgets::Store = undefined;
17 17
/// Capability owner for the test windows.
18 18
unsafe static TABLE: capability::Table = undefined;
295 295
        kernelFrames: frames::Run { first: 0, count: 0 },
296 296
        kernelStack: range::Range {
297 297
            start: 0, end: 0
298 298
        },
299 299
        userStack: range::Range { start: 0, end: 0 },
300 -
        frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 },
300 +
        frame: trap::Frame {
301 +
            registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0, vector: trap::EMPTY_VECTOR
302 +
        },
301 303
    };
302 304
    set DOMAINS.records[child.index] = domains::Domain {
303 305
        state: domains::Lifecycle::Pending,
304 306
        creator: root,
305 307
        parent: root,
kernel/kernel/tests/calls.rad +39 -19
1 1
//! Direct-call scalar results, checked record copies, and architectural replies.
2 2
3 3
use std::testing;
4 -
use kernel::state;
5 -
use kernel::sync;
6 4
use kernel::abi;
7 -
use kernel::frames;
8 -
use kernel::physical;
9 5
use kernel::access;
6 +
use kernel::backing;
7 +
use kernel::budgets;
10 8
use kernel::calls;
9 +
use kernel::capability;
10 +
use kernel::devices;
11 11
use kernel::domains;
12 +
use kernel::events;
13 +
use kernel::frames;
14 +
use kernel::interrupts;
15 +
use kernel::lifecycle;
16 +
use kernel::limits;
12 17
use kernel::pages;
13 -
use kernel::backing;
14 -
use kernel::capability;
15 -
use kernel::slots;
18 +
use kernel::physical;
19 +
use kernel::platform;
16 20
use kernel::registry;
17 -
use kernel::limits;
18 -
use kernel::trap;
19 -
use kernel::budgets;
20 21
use kernel::remote;
21 -
use kernel::interrupts;
22 -
use kernel::events;
23 -
use kernel::devices;
24 -
use kernel::platform;
25 -
use kernel::lifecycle;
22 +
use kernel::slots;
23 +
use kernel::state;
24 +
use kernel::sync;
25 +
use kernel::trap;
26 26
27 27
/// Event storage for administrative destruction targets.
28 28
unsafe static TARGETS: [domains::Memory; 2] = undefined;
29 29
30 30
/// Retained dead-domain records cannot be used or delegated through handles.
873 873
@test unsafe fn invalidQueryAborts() throws (testing::TestError) {
874 874
    for operation in [80 as u64, 81, 82, 83, 84] {
875 875
        let owner = initialize();
876 876
        try! events::open(&mut state::SHARED.domains.events, owner, &mut MEMORY.ring);
877 877
        let mut frame = trap::Frame {
878 -
            registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0
878 +
            registers: [0; 32],
879 +
            pc: 0x80000000,
880 +
            status: 0x80,
881 +
            cause: 8,
882 +
            value: 0,
883 +
            vector: trap::EMPTY_VECTOR,
879 884
        };
880 885
        set frame.registers[17] = operation;
881 886
        calls::handle(owner, &mut frame, 0, 0, unlocked);
882 887
        assert state::SHARED.domains.records[owner.index].state == domains::Lifecycle::Dead;
883 888
        assert state::SHARED.domains.events.terminals[owner.index].event.code == abi::FAULT_ABORT;
887 892
888 893
/// An unknown user operation advances PC and returns only a negative a0 error.
889 894
@test unsafe fn trapReply() throws (testing::TestError) {
890 895
    let owner = initialize();
891 896
    let mut frame = trap::Frame {
892 -
        registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0
897 +
        registers: [123; 32],
898 +
        pc: 0x80000000,
899 +
        status: 0x80,
900 +
        cause: 8,
901 +
        value: 0,
902 +
        vector: trap::EMPTY_VECTOR,
893 903
    };
894 904
    set frame.registers[17] = 0xffffffffffffffff;
895 905
    calls::handle(owner, &mut frame, 0, 0, unlocked);
896 906
    try testing::expect(frame.pc == 0x80000004 and frame.registers[10] == 0xfffffffffffffffc);
897 907
    for i in 0 .. 32 {
957 967
    let owner = initialize();
958 968
    let handle = try! allocate(owner, 1);
959 969
    let base = &RAM[OFFSET] as u64;
960 970
    set RAM[OFFSET + 2] = 0x12345678;
961 971
    let mut frame = trap::Frame {
962 -
        registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0
972 +
        registers: [0; 32],
973 +
        pc: 0x80000000,
974 +
        status: 0x80,
975 +
        cause: 8,
976 +
        value: 0,
977 +
        vector: trap::EMPTY_VECTOR,
963 978
    };
964 979
    set frame.registers[17] = 80;
965 980
    set frame.registers[10] = handle;
966 981
    set frame.registers[14] = base;
967 982
    set frame.registers[15] = 16;
1006 1021
    set state::SHARED.pages.records[object.index].base = 0x7fff00000000000;
1007 1022
    set state::SHARED.pages.records[object.index].count = 0x3000000;
1008 1023
    let region = try! calls::invokeReply(owner, 32, &[handle, 0, 0, 0], 0);
1009 1024
    try testing::expect(region.value == 0x7fff00000000000 and region.extra == 0x3000000 * 4096);
1010 1025
    let mut frame = trap::Frame {
1011 -
        registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0
1026 +
        registers: [0; 32],
1027 +
        pc: 0x80000000,
1028 +
        status: 0x80,
1029 +
        cause: 8,
1030 +
        value: 0,
1031 +
        vector: trap::EMPTY_VECTOR,
1012 1032
    };
1013 1033
    set frame.registers[17] = 32;
1014 1034
    set frame.registers[10] = handle;
1015 1035
    calls::handle(owner, &mut frame, 0, 0, unlocked);
1016 1036
    try testing::expect(
kernel/kernel/tests/dispatch.rad +5 -3
1 1
//! Dispatch selection at CPU-window boundaries and idle gaps.
2 2
3 3
use std::testing;
4 4
use kernel::abi;
5 -
use kernel::slots;
6 -
use kernel::domains;
7 5
use kernel::budgets;
8 6
use kernel::capability;
9 7
use kernel::dispatch;
8 +
use kernel::domains;
10 9
use kernel::platform;
11 10
use kernel::range;
12 11
use kernel::registers;
13 12
use kernel::remote;
13 +
use kernel::slots;
14 14
use kernel::trap;
15 15
16 16
/// Platform resources used to validate CLINT address selection.
17 17
unsafe static PLATFORM: platform::Platform = undefined;
18 18
159 159
fn handler(frame: &mut trap::Frame, hart: &mut trap::Hart) {}
160 160
161 161
/// Exchange preserves an M-mode continuation while another context uses its own stack.
162 162
@test unsafe fn continuations() throws (testing::TestError) {
163 163
    initialize();
164 -
    let mut frame = trap::Frame { registers: [0; 32], pc: 0x40, status: 0x80, cause: 0, value: 0 };
164 +
    let mut frame = trap::Frame {
165 +
        registers: [0; 32], pc: 0x40, status: 0x80, cause: 0, value: 0, vector: trap::EMPTY_VECTOR
166 +
    };
165 167
    set DOMAINS.contexts[0].frame = frame;
166 168
    set DOMAINS.contexts[0].kernelStack = range::Range { start: 0x8000, end: 0xc000 };
167 169
    set DOMAINS.contextSlots[1] = slots::Slot { generation: 1, state: slots::State::Live };
168 170
    set DOMAINS.contexts[1].owner = TABLE.owner;
169 171
    set DOMAINS.contexts[1].state = domains::ContextState::Ready;
kernel/kernel/tests/domains.rad +15 -1
1 1
//! Pending domain creation, initial context state, and creation authority.
2 2
3 +
use std::arch::rv64;
3 4
use std::arch::rv64::shared;
4 5
use std::testing;
5 6
use super::support;
6 7
use kernel::abi;
7 8
use kernel::backing;
19 20
use kernel::physical;
20 21
use kernel::registry;
21 22
use kernel::slots;
22 23
use kernel::state;
23 24
use kernel::transactions;
25 +
use kernel::trap;
24 26
25 27
/// Exclusive resources released by the domain reaper.
26 28
unsafe static WINDOWS: budgets::Store = undefined;
27 29
/// Interrupt ownership released after hardware masking.
28 30
unsafe static IRQS: interrupts::Store = undefined;
995 997
    try testing::expect(
996 998
        created.kernelStack.start >= initial.kernelStack.end
997 999
            or created.kernelStack.end <= initial.kernelStack.start
998 1000
    );
999 1001
    try testing::expect(created.frame.pc == later.entry);
1002 +
    freshVector(&created.frame);
1000 1003
    try testing::expect(
1001 1004
        created.frame.registers[2] == secondPage.base + 8192
1002 1005
            and created.frame.registers[4] == secondPage.base
1003 1006
    );
1004 1007
    try testing::expect(
1368 1371
    try testing::expect(
1369 1372
        context.frame.registers[3] == instances::address(
1370 1373
            &DOMAINS.tables[authority.object.index]
1371 1374
        )
1372 1375
    );
1373 -
    try testing::expect(context.frame.registers[2] == 0 and context.frame.status == 0x80);
1376 +
    try testing::expect(context.frame.registers[2] == 0);
1377 +
    freshVector(&context.frame);
1374 1378
    try testing::expect(context.kernelFrames.count == domains::KERNEL_STACK_PAGES);
1375 1379
    try testing::expect(
1376 1380
        context.kernelStack.end - context.kernelStack.start
1377 1381
            == domains::KERNEL_STACK_PAGES as u64 * 4096
1378 1382
    );
1523 1527
    try testing::expect(DOMAINS.contextSlots[0].state == slots::State::Free);
1524 1528
    for i in 0 .. BACKINGS.pool.count {
1525 1529
        assert frames::available(&BACKINGS.pool, i);
1526 1530
    }
1527 1531
}
1532 +
1533 +
/// New user contexts enable vectors with zero registers and no active elements.
1534 +
fn freshVector(frame: &trap::Frame) {
1535 +
    assert frame.status == (rv64::MSTATUS_MPIE | rv64::MSTATUS_VS_INITIAL);
1536 +
    assert frame.vector.length == 0 and frame.vector.type == 0;
1537 +
    assert frame.vector.start == 0 and frame.vector.control == 0;
1538 +
    for word in &frame.vector.registers[..] {
1539 +
        assert word == 0;
1540 +
    }
1541 +
}
kernel/kernel/tests/trap.rad +3 -3
1 1
//! Trap classification, frame layout, and resume-address boundaries.
2 2
3 3
use std::testing;
4 4
use kernel::trap;
5 5
6 -
/// The assembly frame includes x0-x31 and all four trap CSRs.
6 +
/// The assembly frame includes integer and vector registers and their CSRs.
7 7
@test fn layout() throws (testing::TestError) {
8 -
    try testing::expect(@sizeOf(trap::Frame) == 288);
9 -
    assert @sizeOf(trap::Hart) == 352;
8 +
    try testing::expect(@sizeOf(trap::Frame) == 1344);
9 +
    assert @sizeOf(trap::Hart) == 1408;
10 10
    try testing::expect(trap::FRAME_SIZE % 16 == 0);
11 11
}
12 12
13 13
/// Interrupts retain their source identity; exceptions retain their fault class.
14 14
@test fn classification() throws (testing::TestError) {
kernel/kernel/trap.rad +30 -4
1 -
//! Integer trap contexts and exception classification.
1 +
//! Integer and vector trap contexts and exception classification.
2 2
3 -
use std::mem;
4 3
use std::arch::rv64;
4 +
use std::arch::rv64::vector;
5 +
use std::mem;
6 +
5 7
/// Frame bytes reserved by trap entry. This is a multiple of stack alignment.
6 -
export constant FRAME_SIZE: u32 = 288;
8 +
export constant FRAME_SIZE: u32 = 1344;
9 +
10 +
/// Number of 64-bit words in one register of the native vector ABI.
11 +
constant VECTOR_WORDS_PER_REGISTER: u32 = 4;
12 +
13 +
/// Saved vector registers and CSRs for the 256-bit native vector ABI.
14 +
export record VectorState: Copy {
15 +
    /// Thirty-two vector registers, with four 64-bit words per register.
16 +
    registers: [u64; vector::REGISTER_COUNT * VECTOR_WORDS_PER_REGISTER],
17 +
    /// Number of active vector elements.
18 +
    length: u64,
19 +
    /// Vector element width, grouping and inactive element policies.
20 +
    type: u64,
21 +
    /// Element index at which vector execution resumes.
22 +
    start: u64,
23 +
    /// Fixed-point rounding mode and saturation flag.
24 +
    control: u64,
25 +
}
26 +
27 +
/// Zero vector registers with no active elements.
28 +
export constant EMPTY_VECTOR: VectorState = VectorState {
29 +
    registers: [0; vector::REGISTER_COUNT * VECTOR_WORDS_PER_REGISTER], length: 0, type: 0, start: 0, control: 0
30 +
};
7 31
8 32
/// Saved architectural state. Assembly offsets follow field declaration order.
9 33
export record Frame: Copy {
10 34
    /// Integer registers indexed by architectural register number. x0 is zero.
11 35
    registers: [u64; rv64::NUM_REGISTERS],
15 39
    status: u64,
16 40
    /// Interrupt bit and exception code captured from mcause.
17 41
    cause: u64,
18 42
    /// Fault address or instruction bits captured from mtval.
19 43
    value: u64,
44 +
    /// Complete vector state captured before the trap handler runs.
45 +
    vector: VectorState,
20 46
}
21 47
22 48
/// Per-hart trap entry state named by mscratch. Entry scratch fields are private
23 49
/// to the assembly prologue while machine interrupts are disabled.
24 50
export record Hart: Copy {
111 137
}
112 138
113 139
/// Install trap entry for the current hart with machine interrupts disabled.
114 140
/// The Hart must stay valid for the program and belong to the executing hart.
115 141
export unsafe fn install(hart: *mut Hart);
116 -
/// Restore a complete integer context through mret. The current mscratch must
142 +
/// Restore a complete integer and vector context through mret. The current mscratch must
117 143
/// name an installed Hart, and the frame must have machine interrupts disabled.
118 144
export unsafe fn resume(frame: *Frame) -> !;
119 145
120 146
/// Disable machine interrupts and stop the current hart.
121 147
export fn halt() -> !;
kernel/kernel/trap.ras +57 -2
1 -
//! RV64 integer trap entry. mscratch names the current hart's kernel state.
1 +
//! RV64 integer and vector trap entry. mscratch names the current hart's kernel state.
2 2
// Machine interrupt-enable bit in mstatus.
3 3
.constant TRAP_MIE 8;
4 4
// Mask that clears the machine interrupt-enable bit.
5 5
.constant TRAP_MIE_CLEAR -TRAP_MIE - 1;
6 6
// Bit offset of the previous privilege field.
10 10
// Previous privilege value for machine mode.
11 11
.constant TRAP_MPP_MACHINE 3;
12 12
// Mask for the required 16-byte stack alignment.
13 13
.constant TRAP_STACK_MASK 15;
14 14
// Bytes in the saved Frame record.
15 -
.constant TRAP_FRAME_SIZE 288;
15 +
.constant TRAP_FRAME_SIZE 1344;
16 16
// Hart stackTop field offset.
17 17
.constant TRAP_HART_STACK_TOP 0;
18 18
// Hart stackBottom field offset.
19 19
.constant TRAP_HART_STACK_BOTTOM 8;
20 20
// Hart kernelGp field offset.
34 34
// Frame cause field offset.
35 35
.constant TRAP_FRAME_CAUSE 272;
36 36
// Frame value field offset.
37 37
.constant TRAP_FRAME_VALUE 280;
38 38
39 +
// Initial vector state value in mstatus.
40 +
.constant TRAP_VS_INITIAL 512;
41 +
// Bytes per vector register in the native ABI.
42 +
.constant TRAP_VECTOR_BYTES 32;
43 +
// Bytes in a group of eight vector registers.
44 +
.constant TRAP_VECTOR_GROUP 256;
45 +
// Frame vector register and CSR field offsets.
46 +
.constant TRAP_FRAME_VECTORS 288;
47 +
.constant TRAP_FRAME_VL 1312;
48 +
.constant TRAP_FRAME_VTYPE 1320;
49 +
.constant TRAP_FRAME_VSTART 1328;
50 +
.constant TRAP_FRAME_VCSR 1336;
51 +
39 52
.text;
40 53
.export @kernel::trap::install;
41 54
.export @kernel::trap::resume;
42 55
.export @kernel::trap::vector;
43 56
.export @kernel::trap::halt;
44 57
45 58
// Install a direct trap vector and its per-hart stack anchor.
46 59
@kernel::trap::install
47 60
    csrw mie %zero;
48 61
    csrci mstatus TRAP_MIE;
62 +
    li %t0 TRAP_VS_INITIAL;
63 +
    csrs mstatus %t0;
64 +
    csrr %t0 vlenb;
65 +
    li %t1 TRAP_VECTOR_BYTES;
66 +
    bne %t0 %t1 @stackFault;
49 67
    csrw mscratch %a0;
50 68
    la %t0 @kernel::trap::vector;
51 69
    csrw mtvec %t0;
52 70
    ret;
53 71
117 135
    sd %t0 TRAP_FRAME_STATUS(%sp);
118 136
    csrr %t0 mcause;
119 137
    sd %t0 TRAP_FRAME_CAUSE(%sp);
120 138
    csrr %t0 mtval;
121 139
    sd %t0 TRAP_FRAME_VALUE(%sp);
140 +
    li %t0 TRAP_VS_INITIAL;
141 +
    csrs mstatus %t0;
142 +
    csrr %t0 vstart;
143 +
    sd %t0 TRAP_FRAME_VSTART(%sp);
144 +
    csrw vstart %zero;
145 +
    csrr %t0 vl;
146 +
    sd %t0 TRAP_FRAME_VL(%sp);
147 +
    csrr %t0 vtype;
148 +
    sd %t0 TRAP_FRAME_VTYPE(%sp);
149 +
    csrr %t0 vcsr;
150 +
    sd %t0 TRAP_FRAME_VCSR(%sp);
151 +
    addi %t0 %sp TRAP_FRAME_VECTORS;
152 +
    vs8r.v %v0 (%t0);
153 +
    addi %t0 %t0 TRAP_VECTOR_GROUP;
154 +
    vs8r.v %v8 (%t0);
155 +
    addi %t0 %t0 TRAP_VECTOR_GROUP;
156 +
    vs8r.v %v16 (%t0);
157 +
    addi %t0 %t0 TRAP_VECTOR_GROUP;
158 +
    vs8r.v %v24 (%t0);
122 159
    csrw mscratch %tp;
123 160
    ld %gp TRAP_HART_GP(%tp);
124 161
    mv %a0 %sp;
125 162
    mv %a1 %tp;
126 163
    ld %t0 TRAP_HART_HANDLER(%tp);
131 168
@kernel::trap::resume
132 169
    csrci mstatus TRAP_MIE;
133 170
    mv %sp %a0;
134 171
    // A resumed frame must not use a reservation from another execution context.
135 172
    sc.d %zero %zero (%sp);
173 +
    li %t0 TRAP_VS_INITIAL;
174 +
    csrs mstatus %t0;
175 +
    csrw vstart %zero;
176 +
    addi %t0 %sp TRAP_FRAME_VECTORS;
177 +
    vl8re8.v %v0 (%t0);
178 +
    addi %t0 %t0 TRAP_VECTOR_GROUP;
179 +
    vl8re8.v %v8 (%t0);
180 +
    addi %t0 %t0 TRAP_VECTOR_GROUP;
181 +
    vl8re8.v %v16 (%t0);
182 +
    addi %t0 %t0 TRAP_VECTOR_GROUP;
183 +
    vl8re8.v %v24 (%t0);
184 +
    ld %t0 TRAP_FRAME_VL(%sp);
185 +
    ld %t1 TRAP_FRAME_VTYPE(%sp);
186 +
    vsetvl %zero %t0 %t1;
187 +
    ld %t0 TRAP_FRAME_VCSR(%sp);
188 +
    csrw vcsr %t0;
189 +
    ld %t0 TRAP_FRAME_VSTART(%sp);
190 +
    csrw vstart %t0;
136 191
    ld %t0 TRAP_FRAME_PC(%sp);
137 192
    csrw mepc %t0;
138 193
    ld %t0 TRAP_FRAME_STATUS(%sp);
139 194
    andi %t0 %t0 TRAP_MIE_CLEAR;
140 195
    csrw mstatus %t0;
lib/std/arch/rv64.rad +2 -0
79 79
/// Machine external interrupt cause.
80 80
export constant MCAUSE_MACHINE_EXTERNAL: u64 = MCAUSE_INTERRUPT
81 81
    | IRQ_MACHINE_EXTERNAL;
82 82
/// Machine previous interrupt-enable bit in mstatus.
83 83
export constant MSTATUS_MPIE: u64 = 1 << 7;
84 +
/// Initial vector state field value in mstatus.
85 +
export constant MSTATUS_VS_INITIAL: u64 = 1 << 9;
84 86
/// Previous privilege field mask in mstatus.
85 87
export constant MSTATUS_MPP_MASK: u64 = 3 << 11;
86 88
/// Previous privilege field value for machine mode.
87 89
export constant MSTATUS_MPP_MACHINE: u64 = 3 << 11;
88 90
test/integration/machine.rad +2 -1
71 71
72 72
/// Run the trap fixtures on one hart.
73 73
export fn trap(h: &mut Harness, args: *[*[u8]]) throws (sys::Error) {
74 74
    try h.begin("trap");
75 75
    let mark = h.failed;
76 -
    for fixture in &["machine", "reservations"] {
76 +
    for fixture in &["machine", "reservations", "vector"] {
77 77
        let sources = [h.join(&["test/trap/", fixture, ".ras"]), "kernel/kernel/trap.ras"];
78 78
        let what = h.join(&["trap fixture ", fixture]);
79 79
        try assembleAndRun(h, what, h.join(&["trap.", fixture]), &sources[..], 1);
80 80
    }
81 81
    h.done(
82 82
        mark,
83 83
        "trap contexts: machine/user registers, nested timer, illegal instruction, and stacks passed"
84 84
    );
85 85
    h.done(mark, "trap reservations: entry, resume, and user context isolation passed");
86 +
    h.done(mark, "trap vectors: complete state, nested timer and user context switches passed");
86 87
}
87 88
88 89
/// Compile two packages once and run them with two private state graphs, then
89 90
/// link a package built alone with an entry built beside an unrelated package,
90 91
/// whose typed error tags must agree.
test/trap/machine.ras +4 -2
1 1
//! Machine and user trap entry with a nested machine timer interrupt.
2 +
// Bytes in the complete integer and vector trap frame.
3 +
.constant TRAP_TEST_FRAME_SIZE 1344;
2 4
// Machine completion device address.
3 5
.constant TRAP_TEST_FINISH_ADDRESS 0x10001000;
4 6
// Successful machine completion value.
5 7
.constant TRAP_TEST_FINISH_PASS 0x5555;
6 8
// Failed machine completion value.
302 304
    ld %t0 264(%a0);
303 305
    li %t1 TRAP_TEST_MPP_MACHINE;
304 306
    and %t0 %t0 %t1;
305 307
    bne %t0 %t1 @fail;
306 308
    ld %t0 16(%a0);
307 -
    addi %t1 %a0 288;
309 +
    addi %t1 %a0 TRAP_TEST_FRAME_SIZE;
308 310
    bne %t0 %t1 @fail;
309 311
    li %t0 TRAP_TEST_TIMER_COMPARE;
310 312
    li %t1 -1;
311 313
    sd %t1 0(%t0);
312 314
    li %t0 2;
334 336
    ld %t0 264(%a0);
335 337
    li %t1 TRAP_TEST_MPP_MACHINE;
336 338
    and %t0 %t0 %t1;
337 339
    bnez %t0 @fail;
338 340
    ld %t0 0(%tp);
339 -
    addi %t0 %t0 -288;
341 +
    addi %t0 %t0 -TRAP_TEST_FRAME_SIZE;
340 342
    bne %a0 %t0 @fail;
341 343
@advance
342 344
    ld %t0 256(%a0);
343 345
    addi %t0 %t0 4;
344 346
    sd %t0 256(%a0);
test/trap/vector.ras added +221 -0
1 +
//! Complete vector state across machine traps, nested timers and user switches.
2 +
// Machine completion device address and result values.
3 +
.constant VECTOR_TEST_FINISH 0x10001000;
4 +
.constant VECTOR_TEST_PASS 0x5555;
5 +
.constant VECTOR_TEST_FAIL 0x13333;
6 +
// Vector state enable bits in mstatus.
7 +
.constant VECTOR_TEST_VS 0x600;
8 +
// Bytes in the complete integer and vector trap frame.
9 +
.constant VECTOR_TEST_FRAME 1344;
10 +
// Hart zero timer compare register.
11 +
.constant VECTOR_TEST_COMPARE 0x02004000;
12 +
13 +
.text;
14 +
    li %s0 0x80020000;
15 +
    sd %sp 0(%s0);
16 +
    li %t0 16384;
17 +
    sub %t0 %sp %t0;
18 +
    sd %t0 8(%s0);
19 +
    sd %zero 16(%s0);
20 +
    la %t0 @handler;
21 +
    sd %t0 24(%s0);
22 +
    sd %zero 64(%s0);
23 +
    mv %a0 %s0;
24 +
    call @kernel::trap::install;
25 +
    li %t0 VECTOR_TEST_VS;
26 +
    csrs mstatus %t0;
27 +
    call @pattern;
28 +
    li %t0 0x80026000;
29 +
    call @store;
30 +
    call @configure;
31 +
    ecall;
32 +
    call @check;
33 +
    // Enter a user context with the same known vector contents.
34 +
    call @configure;
35 +
    la %t0 @user;
36 +
    csrw mepc %t0;
37 +
    li %t0 VECTOR_TEST_VS;
38 +
    csrw mstatus %t0;
39 +
    li %sp 0x80023000;
40 +
    mret;
41 +
@user
42 +
    ecall;
43 +
    call @check;
44 +
    li %a7 1;
45 +
    ecall;
46 +
    j @fail;
47 +
@peer
48 +
    call @clobber;
49 +
    ecall;
50 +
    j @fail;
51 +
52 +
// Fill all vector lanes and registers with group-specific values.
53 +
@pattern
54 +
    li %t0 64;
55 +
    vsetvli %zero %t0 e32 m8 ta ma;
56 +
    vid.v %v0;
57 +
    vadd.vi %v8 %v0 1;
58 +
    vadd.vi %v16 %v0 2;
59 +
    vadd.vi %v24 %v0 3;
60 +
    ret;
61 +
62 +
// Store every vector register at t0 with vstart cleared.
63 +
@store
64 +
    csrw vstart %zero;
65 +
    vs8r.v %v0 (%t0);
66 +
    addi %t0 %t0 256;
67 +
    vs8r.v %v8 (%t0);
68 +
    addi %t0 %t0 256;
69 +
    vs8r.v %v16 (%t0);
70 +
    addi %t0 %t0 256;
71 +
    vs8r.v %v24 (%t0);
72 +
    ret;
73 +
74 +
// Configure a partial vector, a restart index and fixed-point state.
75 +
@configure
76 +
    vsetivli %zero 3 e32 m2 ta ma;
77 +
    li %t0 5;
78 +
    csrw vcsr %t0;
79 +
    li %t0 2;
80 +
    csrw vstart %t0;
81 +
    ret;
82 +
83 +
// Check CSRs and every byte, including vector tails.
84 +
@check
85 +
    csrr %t0 vstart;
86 +
    li %t1 2;
87 +
    bne %t0 %t1 @fail;
88 +
    csrw vstart %zero;
89 +
    csrr %t0 vl;
90 +
    li %t1 3;
91 +
    bne %t0 %t1 @fail;
92 +
    csrr %t0 vtype;
93 +
    li %t1 209;
94 +
    bne %t0 %t1 @fail;
95 +
    csrr %t0 vcsr;
96 +
    li %t1 5;
97 +
    bne %t0 %t1 @fail;
98 +
    mv %t6 %ra;
99 +
    li %t0 0x80027000;
100 +
    call @store;
101 +
    mv %ra %t6;
102 +
    li %t0 0x80026000;
103 +
    li %t1 0x80027000;
104 +
    li %t2 128;
105 +
@compare
106 +
    ld %t3 0(%t0);
107 +
    ld %t4 0(%t1);
108 +
    bne %t3 %t4 @fail;
109 +
    addi %t0 %t0 8;
110 +
    addi %t1 %t1 8;
111 +
    addi %t2 %t2 -1;
112 +
    bnez %t2 @compare;
113 +
    ret;
114 +
115 +
// Replace all live vector data and configuration.
116 +
@clobber
117 +
    csrw vstart %zero;
118 +
    li %t0 64;
119 +
    vsetvli %zero %t0 e32 m8 ta ma;
120 +
    vmv.v.i %v0 -1;
121 +
    vmv.v.i %v8 -1;
122 +
    vmv.v.i %v16 -1;
123 +
    vmv.v.i %v24 -1;
124 +
    csrw vcsr %zero;
125 +
    ret;
126 +
127 +
// A timer interrupts an outer handler while vector state is live.
128 +
@handler
129 +
    ld %t0 272(%a0);
130 +
    blt %t0 %zero @timer;
131 +
    ld %t0 64(%s0);
132 +
    beqz %t0 @machine;
133 +
    li %t1 2;
134 +
    beq %t0 %t1 @switchPeer;
135 +
    li %t1 3;
136 +
    beq %t0 %t1 @switchUser;
137 +
    li %t0 VECTOR_TEST_FINISH;
138 +
    li %t1 VECTOR_TEST_PASS;
139 +
    sw %t1 0(%t0);
140 +
@machine
141 +
    li %t0 1;
142 +
    sd %t0 64(%s0);
143 +
    addi %sp %sp -16;
144 +
    sd %ra 0(%sp);
145 +
    sd %a0 8(%sp);
146 +
    call @pattern;
147 +
    call @configure;
148 +
    li %t0 VECTOR_TEST_COMPARE;
149 +
    sd %zero 0(%t0);
150 +
    li %t0 128;
151 +
    csrw mie %t0;
152 +
    csrsi mstatus 8;
153 +
@await
154 +
    ld %t0 64(%s0);
155 +
    li %t1 2;
156 +
    bne %t0 %t1 @await;
157 +
    csrci mstatus 8;
158 +
    csrw mie %zero;
159 +
    call @check;
160 +
    call @clobber;
161 +
    ld %ra 0(%sp);
162 +
    ld %a0 8(%sp);
163 +
    addi %sp %sp 16;
164 +
    j @advance;
165 +
@timer
166 +
    li %t0 VECTOR_TEST_COMPARE;
167 +
    li %t1 -1;
168 +
    sd %t1 0(%t0);
169 +
    mv %t5 %ra;
170 +
    call @clobber;
171 +
    mv %ra %t5;
172 +
    li %t0 2;
173 +
    sd %t0 64(%s0);
174 +
    ret;
175 +
@switchPeer
176 +
    // Save the first context and initialize a separate zero vector context.
177 +
    li %t0 0x80024000;
178 +
    mv %t1 %a0;
179 +
    li %t2 VECTOR_TEST_FRAME / 8;
180 +
@saveFrame
181 +
    ld %t3 0(%t1);
182 +
    sd %t3 0(%t0);
183 +
    addi %t0 %t0 8;
184 +
    addi %t1 %t1 8;
185 +
    addi %t2 %t2 -1;
186 +
    bnez %t2 @saveFrame;
187 +
    addi %t0 %a0 288;
188 +
    li %t1 132;
189 +
@clearVector
190 +
    sd %zero 0(%t0);
191 +
    addi %t0 %t0 8;
192 +
    addi %t1 %t1 -1;
193 +
    bnez %t1 @clearVector;
194 +
    la %t0 @peer;
195 +
    sd %t0 256(%a0);
196 +
    li %t0 3;
197 +
    sd %t0 64(%s0);
198 +
    ret;
199 +
@switchUser
200 +
    li %t0 0x80024000;
201 +
    mv %t1 %a0;
202 +
    li %t2 VECTOR_TEST_FRAME / 8;
203 +
@restoreFrame
204 +
    ld %t3 0(%t0);
205 +
    sd %t3 0(%t1);
206 +
    addi %t0 %t0 8;
207 +
    addi %t1 %t1 8;
208 +
    addi %t2 %t2 -1;
209 +
    bnez %t2 @restoreFrame;
210 +
    li %t0 4;
211 +
    sd %t0 64(%s0);
212 +
@advance
213 +
    ld %t0 256(%a0);
214 +
    addi %t0 %t0 4;
215 +
    sd %t0 256(%a0);
216 +
    ret;
217 +
@fail
218 +
    li %t0 VECTOR_TEST_FINISH;
219 +
    li %t1 VECTOR_TEST_FAIL;
220 +
    sw %t1 0(%t0);
221 +
    ebreak;