kernel: preserve complete user state across machine traps

6eee4d97d4c4edae833a87654fe4efe39ffbdb47a07cd965fe205c0e2b449d3d
Verified: make -C kernel check; make std-test bin-test with the machine-capable emulator; all pass.
Alexis Sellier committed ago 1 parent d84aebbc
kernel/Makefile +8 -4
3 3
HOST_EMU ?= $(EMU)
4 4
COMPILER := ../bin/radiance.rv64.dev
5 5
COMPILE := $(HOST_EMU) -memory-size=385024 -data-size=348160 -stack-size=512 -run $(COMPILER)
6 6
MODULES := core/fdt.rad core/platform.rad core/frames.rad core/abi.rad core/handles.rad \
7 7
	core/events.rad core/domains.rad core/resources.rad core/capabilities.rad \
8 -
	core/memory.rad core/state.rad core/pages.rad core/atomic.rad
9 -
CORE_ASM := arch/atomic.ras
8 +
	core/memory.rad core/state.rad core/pages.rad core/atomic.rad core/cpu.rad
9 +
CORE_ASM := arch/atomic.ras arch/context.ras
10 10
CORE := -pkg core -mod core.rad $(addprefix -mod ,$(MODULES) $(CORE_ASM))
11 11
CHECK_MODULES := check/boot.rad check/fixture.rad check/frames.rad check/handles.rad \
12 12
	check/domains.rad check/capabilities.rad check/pages.rad check/events.rad
13 13
14 14
.PHONY: all check clean compiler-check
23 23
	$(COMPILE) $(CORE) -pkg kernel -start arch/entry.ras -mod main.rad -entry kernel -o $@
24 24
25 25
check.rv64: check.rad core.rad $(MODULES) $(CORE_ASM) $(CHECK_MODULES) $(COMPILER)
26 26
	$(COMPILE) $(CORE) -pkg check -mod check.rad $(addprefix -mod ,$(CHECK_MODULES)) -entry check -o $@
27 27
28 -
check: all check.rv64
28 +
context.rv64: context.rad check/context.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) $(COMPILER)
29 +
	$(COMPILE) $(CORE) -pkg context -start arch/entry.ras -mod context.rad -mod check/context.ras -entry context -o $@
30 +
31 +
check: all check.rv64 context.rv64
29 32
	$(HOST_EMU) -run check.rv64
30 33
	$(EMU) -machine -no-guard-stack -max-steps=100000000 -count-instructions -run kernel.rv64
34 +
	$(EMU) -machine -no-guard-stack -max-steps=1000000 -count-instructions -run context.rv64
31 35
32 36
clean:
33 -
	rm -f kernel.rv64 check.rv64
37 +
	rm -f kernel.rv64 check.rv64 context.rv64
kernel/NOTES.md +25 -2
1 1
# Kernel implementation decisions
2 2
3 3
The specification at https://radiant.computer/system/kernel takes precedence
4 4
for fixed call numbers, handle layout, rights, and object behavior. These notes
5 -
record the contracts established through step 9 of the 22-step plan.
5 +
record the contracts established through step 10 of the 22-step plan.
6 6
7 7
## Source and trust boundary
8 8
9 9
- Kernel mechanisms use freestanding Radiance; RAS owns machine entry, register
10 10
  state, atomics, and MMIO. Hosted checks exercise the same mechanism modules.
114 114
  relationship credits; consumption releases the credit. At most one entry per
115 115
  IRQ source is outstanding, and validated consumption permits its next event.
116 116
- Boundary checks cover full capacity, u32 wraparound, malformed head progress,
117 117
  lifecycle credit integrity, and coalescing through IRQ source 127.
118 118
119 +
## Architectural execution boundary
120 +
121 +
- A private Frame holds x0..x31, mepc, mcause, mtval, mstatus, and the suspended
122 +
  machine stack at offsets 0, 256, 264, 272, 280, and 288.
123 +
- mscratch names the active Frame. Trap entry does not use the user stack or
124 +
  trust user gp, tp, or ra. Restore the machine stack and callee-saved registers
125 +
  before returning to safe kernel code.
126 +
- Trap entry clears the hart's LR/SC reservation through the private Frame before
127 +
  kernel code resumes. A context cannot consume another context's reservation.
128 +
- Entry sets MPP=U, clears privileged execution flags, and disables floating-point
129 +
  state. Kernel return keeps machine interrupts disabled. A separate M-mode
130 +
  idle entry waits for an enabled interrupt.
131 +
- Direct calls use a7 for the operation and a0..a6 for inputs. Error returns in
132 +
  a0 and up to four values in a1..a4; other user registers survive. Returning
133 +
  calls advance past ecall on both success and failure.
134 +
- RAS supports RV64A word/doubleword LR, SC, and AMOs with aq/rl/aqrl ordering.
135 +
  Machine checks cover register preservation, acquire/release exchange, call
136 +
  replies, privilege faults, misalignment, inaccessible memory, and idle wakeup.
137 +
119 138
## Validation
120 139
121 140
Use the current machine-capable sibling emulator. Set `RAD_EMULATOR`, pass
122 141
`EMU` to the kernel Make invocation, or put `emulator` on PATH. The kernel build
123 142
checks compiler dependencies. From the repository root, run:
124 143
125 144
```sh
126 145
make -C kernel check
146 +
make std-test bin-test
127 147
```
128 148
129 -
Exercise all ring slots, ordinary backpressure, reserved delivery, IRQ coalescing, wraparound, and invalid consumer progress.
149 +
Run the context machine scenario: full integer state, ecall replies, faults, atomics, and timer wakeup from machine idle.
150 +
151 +
Run the context reservation probe with an emulator that retains LR/SC
152 +
reservations across traps. This checks the kernel's reservation invalidation.
kernel/arch/atomic.ras +6 -0
8 8
.export @"core::atomic::store";
9 9
@"core::atomic::store"
10 10
	fence;
11 11
	sw     %a1    0(%a0);
12 12
	ret;
13 +
.export @"core::atomic::exchange";
14 +
@"core::atomic::exchange"
15 +
	amoswap.w.aqrl %a0 %a1 0(%a0);
16 +
	slli   %a0    %a0    32;
17 +
	srli   %a0    %a0    32;
18 +
	ret;
kernel/arch/context.ras added +147 -0
1 +
// Integer context boundary. mscratch owns the active private Frame pointer.
2 +
.text;
3 +
.export @"core::cpu::enter";
4 +
@"core::cpu::enter"
5 +
	li     %a1    0;
6 +
	j      @saveKernel;
7 +
.export @"core::cpu::idle";
8 +
@"core::cpu::idle"
9 +
	li     %a1    1;
10 +
@saveKernel
11 +
	addi   %sp    %sp    -128;
12 +
	sd     %ra    0(%sp);
13 +
	sd     %s0    8(%sp);
14 +
	sd     %s1    16(%sp);
15 +
	sd     %s2    24(%sp);
16 +
	sd     %s3    32(%sp);
17 +
	sd     %s4    40(%sp);
18 +
	sd     %s5    48(%sp);
19 +
	sd     %s6    56(%sp);
20 +
	sd     %s7    64(%sp);
21 +
	sd     %s8    72(%sp);
22 +
	sd     %s9    80(%sp);
23 +
	sd     %s10   88(%sp);
24 +
	sd     %s11   96(%sp);
25 +
	sd     %gp    104(%sp);
26 +
	sd     %tp    112(%sp);
27 +
	sd     %sp    288(%a0);
28 +
	la     %t0    @trap;
29 +
	csrw   mtvec  %t0;
30 +
	csrw   mscratch %a0;
31 +
	beqz   %a1    @enterUser;
32 +
	la     %t0    @idleLoop;
33 +
	csrw   mepc   %t0;
34 +
	li     %t0    0x1880;
35 +
	csrw   mstatus %t0;
36 +
	mret;
37 +
@idleLoop
38 +
	wfi;
39 +
	j      @idleLoop;
40 +
@enterUser
41 +
	mv     %t6    %a0;
42 +
	ld     %t0    256(%t6);
43 +
	csrw   mepc   %t0;
44 +
	li     %t0    0x80;
45 +
	csrw   mstatus %t0;
46 +
	ld     %ra    8(%t6);
47 +
	ld     %sp    16(%t6);
48 +
	ld     %gp    24(%t6);
49 +
	ld     %tp    32(%t6);
50 +
	ld     %t0    40(%t6);
51 +
	ld     %t1    48(%t6);
52 +
	ld     %t2    56(%t6);
53 +
	ld     %s0    64(%t6);
54 +
	ld     %s1    72(%t6);
55 +
	ld     %a0    80(%t6);
56 +
	ld     %a1    88(%t6);
57 +
	ld     %a2    96(%t6);
58 +
	ld     %a3    104(%t6);
59 +
	ld     %a4    112(%t6);
60 +
	ld     %a5    120(%t6);
61 +
	ld     %a6    128(%t6);
62 +
	ld     %a7    136(%t6);
63 +
	ld     %s2    144(%t6);
64 +
	ld     %s3    152(%t6);
65 +
	ld     %s4    160(%t6);
66 +
	ld     %s5    168(%t6);
67 +
	ld     %s6    176(%t6);
68 +
	ld     %s7    184(%t6);
69 +
	ld     %s8    192(%t6);
70 +
	ld     %s9    200(%t6);
71 +
	ld     %s10   208(%t6);
72 +
	ld     %s11   216(%t6);
73 +
	ld     %t3    224(%t6);
74 +
	ld     %t4    232(%t6);
75 +
	ld     %t5    240(%t6);
76 +
	ld     %t6    248(%t6);
77 +
	mret;
78 +
@trap
79 +
	csrrw  %t6    mscratch %t6;
80 +
	beqz   %t6    @fatal;
81 +
	sd     %zero  0(%t6);
82 +
	sd     %ra    8(%t6);
83 +
	sd     %sp    16(%t6);
84 +
	sd     %gp    24(%t6);
85 +
	sd     %tp    32(%t6);
86 +
	sd     %t0    40(%t6);
87 +
	// Any SC clears the hart reservation; saved x0 is safe even if it succeeds.
88 +
	sc.d   %t0    %zero  0(%t6);
89 +
	sd     %t1    48(%t6);
90 +
	sd     %t2    56(%t6);
91 +
	sd     %s0    64(%t6);
92 +
	sd     %s1    72(%t6);
93 +
	sd     %a0    80(%t6);
94 +
	sd     %a1    88(%t6);
95 +
	sd     %a2    96(%t6);
96 +
	sd     %a3    104(%t6);
97 +
	sd     %a4    112(%t6);
98 +
	sd     %a5    120(%t6);
99 +
	sd     %a6    128(%t6);
100 +
	sd     %a7    136(%t6);
101 +
	sd     %s2    144(%t6);
102 +
	sd     %s3    152(%t6);
103 +
	sd     %s4    160(%t6);
104 +
	sd     %s5    168(%t6);
105 +
	sd     %s6    176(%t6);
106 +
	sd     %s7    184(%t6);
107 +
	sd     %s8    192(%t6);
108 +
	sd     %s9    200(%t6);
109 +
	sd     %s10   208(%t6);
110 +
	sd     %s11   216(%t6);
111 +
	sd     %t3    224(%t6);
112 +
	sd     %t4    232(%t6);
113 +
	sd     %t5    240(%t6);
114 +
	csrr   %t0    mscratch;
115 +
	sd     %t0    248(%t6);
116 +
	csrr   %t0    mepc;
117 +
	sd     %t0    256(%t6);
118 +
	csrr   %t0    mcause;
119 +
	sd     %t0    264(%t6);
120 +
	csrr   %t0    mtval;
121 +
	sd     %t0    272(%t6);
122 +
	csrr   %t0    mstatus;
123 +
	sd     %t0    280(%t6);
124 +
	ld     %sp    288(%t6);
125 +
	csrw   mscratch %zero;
126 +
	ld     %ra    0(%sp);
127 +
	ld     %s0    8(%sp);
128 +
	ld     %s1    16(%sp);
129 +
	ld     %s2    24(%sp);
130 +
	ld     %s3    32(%sp);
131 +
	ld     %s4    40(%sp);
132 +
	ld     %s5    48(%sp);
133 +
	ld     %s6    56(%sp);
134 +
	ld     %s7    64(%sp);
135 +
	ld     %s8    72(%sp);
136 +
	ld     %s9    80(%sp);
137 +
	ld     %s10   88(%sp);
138 +
	ld     %s11   96(%sp);
139 +
	ld     %gp    104(%sp);
140 +
	ld     %tp    112(%sp);
141 +
	addi   %sp    %sp    128;
142 +
	ret;
143 +
@fatal
144 +
	csrw   mie    %zero;
145 +
@fatalIdle
146 +
	wfi;
147 +
	j      @fatalIdle;
kernel/check/context.ras added +92 -0
1 +
// Trusted machine regression probes; these are not admitted domain images.
2 +
.text;
3 +
.export @"context::entry";
4 +
@"context::entry"
5 +
	la     %a0    @user;
6 +
	ret;
7 +
.export @"context::checkpoint";
8 +
@"context::checkpoint"
9 +
	la     %a0    @checkpoint;
10 +
	ret;
11 +
.export @"context::alarm";
12 +
@"context::alarm"
13 +
	li     %t0    0x0200bff8;
14 +
	ld     %t1    0(%t0);
15 +
	addi   %t1    %t1    200;
16 +
	li     %t0    0x02004000;
17 +
	sd     %t1    0(%t0);
18 +
	li     %t0    128;
19 +
	csrw   mie    %t0;
20 +
	ret;
21 +
.export @"context::cancel";
22 +
@"context::cancel"
23 +
	csrw   mie    %zero;
24 +
	li     %t0    0x02004000;
25 +
	li     %t1    -1;
26 +
	sd     %t1    0(%t0);
27 +
	ret;
28 +
@user
29 +
	li     %ra    1;
30 +
	li     %sp    2;
31 +
	li     %gp    3;
32 +
	li     %tp    4;
33 +
	li     %t0    5;
34 +
	li     %t1    6;
35 +
	li     %t2    7;
36 +
	li     %s0    8;
37 +
	li     %s1    9;
38 +
	li     %a0    10;
39 +
	li     %a1    11;
40 +
	li     %a2    12;
41 +
	li     %a3    13;
42 +
	li     %a4    14;
43 +
	li     %a5    15;
44 +
	li     %a6    16;
45 +
	li     %a7    17;
46 +
	li     %s2    18;
47 +
	li     %s3    19;
48 +
	li     %s4    20;
49 +
	li     %s5    21;
50 +
	li     %s6    22;
51 +
	li     %s7    23;
52 +
	li     %s8    24;
53 +
	li     %s9    25;
54 +
	li     %s10   26;
55 +
	li     %s11   27;
56 +
	li     %t3    28;
57 +
	li     %t4    29;
58 +
	li     %t5    30;
59 +
	li     %t6    31;
60 +
@checkpoint
61 +
	ecall;
62 +
	addi   %a0    %a0    100;
63 +
	ecall;
64 +
	csrr   %a0    mstatus;
65 +
	ld     %a0    1(%zero);
66 +
	li     %t0    0x40000000;
67 +
	sd     %zero  0(%t0);
68 +
	ebreak;
69 +
70 +
// Return the two user entries; Frame initialization supplies shared memory in a0.
71 +
.export @"context::reservationEntry";
72 +
@"context::reservationEntry"
73 +
	la     %a0    @reservation;
74 +
	ret;
75 +
.export @"context::reservationPeerEntry";
76 +
@"context::reservationPeerEntry"
77 +
	la     %a0    @reservationPeer;
78 +
	ret;
79 +
// Frame A observes zero, yields, then tries to overwrite frame B's store.
80 +
@reservation
81 +
	lr.d   %a1    0(%a0);
82 +
	ecall;
83 +
	li     %t0    2;
84 +
	sc.d   %a2    %t0    0(%a0);
85 +
	ecall;
86 +
// Frame B stores one and replaces A's reservation with its own before yielding.
87 +
@reservationPeer
88 +
	li     %t0    1;
89 +
	sd     %t0    0(%a0);
90 +
	lr.d   %a1    0(%a0);
91 +
	ecall;
92 +
kernel/context.rad added +87 -0
1 +
//! Machine-mode regression for real user traps and protected kernel resumption.
2 +
3 +
use core::abi;
4 +
use core::atomic;
5 +
use core::cpu;
6 +
7 +
/// Address of the fixed user register and fault probe.
8 +
fn entry() -> u64;
9 +
/// Instruction address of the first user ecall.
10 +
fn checkpoint() -> u64;
11 +
/// Arm hart zero's machine timer for the idle-path check.
12 +
unsafe fn alarm();
13 +
/// Disable the machine timer after its interrupt.
14 +
unsafe fn cancel();
15 +
/// Address of the user probe that attempts SC after another frame runs.
16 +
fn reservationEntry() -> u64;
17 +
/// Address of the user probe that replaces the hart's LR reservation.
18 +
fn reservationPeerEntry() -> u64;
19 +
20 +
/// A resumed frame must not consume another frame's reservation on shared memory.
21 +
unsafe fn reservations() {
22 +
    let mut shared: u64 = 0;
23 +
    let mut first: cpu::Frame = undefined;
24 +
    let mut second: cpu::Frame = undefined;
25 +
    cpu::init(&mut first, reservationEntry(), 0, &mut shared as u64, 0);
26 +
    cpu::init(&mut second, reservationPeerEntry(), 0, &mut shared as u64, 0);
27 +
    cpu::enter(&mut first);
28 +
    assert first.cause == 8 and (first.status & 0x1800) == 0;
29 +
    assert first.registers[11] == 0;
30 +
    // Preserve a0's shared pointer instead of installing a syscall reply.
31 +
    set first.pc += 4;
32 +
    cpu::enter(&mut second);
33 +
    assert second.cause == 8 and (second.status & 0x1800) == 0;
34 +
    assert second.registers[11] == 1 and shared == 1;
35 +
    cpu::enter(&mut first);
36 +
    assert first.cause == 8 and (first.status & 0x1800) == 0;
37 +
    assert first.registers[12] <> 0;
38 +
    assert shared == 1;
39 +
}
40 +
41 +
/// Check saved user state even when user sp, gp, tp, and ra are unusable.
42 +
@default unsafe fn main() -> u32 {
43 +
    let mut word: u32 = 0xffffffff;
44 +
    assert atomic::exchange(&mut word, 123) == 0xffffffff;
45 +
    assert atomic::exchange(&mut word, 0) == 123;
46 +
    assert atomic::load(&word) == 0;
47 +
    let mut frame: cpu::Frame = undefined;
48 +
    cpu::init(&mut frame, entry(), 0, 0, 0);
49 +
    cpu::enter(&mut frame);
50 +
    assert frame.cause == 8 and frame.pc == checkpoint();
51 +
    assert (frame.status & 0x1800) == 0;
52 +
    for i in 0..32 { assert frame.registers[i] == i as u64; }
53 +
54 +
    cpu::reply(&mut frame, abi::Error::Denied, 11, 12, 13, 14);
55 +
    cpu::enter(&mut frame);
56 +
    assert frame.cause == 8 and frame.pc == checkpoint() + 8;
57 +
    for i in 0..32 {
58 +
        if i == 10 { assert frame.registers[i] == 102; }
59 +
        else { assert frame.registers[i] == i as u64; }
60 +
    }
61 +
    cpu::reply(&mut frame, abi::Error::Ok, 21, 22, 23, 24);
62 +
    cpu::enter(&mut frame);
63 +
    assert frame.cause == 2 and frame.pc == checkpoint() + 12;
64 +
    assert frame.registers[10] == 0 and frame.registers[11] == 21;
65 +
    assert (frame.status & 0x1800) == 0;
66 +
    set frame.pc += 4;
67 +
    cpu::enter(&mut frame);
68 +
    assert frame.cause == 4 and frame.value == 1;
69 +
    set frame.pc += 4;
70 +
    cpu::enter(&mut frame);
71 +
    assert frame.cause == 7 and frame.value == 0x40000000;
72 +
    set frame.pc += 4;
73 +
    cpu::enter(&mut frame);
74 +
    assert frame.cause == 3;
75 +
    set frame.pc = 0x40000000;
76 +
    cpu::enter(&mut frame);
77 +
    assert frame.cause == 1 and frame.value == 0x40000000;
78 +
79 +
    reservations();
80 +
81 +
    alarm();
82 +
    cpu::idle(&mut frame);
83 +
    cancel();
84 +
    assert frame.cause == 0x8000000000000007;
85 +
    assert (frame.status & 0x1800) == 0x1800;
86 +
    return 0;
87 +
}
kernel/core.rad +1 -0
11 11
export mod capabilities;
12 12
export mod memory;
13 13
export mod state;
14 14
export mod pages;
15 15
export mod atomic;
16 +
export mod cpu;
kernel/core/atomic.rad +3 -0
3 3
/// Load one naturally aligned u32 with acquire ordering.
4 4
export fn load(pointer: *u32) -> u32;
5 5
6 6
/// Store one naturally aligned u32 with release ordering.
7 7
export fn store(pointer: *mut u32, value: u32);
8 +
9 +
/// Atomically exchange one naturally aligned u32 with acquire/release ordering.
10 +
export fn exchange(pointer: *mut u32, value: u32) -> u32;
kernel/core/cpu.rad added +52 -0
1 +
//! Architectural execution frames and the direct-call register convention.
2 +
3 +
use core::abi;
4 +
5 +
/// Complete integer state saved at the machine protection boundary.
6 +
export record Frame: Copy {
7 +
    /// Architectural x0..x31; x0 remains zero.
8 +
    registers: [u64; 32],
9 +
    /// Saved user instruction address.
10 +
    pc: u64,
11 +
    /// Raw machine trap cause, including its interrupt bit.
12 +
    cause: u64,
13 +
    /// Machine trap value associated with a fault.
14 +
    value: u64,
15 +
    /// Trap-time machine status, used to distinguish privilege levels.
16 +
    status: u64,
17 +
    /// Suspended machine stack, written only by arch/context.ras.
18 +
    kernelStack: u64,
19 +
}
20 +
21 +
/// Clear registers and install a checked image entry and startup environment.
22 +
export fn init(frame: *mut Frame, entry: u64, stack: u64, environment: u64, state: u64) {
23 +
    set frame.registers = [0; 32];
24 +
    set frame.registers[2] = stack;
25 +
    set frame.registers[3] = state;
26 +
    set frame.registers[4] = environment;
27 +
    set frame.registers[10] = environment;
28 +
    set frame.pc = entry;
29 +
    set frame.cause = 0;
30 +
    set frame.value = 0;
31 +
    set frame.status = 0;
32 +
    set frame.kernelStack = 0;
33 +
}
34 +
35 +
/// Run checked U-mode state until its next trap, with machine interrupts disabled on return.
36 +
/// The caller owns the frame and has installed all executable and memory authority.
37 +
export unsafe fn enter(frame: *mut Frame);
38 +
39 +
/// Sleep in M-mode until an enabled interrupt, using a private idle frame.
40 +
/// The caller must configure a bounded interrupt source before entering idle.
41 +
export unsafe fn idle(frame: *mut Frame);
42 +
43 +
/// Return Error in a0 and up to four value words in a1..a4; advance past ecall.
44 +
export fn reply(frame: *mut Frame, error: abi::Error, first: u64, second: u64, third: u64, fourth: u64) {
45 +
    assert frame.cause == 8;
46 +
    set frame.pc += 4;
47 +
    set frame.registers[10] = error as u64;
48 +
    set frame.registers[11] = first;
49 +
    set frame.registers[12] = second;
50 +
    set frame.registers[13] = third;
51 +
    set frame.registers[14] = fourth;
52 +
}
kernel/core/domains.rad +8 -2
26 26
    eventsHandle: u64,
27 27
    /// Physical address of this domain's shared event ring.
28 28
    eventsPointer: u64,
29 29
    /// Physical base of this image instance's mutable data.
30 30
    stateBase: u64,
31 -
    /// Lowest address available to this context's stack frames.
31 +
    /// Lowest address available to Page-backed local stack frames.
32 32
    stackBase: u64,
33 -
    /// Exclusive upper bound of this context's stack.
33 +
    /// Exclusive upper bound of the addressable local stack.
34 34
    stackTop: u64,
35 35
    /// Kernel execution-context identity.
36 36
    context: u64,
37 +
    /// Lowest private address for SSA values and saved registers.
38 +
    privateStackBase: u64,
39 +
    /// Exclusive upper bound of the never-granted private stack.
40 +
    privateStackTop: u64,
37 41
}
38 42
39 43
/// One protection domain and its private authority boundary.
40 44
export record Domain: Copy {
41 45
    /// Domain-relative capability names.
74 78
        set domain.parent = none();
75 79
        set domain.image = 0;
76 80
        set domain.env = Env {
77 81
            argsPointer: 0, argsSize: 0, eventsHandle: 0, eventsPointer: 0,
78 82
            stateBase: 0, stackBase: 0, stackTop: 0, context: 0,
83 +
            privateStackBase: 0, privateStackTop: 0,
79 84
        };
80 85
    }
81 86
}
82 87
83 88
/// Check that an object identifies a live, non-dying domain incarnation.
129 134
    let eventObject = abi::Object { kind: abi::Kind::Events, index, epoch: domain.epoch };
130 135
    let eventHandle = handles::install(&mut domain.handles, 1, eventObject, abi::READ | abi::WRITE);
131 136
    set domain.env = Env {
132 137
        argsPointer: 0, argsSize: 0, eventsHandle: eventHandle.bits, eventsPointer: 0,
133 138
        stateBase: 0, stackBase: 0, stackTop: 0, context: 0,
139 +
        privateStackBase: 0, privateStackTop: 0,
134 140
    };
135 141
    set domain.state = Lifecycle::Pending;
136 142
    return object;
137 143
}
138 144
lib/std/arch/rv64/asm.rad +14 -1
142 142
    WordShift { enc: fn(gen::Reg, gen::Reg, i32) -> u32 },
143 143
    /// Load instruction with memory operand syntax.
144 144
    Load { enc: fn(gen::Reg, gen::Reg, i32) -> u32 },
145 145
    /// Store instruction with memory operand syntax.
146 146
    Store { enc: fn(gen::Reg, gen::Reg, i32) -> u32 },
147 +
    /// RV64A base mnemonic followed by .w/.d and optional .aq/.rl/.aqrl.
148 +
    Atomic { funct5: u32 },
147 149
    /// Two-register branch instruction.
148 150
    Branch { op: BranchOp },
149 151
    /// One-register branch-to-zero pseudo-instruction.
150 152
    BranchZero { op: BranchOp },
151 153
    /// `jal` instruction with explicit destination register.
260 262
export constant SHIFT_LIMIT: i32 = 64;
261 263
/// Largest `lui` or `auipc` immediate.
262 264
export constant UPPER_IMM_MAX_VALUE: i64 = 0xFFFFF;
263 265
264 266
/// Sorted instruction descriptor table used by the assembler parser.
265 -
export constant INSTRUCTIONS: [InstructionEntry; 88] = [
267 +
export constant INSTRUCTIONS: [InstructionEntry; 99] = [
266 268
    { name: "add",    encoder: InstructionEncoder::RRR { enc: encode::add } },
267 269
    { name: "addi",   encoder: InstructionEncoder::RRI { enc: encode::addi } },
268 270
    { name: "addiw",  encoder: InstructionEncoder::RRI { enc: encode::addiw } },
269 271
    { name: "addw",   encoder: InstructionEncoder::RRR { enc: encode::addw } },
272 +
    { name: "amoadd",  encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOADD } },
273 +
    { name: "amoand",  encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOAND } },
274 +
    { name: "amomax",  encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOMAX } },
275 +
    { name: "amomaxu", encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOMAXU } },
276 +
    { name: "amomin",  encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOMIN } },
277 +
    { name: "amominu", encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOMINU } },
278 +
    { name: "amoor",   encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOOR } },
279 +
    { name: "amoswap", encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOSWAP } },
280 +
    { name: "amoxor",  encoder: InstructionEncoder::Atomic { funct5: encode::F5_AMOXOR } },
270 281
    { name: "and",    encoder: InstructionEncoder::RRR { enc: encode::and_ } },
271 282
    { name: "andi",   encoder: InstructionEncoder::RRI { enc: encode::andi } },
272 283
    { name: "auipc",  encoder: InstructionEncoder::Upper { enc: encode::auipc } },
273 284
    { name: "beq",    encoder: InstructionEncoder::Branch { op: BranchOp::Beq } },
274 285
    { name: "beqz",   encoder: InstructionEncoder::BranchZero { op: BranchOp::Beq } },
301 312
    { name: "lbu",    encoder: InstructionEncoder::Load { enc: encode::lbu } },
302 313
    { name: "ld",     encoder: InstructionEncoder::Load { enc: encode::ld } },
303 314
    { name: "lh",     encoder: InstructionEncoder::Load { enc: encode::lh } },
304 315
    { name: "lhu",    encoder: InstructionEncoder::Load { enc: encode::lhu } },
305 316
    { name: "li",     encoder: InstructionEncoder::Li },
317 +
    { name: "lr",     encoder: InstructionEncoder::Atomic { funct5: encode::F5_LR } },
306 318
    { name: "lui",    encoder: InstructionEncoder::Upper { enc: encode::lui } },
307 319
    { name: "lw",     encoder: InstructionEncoder::Load { enc: encode::lw } },
308 320
    { name: "lwu",    encoder: InstructionEncoder::Load { enc: encode::lwu } },
309 321
    { name: "mret",   encoder: InstructionEncoder::NoOperand { enc: encode::mret } },
310 322
    { name: "mul",    encoder: InstructionEncoder::RRR { enc: encode::mul } },
322 334
    { name: "remu",   encoder: InstructionEncoder::RRR { enc: encode::remu } },
323 335
    { name: "remuw",  encoder: InstructionEncoder::RRR { enc: encode::remuw } },
324 336
    { name: "remw",   encoder: InstructionEncoder::RRR { enc: encode::remw } },
325 337
    { name: "ret",    encoder: InstructionEncoder::NoOperand { enc: encode::ret } },
326 338
    { name: "sb",     encoder: InstructionEncoder::Store { enc: encode::sb } },
339 +
    { name: "sc",     encoder: InstructionEncoder::Atomic { funct5: encode::F5_SC } },
327 340
    { name: "sd",     encoder: InstructionEncoder::Store { enc: encode::sd } },
328 341
    { name: "seqz",   encoder: InstructionEncoder::RR { enc: encode::seqz } },
329 342
    { name: "sh",     encoder: InstructionEncoder::Store { enc: encode::sh } },
330 343
    { name: "sll",    encoder: InstructionEncoder::RRR { enc: encode::sll } },
331 344
    { name: "slli",   encoder: InstructionEncoder::Shift { enc: encode::slli } },
lib/std/arch/rv64/asm/parser.rad +44 -0
303 303
            return try parseShift(a, enc, super::SHIFT_LIMIT, "shift amount out of range"),
304 304
        case super::InstructionEncoder::WordShift { enc } =>
305 305
            return try parseShift(a, enc, super::WORD_SHIFT_LIMIT, "word shift amount out of range"),
306 306
        case super::InstructionEncoder::Load { enc } => return try parseLoad(a, enc),
307 307
        case super::InstructionEncoder::Store { enc } => return try parseStore(a, enc),
308 +
        case super::InstructionEncoder::Atomic { funct5 } => return try parseAtomic(a, tok, funct5),
308 309
        case super::InstructionEncoder::Branch { op } => return try parseBranch(a, op),
309 310
        case super::InstructionEncoder::BranchZero { op } => return try parseBranchZero(a, op),
310 311
        case super::InstructionEncoder::Jal => return try parseJal(a),
311 312
        case super::InstructionEncoder::Jump { rd } => return try parseJ(a, rd),
312 313
        case super::InstructionEncoder::RdCsr { enc } => return try parseRdCsr(a, enc),
438 439
    let memop = try parseMemory(a);
439 440
440 441
    try emit::emitText(a, enc(rs2, memop.base, memop.offset));
441 442
}
442 443
444 +
/// Parse the adjacent width/order suffixes and zero-offset RV64A operands.
445 +
fn parseAtomic(a: *mut super::Assembler, tok: scanner::Token, funct5: u32) throws (super::Error) {
446 +
    let widthToken = try expectToken(a, scanner::TokenKind::Directive, "expected atomic width suffix");
447 +
    if widthToken.offset <> tok.offset + tok.source.len {
448 +
        throw failOnToken(widthToken, "atomic width must adjoin mnemonic");
449 +
    }
450 +
    let mut width: u32 = encode::F3_WORD;
451 +
    if mem::eq(widthToken.source, ".d") {
452 +
        set width = encode::F3_DWORD;
453 +
    } else if not mem::eq(widthToken.source, ".w") {
454 +
        throw failOnToken(widthToken, "expected `.w` or `.d` atomic width");
455 +
    }
456 +
    let mut ordering: u32 = 0;
457 +
    if a.scan.current.kind == scanner::TokenKind::Directive {
458 +
        let orderToken = a.scan.current;
459 +
        if orderToken.offset <> widthToken.offset + widthToken.source.len {
460 +
            throw failOnToken(orderToken, "atomic ordering must adjoin width");
461 +
        }
462 +
        if mem::eq(orderToken.source, ".aq") {
463 +
            set ordering = 2;
464 +
        } else if mem::eq(orderToken.source, ".rl") {
465 +
            set ordering = 1;
466 +
        } else if mem::eq(orderToken.source, ".aqrl") {
467 +
            set ordering = 3;
468 +
        } else {
469 +
            throw failOnToken(orderToken, "expected `.aq`, `.rl`, or `.aqrl` atomic ordering");
470 +
        }
471 +
        advance(a);
472 +
    }
473 +
    let rd = try parseRegister(a);
474 +
    // LR reserves rs2 = x0 and has no source register operand.
475 +
    let mut rs2 = rv64::ZERO;
476 +
    if funct5 <> encode::F5_LR {
477 +
        set rs2 = try parseRegister(a);
478 +
    }
479 +
    let memop = try parseMemory(a);
480 +
    if memop.offset <> 0 {
481 +
        throw fail(a, "atomic memory offset must be zero");
482 +
    }
483 +
    try expectTerminator(a, "unexpected atomic operand");
484 +
    try emit::emitText(a, encode::atomic(rd, memop.base, rs2, funct5, width, ordering));
485 +
}
486 +
443 487
/// Parse a two-register branch instruction.
444 488
fn parseBranch(a: *mut super::Assembler, op: super::BranchOp) throws (super::Error) {
445 489
    let rs1 = try parseRegister(a);
446 490
    let rs2 = try parseRegister(a);
447 491
lib/std/arch/rv64/asm/tests.rad +90 -0
227 227
    try expectAssembleFail(
228 228
        ".text;\ncsrsi mstatus 32;\n"
229 229
    );
230 230
}
231 231
232 +
@test fn testAssembleAtomicOperationsAndWidths() throws (testing::TestError) {
233 +
    let sources: [*[u8]; 22] = [
234 +
        "lr.w %t0 0(%t2);",
235 +
        "lr.d %t0 0(%t2);",
236 +
        "sc.w %t0 %t1 0(%t2);",
237 +
        "sc.d %t0 %t1 0(%t2);",
238 +
        "amoswap.w %t0 %t1 0(%t2);",
239 +
        "amoswap.d %t0 %t1 0(%t2);",
240 +
        "amoadd.w %t0 %t1 0(%t2);",
241 +
        "amoadd.d %t0 %t1 0(%t2);",
242 +
        "amoxor.w %t0 %t1 0(%t2);",
243 +
        "amoxor.d %t0 %t1 0(%t2);",
244 +
        "amoand.w %t0 %t1 0(%t2);",
245 +
        "amoand.d %t0 %t1 0(%t2);",
246 +
        "amoor.w %t0 %t1 0(%t2);",
247 +
        "amoor.d %t0 %t1 0(%t2);",
248 +
        "amomin.w %t0 %t1 0(%t2);",
249 +
        "amomin.d %t0 %t1 0(%t2);",
250 +
        "amomax.w %t0 %t1 0(%t2);",
251 +
        "amomax.d %t0 %t1 0(%t2);",
252 +
        "amominu.w %t0 %t1 0(%t2);",
253 +
        "amominu.d %t0 %t1 0(%t2);",
254 +
        "amomaxu.w %t0 %t1 0(%t2);",
255 +
        "amomaxu.d %t0 %t1 0(%t2);",
256 +
    ];
257 +
    // Fixed ISA words keep encoder bugs visible instead of sharing its formula.
258 +
    let expected: [u32; 22] = [
259 +
        0x1003a2af, 0x1003b2af,
260 +
        0x1863a2af, 0x1863b2af,
261 +
        0x0863a2af, 0x0863b2af,
262 +
        0x0063a2af, 0x0063b2af,
263 +
        0x2063a2af, 0x2063b2af,
264 +
        0x6063a2af, 0x6063b2af,
265 +
        0x4063a2af, 0x4063b2af,
266 +
        0x8063a2af, 0x8063b2af,
267 +
        0xa063a2af, 0xa063b2af,
268 +
        0xc063a2af, 0xc063b2af,
269 +
        0xe063a2af, 0xe063b2af,
270 +
    ];
271 +
    for i in 0..sources.len {
272 +
        let program = try assembleSource(sources[i]);
273 +
        try testing::expect(program.text.len == 1);
274 +
        try testing::expect(program.text[0] == expected[i]);
275 +
    }
276 +
}
277 +
278 +
@test fn testAssembleAtomicOrderingAndRegisterFields() throws (testing::TestError) {
279 +
    let sources: [*[u8]; 5] = [
280 +
        "amoswap.w.aq %t0 %t1 0(%t2);",
281 +
        "amoswap.d.rl %t0 %t1 0(%t2);",
282 +
        "amoswap.d.aqrl %t0 %t1 0(%t2);",
283 +
        "lr.w.aqrl %t6 (%ra);",
284 +
        "sc.d.aq %zero %t6 (%t5);",
285 +
    ];
286 +
    let expected: [u32; 5] = [
287 +
        0x0c63a2af, 0x0a63b2af, 0x0e63b2af, 0x1600afaf, 0x1dff302f,
288 +
    ];
289 +
    for i in 0..sources.len {
290 +
        let program = try assembleSource(sources[i]);
291 +
        try testing::expect(program.text.len == 1);
292 +
        try testing::expect(program.text[0] == expected[i]);
293 +
    }
294 +
}
295 +
296 +
@test fn testAssembleInvalidAtomicOperandsFail() throws (testing::TestError) {
297 +
    try expectAssembleFail("lr.w %t0 %t1 0(%t2);");
298 +
    try expectAssembleFail("lr.d %t0;");
299 +
    try expectAssembleFail("sc.w %t0 0(%t2);");
300 +
    try expectAssembleFail("sc.d %t0 %t1 0(%t2) %t3;");
301 +
    try expectAssembleFail("amoadd.d %t0 %t1;");
302 +
    try expectAssembleFail("amoswap.w %t0 %t1 %t2;");
303 +
    try expectAssembleFail("lr.w %t0 1(%t2);");
304 +
    try expectAssembleFail("sc.d %t0 %t1 -1(%t2);");
305 +
    try expectAssembleFail("amoadd.w %t0 %t1 4294967296(%t2);");
306 +
}
307 +
308 +
@test fn testAssembleInvalidAtomicSuffixesFail() throws (testing::TestError) {
309 +
    try expectAssembleFail("lr %t0 0(%t2);");
310 +
    try expectAssembleFail("lr.b %t0 0(%t2);");
311 +
    try expectAssembleFail("sc.q %t0 %t1 0(%t2);");
312 +
    try expectAssembleFail("amoadd.wu %t0 %t1 0(%t2);");
313 +
    try expectAssembleFail("amoswap.aq.w %t0 %t1 0(%t2);");
314 +
    try expectAssembleFail("amoswap.w.rlaq %t0 %t1 0(%t2);");
315 +
    try expectAssembleFail("amoswap.w.aq.rl %t0 %t1 0(%t2);");
316 +
    try expectAssembleFail("amoswap.w.aq.aq %t0 %t1 0(%t2);");
317 +
    try expectAssembleFail("amoswap .w %t0 %t1 0(%t2);");
318 +
    try expectAssembleFail("amoswap.w .aq %t0 %t1 0(%t2);");
319 +
    try expectAssembleFail("add.w.aq %t0 %t1 %t2;");
320 +
}
321 +
232 322
@test fn testPrintInstrUsesPercentPrefixedRegisters() throws (testing::TestError) {
233 323
    let text = printInstrText(encode::addi(rv64::A0, rv64::SP, 42));
234 324
    try testing::expect(mem::eq(text, "addi    %a0, %sp, 42"));
235 325
}
lib/std/arch/rv64/encode.rad +42 -1
1 -
//! RISC-V RV64I+M instruction encoding.
1 +
//! RISC-V RV64I+M+A instruction encoding.
2 2
//!
3 3
//! Provides type-safe functions for encoding RV64 instructions.
4 4
5 5
use std::lang::gen;
6 6
8 8
// Opcode Constants //
9 9
//////////////////////
10 10
11 11
export constant OP_LOAD:   u32 = 0x03;
12 12
export constant OP_STORE:  u32 = 0x23;
13 +
/// RV64A LR, SC, and atomic read-modify-write opcode.
14 +
export constant OP_AMO:    u32 = 0x2F;
13 15
export constant OP_BRANCH: u32 = 0x63;
14 16
export constant OP_JALR:   u32 = 0x67;
15 17
export constant OP_JAL:    u32 = 0x6F;
16 18
export constant OP_OP:     u32 = 0x33;
17 19
export constant OP_IMM:    u32 = 0x13;
71 73
export constant F7_NORMAL: u32 = 0b0000000;
72 74
export constant F7_SUB:    u32 = 0b0100000;  // Bit 5 set
73 75
export constant F7_SRA:    u32 = 0b0100000;  // Bit 5 set
74 76
export constant F7_MUL:    u32 = 0b0000001;  // Bit 0 set
75 77
78 +
//////////////////////
79 +
// Funct5 Constants //
80 +
//////////////////////
81 +
82 +
/// Atomic addition function field.
83 +
export constant F5_AMOADD:  u32 = 0b00000;
84 +
/// Atomic swap function field.
85 +
export constant F5_AMOSWAP: u32 = 0b00001;
86 +
/// Load-reserved function field; the source register field must be zero.
87 +
export constant F5_LR:      u32 = 0b00010;
88 +
/// Store-conditional function field.
89 +
export constant F5_SC:      u32 = 0b00011;
90 +
/// Atomic XOR function field.
91 +
export constant F5_AMOXOR:  u32 = 0b00100;
92 +
/// Atomic OR function field.
93 +
export constant F5_AMOOR:   u32 = 0b01000;
94 +
/// Atomic AND function field.
95 +
export constant F5_AMOAND:  u32 = 0b01100;
96 +
/// Atomic signed minimum function field.
97 +
export constant F5_AMOMIN:  u32 = 0b10000;
98 +
/// Atomic signed maximum function field.
99 +
export constant F5_AMOMAX:  u32 = 0b10100;
100 +
/// Atomic unsigned minimum function field.
101 +
export constant F5_AMOMINU: u32 = 0b11000;
102 +
/// Atomic unsigned maximum function field.
103 +
export constant F5_AMOMAXU: u32 = 0b11100;
104 +
76 105
/////////////////////////
77 106
// Validation Helpers  //
78 107
/////////////////////////
79 108
80 109
/// Returns `true` if the value fits in a signed 12-bit immediate.
177 206
         | (imm11        << 20)
178 207
         | (imm10_1      << 21)
179 208
         | (imm20        << 31);
180 209
}
181 210
211 +
/// Encode an RV64A operation with a standard F5_* function and zero byte offset.
212 +
/// Width is F3_WORD or F3_DWORD; ordering holds aq in bit 1 and rl in bit 0.
213 +
/// LR requires rs2 = x0; rs1 is always the address register.
214 +
export fn atomic(rd: gen::Reg, rs1: gen::Reg, rs2: gen::Reg, funct5: u32, width: u32, ordering: u32) -> u32 {
215 +
    assert funct5 < 32;
216 +
    assert width == F3_WORD or width == F3_DWORD;
217 +
    assert ordering < 4;
218 +
    assert funct5 <> F5_LR or *rs2 == 0;
219 +
220 +
    return encodeR(OP_AMO, rd, rs1, rs2, width, (funct5 << 2) | ordering);
221 +
}
222 +
182 223
////////////////////////////
183 224
// ALU Immediate (I-type) //
184 225
////////////////////////////
185 226
186 227
/// Add immediate: `rd = rs1 + imm`.