export mod trap;

use super::trap;

/// Trap anchors owned by their physical harts.

unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Stop on a trap before a domain or interrupt dispatcher is installed.

unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {

    print("kernel: unexpected trap\n");

    trap::halt();

}

    let stack = PLATFORM.stacks[hart as u32];

    set HARTS[hart as u32] = trap::Hart {

        stackTop: stack.end, stackBottom: stack.start, kernelGp: 0, handler: unexpected,

        savedT0: 0, savedT1: 0, savedSp: 0,

    };

    trap::install(&mut HARTS[hart as u32]);

export mod trap;

//! Trap classification, frame layout, and resume-address boundaries.

use std::testing;

use kernel::trap;

/// The assembly frame includes x0-x31 and all four trap CSRs.

@test fn layout() throws (testing::TestError) {

    try testing::expect(@sizeOf(trap::Frame) == 288);

    try testing::expect(@sizeOf(trap::Hart) == 56);

    try testing::expect(trap::FRAME_SIZE % 16 == 0);

}

/// Interrupts retain their source identity; exceptions retain their fault class.

@test fn classification() throws (testing::TestError) {

    try testing::expect(trap::classify(0x8000000000000003) == trap::Cause::Software);

    try testing::expect(trap::classify(0x8000000000000007) == trap::Cause::Timer);

    try testing::expect(trap::classify(0x800000000000000b) == trap::Cause::External);

    try testing::expect(trap::classify(8) == trap::Cause::UserCall);

    try testing::expect(trap::classify(11) == trap::Cause::MachineCall);

    try testing::expect(trap::classify(2) == trap::Cause::IllegalInstruction);

    try testing::expect(trap::classify(3) == trap::Cause::Breakpoint);

    for code in &[0 as u64, 1, 4, 5, 6, 7, 12, 13, 15] {

        try testing::expect(trap::classify(code) == trap::Cause::MemoryFault);

    }

    try testing::expect(trap::classify(0x8000000000000008) == trap::Cause::Unknown);

    try testing::expect(trap::classify(9) == trap::Cause::Unknown);

}

/// Only an aligned ecall resume address can advance by one RV64 instruction.

@test unsafe fn resumeAddress() throws (testing::TestError) {

    let mut frame: trap::Frame = undefined;

    set frame.pc = 0x80010000; set frame.cause = 8; set frame.status = 0;

    try testing::expect(trap::fromUser(&frame));

    try testing::expect(trap::advanceCall(&mut frame) and frame.pc == 0x80010004);

    set frame.status = 0x1800;

    try testing::expect(not trap::fromUser(&frame));

    set frame.cause = 0x8000000000000007;

    try testing::expect(not trap::advanceCall(&mut frame) and frame.pc == 0x80010004);

    set frame.cause = 11; set frame.pc = 0xfffffffffffffffc;

    try testing::expect(not trap::advanceCall(&mut frame) and frame.pc == 0xfffffffffffffffc);

    set frame.pc = 2;

    try testing::expect(not trap::advanceCall(&mut frame) and frame.pc == 2);

}

//! Integer trap contexts and exception classification.

/// Frame bytes reserved by trap entry. This is a multiple of stack alignment.

export constant FRAME_SIZE: u32 = 288;

/// Saved architectural state. Assembly offsets follow field declaration order.

export record Frame: Copy {

    /// Integer registers indexed by architectural register number. x0 is zero.

    registers: [u64; 32],

    /// Instruction address captured from mepc.

    pc: u64,

    /// Trap-entry mstatus, including previous privilege and interrupt state.

    status: u64,

    /// Interrupt bit and exception code captured from mcause.

    cause: u64,

    /// Fault address or instruction bits captured from mtval.

    value: u64,

}

/// Per-hart trap entry state named by mscratch. Entry scratch fields are private

/// to the assembly prologue while machine interrupts are disabled.

export record Hart: Copy {

    /// Exclusive upper bound of the kernel stack.

    stackTop: u64,

    /// First valid kernel stack byte.

    stackBottom: u64,

    /// Package-state base used by kernel code.

    kernelGp: u64,

    /// Handler called with a complete frame and this hart's state.

    handler: unsafe fn(&mut Frame, &mut Hart),

    /// Temporary preservation of x5 during entry.

    savedT0: u64,

    /// Temporary preservation of x6 during entry.

    savedT1: u64,

    /// Interrupted stack pointer, before stack selection.

    savedSp: u64,

}

/// Trap classes consumed by kernel dispatch.

export union Cause: Copy {

    /// Machine software interrupt.

    Software,

    /// Machine timer interrupt.

    Timer,

    /// Machine external interrupt.

    External,

    /// Environment call from user mode.

    UserCall,

    /// Environment call from machine mode.

    MachineCall,

    /// Unsupported or privileged instruction.

    IllegalInstruction,

    /// Breakpoint instruction.

    Breakpoint,

    /// Instruction, load, or store address/access fault.

    MemoryFault,

    /// Unhandled architectural cause.

    Unknown,

}

/// Classify mcause without discarding its interrupt bit.

export fn classify(cause: u64) -> Cause {

    match cause {

        case 0x8000000000000003 => return Cause::Software,

        case 0x8000000000000007 => return Cause::Timer,

        case 0x800000000000000b => return Cause::External,

        case 8 => return Cause::UserCall,

        case 11 => return Cause::MachineCall,

        case 2 => return Cause::IllegalInstruction,

        case 3 => return Cause::Breakpoint,

        case 0, 1, 4, 5, 6, 7, 12, 13, 15 => return Cause::MemoryFault,

        else => return Cause::Unknown,

    }

}

/// Check the privilege captured in the frame's MPP field.

export fn fromUser(frame: &Frame) -> bool { return (frame.status & 0x1800) == 0; }

/// Advance a completed ecall. Leave interrupts, faults, and invalid PCs unchanged.

export fn advanceCall(frame: &mut Frame) -> bool {

    if frame.cause <> 8 and frame.cause <> 11 {

        return false;

    }

    if (frame.pc & 3) <> 0 or frame.pc > 0xffffffffffffffff - 4 {

        return false;

    }

    set frame.pc += 4;

    return true;

}

/// Install trap entry for the current hart with machine interrupts disabled.

export fn install(hart: *mut Hart);

/// Restore a complete integer context through mret. The current mscratch must

/// name an installed Hart, and the frame must have machine interrupts disabled.

export fn resume(frame: *Frame) -> !;

/// Disable machine interrupts and stop the current hart.

export fn halt() -> !;

//! RV64 integer trap entry. mscratch names the current hart's kernel state.

.text;

.export @kernel::trap::install;

.export @kernel::trap::resume;

.export @kernel::trap::vector;

.export @kernel::trap::halt;

// Install a direct trap vector and its per-hart stack anchor.

@kernel::trap::install

    csrw mie %zero;

    csrr %t0 mstatus;

    andi %t0 %t0 -9;

    csrw mstatus %t0;

    csrw mscratch %a0;

    la %t0 @kernel::trap::vector;

    csrw mtvec %t0;

    ret;

// Hardware has cleared MIE. Preserve entry scratch registers before stack selection.

@kernel::trap::vector

    csrrw %tp mscratch %tp;

    sd %t0 32(%tp);

    sd %t1 40(%tp);

    sd %sp 48(%tp);

    csrr %t0 mstatus;

    srli %t0 %t0 11;

    andi %t0 %t0 3;

    li %t1 3;

    beq %t0 %t1 @machineStack;

    ld %sp 0(%tp);

@machineStack

    andi %t0 %sp 15;

    bnez %t0 @stackFault;

    ld %t0 0(%tp);

    bltu %t0 %sp @stackFault;

    addi %t1 %sp -288;

    bltu %sp %t1 @stackFault;

    ld %t0 8(%tp);

    bltu %t1 %t0 @stackFault;

    mv %sp %t1;

    sd %zero 0(%sp);

    // SC clears the hart reservation. The saved x0 slot accepts a zero store.

    sc.d %zero %zero (%sp);

    sd %ra 8(%sp);

    sd %gp 24(%sp);

    sd %t2 56(%sp);

    sd %s0 64(%sp);

    sd %s1 72(%sp);

    sd %a0 80(%sp);

    sd %a1 88(%sp);

    sd %a2 96(%sp);

    sd %a3 104(%sp);

    sd %a4 112(%sp);

    sd %a5 120(%sp);

    sd %a6 128(%sp);

    sd %a7 136(%sp);

    sd %s2 144(%sp);

    sd %s3 152(%sp);

    sd %s4 160(%sp);

    sd %s5 168(%sp);

    sd %s6 176(%sp);

    sd %s7 184(%sp);

    sd %s8 192(%sp);

    sd %s9 200(%sp);

    sd %s10 208(%sp);

    sd %s11 216(%sp);

    sd %t3 224(%sp);

    sd %t4 232(%sp);

    sd %t5 240(%sp);

    sd %t6 248(%sp);

    ld %t0 48(%tp);

    sd %t0 16(%sp);

    csrr %t0 mscratch;

    sd %t0 32(%sp);

    ld %t0 32(%tp);

    sd %t0 40(%sp);

    ld %t0 40(%tp);

    sd %t0 48(%sp);

    csrr %t0 mepc;

    sd %t0 256(%sp);

    csrr %t0 mstatus;

    sd %t0 264(%sp);

    csrr %t0 mcause;

    sd %t0 272(%sp);

    csrr %t0 mtval;

    sd %t0 280(%sp);

    csrw mscratch %tp;

    ld %gp 16(%tp);

    mv %a0 %sp;

    mv %a1 %tp;

    ld %t0 24(%tp);

    jalr %ra %t0 0;

    mv %a0 %sp;

// Interrupts remain disabled until mret restores the saved MPIE state.

@kernel::trap::resume

    csrr %t0 mstatus;

    andi %t0 %t0 -9;

    csrw mstatus %t0;

    mv %sp %a0;

    // A resumed frame must not use a reservation from another execution context.

    sc.d %zero %zero (%sp);

    ld %t0 256(%sp);

    csrw mepc %t0;

    ld %t0 264(%sp);

    andi %t0 %t0 -9;

    csrw mstatus %t0;

    ld %ra 8(%sp);

    ld %gp 24(%sp);

    ld %tp 32(%sp);

    ld %t0 40(%sp);

    ld %t1 48(%sp);

    ld %t2 56(%sp);

    ld %s0 64(%sp);

    ld %s1 72(%sp);

    ld %a0 80(%sp);

    ld %a1 88(%sp);

    ld %a2 96(%sp);

    ld %a3 104(%sp);

    ld %a4 112(%sp);

    ld %a5 120(%sp);

    ld %a6 128(%sp);

    ld %a7 136(%sp);

    ld %s2 144(%sp);

    ld %s3 152(%sp);

    ld %s4 160(%sp);

    ld %s5 168(%sp);

    ld %s6 176(%sp);

    ld %s7 184(%sp);

    ld %s8 192(%sp);

    ld %s9 200(%sp);

    ld %s10 208(%sp);

    ld %s11 216(%sp);

    ld %t3 224(%sp);

    ld %t4 232(%sp);

    ld %t5 240(%sp);

    ld %t6 248(%sp);

    ld %sp 16(%sp);

    mret;

// A frame must never extend outside the reserved kernel stack.

@stackFault

@kernel::trap::halt

    csrw mie %zero;

    wfi;

    j @stackFault;

cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras > "$work/boot.ras"

static MEMORY: [u8; 16777216] = [0; 16777216];

//! Machine and user trap entry with a nested machine timer interrupt.

.text;

    li %s0 1073807360;

    slli %s0 %s0 1;

    sd %sp 0(%s0);

    li %t0 4096;

    sub %t0 %sp %t0;

    sd %t0 8(%s0);

    li %t0 0x12345678;

    sd %t0 16(%s0);

    la %t0 @handler;

    sd %t0 24(%s0);

    mv %a0 %s0;

    call @kernel::trap::install;

    li %ra 17;

    li %gp 51;

    li %tp 68;

    li %t0 85;

    li %t1 102;

    li %t2 119;

    li %s1 153;

    li %a0 170;

    li %a1 187;

    li %a2 204;

    li %a3 221;

    li %a4 238;

    li %a5 255;

    li %a6 272;

    li %a7 289;

    li %s2 306;

    li %s3 323;

    li %s4 340;

    li %s5 357;

    li %s6 374;

    li %s7 391;

    li %s8 408;

    li %s9 425;

    li %s10 442;

    li %s11 459;

    li %t3 476;

    li %t4 493;

    li %t5 510;

    li %t6 527;

    ecall;

    addi %t0 %t0 -85;

    bnez %t0 @fail;

    li %t0 17;

    bne %ra %t0 @fail;

    li %t0 51;

    bne %gp %t0 @fail;

    li %t0 68;

    bne %tp %t0 @fail;

    li %t0 102;

    bne %t1 %t0 @fail;

    li %t0 119;

    bne %t2 %t0 @fail;

    li %t0 153;

    bne %s1 %t0 @fail;

    li %t0 170;

    bne %a0 %t0 @fail;

    li %t0 187;

    bne %a1 %t0 @fail;

    li %t0 204;

    bne %a2 %t0 @fail;

    li %t0 221;

    bne %a3 %t0 @fail;

    li %t0 238;

    bne %a4 %t0 @fail;

    li %t0 255;

    bne %a5 %t0 @fail;

    li %t0 272;

    bne %a6 %t0 @fail;

    li %t0 289;

    bne %a7 %t0 @fail;

    li %t0 306;

    bne %s2 %t0 @fail;

    li %t0 323;

    bne %s3 %t0 @fail;

    li %t0 340;

    bne %s4 %t0 @fail;

    li %t0 357;

    bne %s5 %t0 @fail;

    li %t0 374;

    bne %s6 %t0 @fail;

    li %t0 391;

    bne %s7 %t0 @fail;

    li %t0 408;

    bne %s8 %t0 @fail;

    li %t0 425;

    bne %s9 %t0 @fail;

    li %t0 442;

    bne %s10 %t0 @fail;

    li %t0 459;

    bne %s11 %t0 @fail;

    li %t0 476;

    bne %t3 %t0 @fail;

    li %t0 493;

    bne %t4 %t0 @fail;

    li %t0 510;

    bne %t5 %t0 @fail;

    li %t0 527;

    bne %t6 %t0 @fail;

    li %t0 1073807360;

    slli %t0 %t0 1;

    bne %s0 %t0 @fail;

    ld %t0 0(%s0);

    bne %sp %t0 @fail;

    ld %t0 64(%s0);

    li %t1 2;

    bne %t0 %t1 @fail;

    la %t0 @user;

    csrw mepc %t0;

    csrw mstatus %zero;

    li %sp 1073809400;

    slli %sp %sp 1;

    mret;

@user

    li %ra 17;

    li %gp 51;

    li %tp 68;

    li %t0 85;

    li %t1 102;

    li %t2 119;

    li %s1 153;

    li %a0 170;

    li %a1 187;

    li %a2 204;

    li %a3 221;

    li %a4 238;

    li %a5 255;

    li %a6 272;

    li %a7 289;

    li %s2 306;

    li %s3 323;

    li %s4 340;

    li %s5 357;

    li %s6 374;

    li %s7 391;

    li %s8 408;

    li %s9 425;

    li %s10 442;

    li %s11 459;

    li %t3 476;

    li %t4 493;

    li %t5 510;

    li %t6 527;

    ecall;

    addi %t0 %t0 -85;

    bnez %t0 @fail;

    li %t0 17;

    bne %ra %t0 @fail;

    li %t0 51;

    bne %gp %t0 @fail;

    li %t0 68;

    bne %tp %t0 @fail;

    li %t0 102;

    bne %t1 %t0 @fail;

    li %t0 119;

    bne %t2 %t0 @fail;

    li %t0 153;

    bne %s1 %t0 @fail;

    li %t0 170;

    bne %a0 %t0 @fail;

    li %t0 187;

    bne %a1 %t0 @fail;

    li %t0 204;

    bne %a2 %t0 @fail;

    li %t0 221;

    bne %a3 %t0 @fail;

    li %t0 238;

    bne %a4 %t0 @fail;

    li %t0 255;

    bne %a5 %t0 @fail;

    li %t0 272;

    bne %a6 %t0 @fail;

    li %t0 289;

    bne %a7 %t0 @fail;

    li %t0 306;

    bne %s2 %t0 @fail;

    li %t0 323;

    bne %s3 %t0 @fail;

    li %t0 340;

    bne %s4 %t0 @fail;

    li %t0 357;

    bne %s5 %t0 @fail;

    li %t0 374;

    bne %s6 %t0 @fail;

    li %t0 391;

    bne %s7 %t0 @fail;

    li %t0 408;

    bne %s8 %t0 @fail;

    li %t0 425;

    bne %s9 %t0 @fail;

    li %t0 442;

    bne %s10 %t0 @fail;

    li %t0 459;

    bne %s11 %t0 @fail;

    li %t0 476;

    bne %t3 %t0 @fail;

    li %t0 493;

    bne %t4 %t0 @fail;

    li %t0 510;

    bne %t5 %t0 @fail;

    li %t0 527;

    bne %t6 %t0 @fail;

    li %t0 1073807360;

    slli %t0 %t0 1;

    bne %s0 %t0 @fail;

    li %t0 1073809400;

    slli %t0 %t0 1;

    bne %sp %t0 @fail;

    li %t0 85;

@illegalInstruction

    csrr %t0 mhartid;

    addi %t0 %t0 -85;

    bnez %t0 @fail;

    li %a7 1234;

    ecall;

    j @fail;

@success

    li %t0 0x10001000;

    li %t1 0x5555;

    sw %t1 0(%t0);

@fail

    li %t0 0x10001000;

    li %t1 0x13333;

    sw %t1 0(%t0);

@park

    wfi;

    j @park;

// Validate stack/base switching before enabling a nested timer interrupt.

@handler

    li %t0 0x12345678;

    bne %gp %t0 @fail;

    li %t0 1073807360;

    slli %t0 %t0 1;

    bne %tp %t0 @fail;

    bne %a1 %tp @fail;

    bne %a0 %sp @fail;

    ld %t0 272(%a0);

    li %t1 1;

    slli %t1 %t1 63;

    ori %t1 %t1 7;

    beq %t0 %t1 @timer;

    li %t1 8;

    beq %t0 %t1 @userCall;

    li %t1 2;

    beq %t0 %t1 @illegal;

    li %t1 11;

    bne %t0 %t1 @fail;

    ld %t0 64(%tp);

    bnez %t0 @fail;

    li %t0 1;

    sd %t0 64(%tp);

    addi %sp %sp -16;

    sd %ra 0(%sp);

    sd %a0 8(%sp);

    li %t0 0x02004000;

    sd %zero 0(%t0);

    li %t0 128;

    csrw mie %t0;

    li %s10 0;

    csrr %t0 mstatus;

    ori %t0 %t0 8;

    csrw mstatus %t0;

@awaitTimer

    addi %s10 %s10 1;

    ld %t0 64(%tp);

    li %t1 2;

    bne %t0 %t1 @awaitTimer;

    li %t0 1;

    bne %s10 %t0 @fail;

    csrr %t0 mstatus;

    andi %t0 %t0 8;

    beqz %t0 @fail;

    csrr %t0 mstatus;

    andi %t0 %t0 -9;

    csrw mstatus %t0;

    csrw mie %zero;

    ld %ra 0(%sp);

    ld %a0 8(%sp);

    addi %sp %sp 16;

    j @advance;

@timer

    ld %t0 256(%a0);

    la %t1 @awaitTimer;

    bne %t0 %t1 @fail;

    ld %t0 264(%a0);

    li %t1 0x1800;

    and %t0 %t0 %t1;

    bne %t0 %t1 @fail;

    ld %t0 16(%a0);

    addi %t1 %a0 288;

    bne %t0 %t1 @fail;

    li %t0 0x02004000;

    li %t1 -1;

    sd %t1 0(%t0);

    li %t0 2;

    sd %t0 64(%tp);

    ret;

@illegal

    ld %t0 256(%a0);

    la %t1 @illegalInstruction;

    bne %t0 %t1 @fail;

    ld %t0 280(%a0);

    li %t1 2023756153;

    slli %t1 %t1 1;

    addi %t1 %t1 1;

    bne %t0 %t1 @fail;

    li %t0 1;

    sd %t0 72(%tp);

    j @advance;

@userCall

    ld %t0 136(%a0);

    li %t1 1234;

    bne %t0 %t1 @checkUserFrame;

    ld %t0 72(%tp);

    li %t1 1;

    bne %t0 %t1 @fail;

    j @success;

@checkUserFrame

    ld %t0 264(%a0);

    li %t1 0x1800;

    and %t0 %t0 %t1;

    bnez %t0 @fail;

    ld %t0 0(%tp);

    addi %t0 %t0 -288;

    bne %a0 %t0 @fail;

@advance

    ld %t0 256(%a0);

    addi %t0 %t0 4;

    sd %t0 256(%a0);

    ret;

//! Reservation isolation at trap entry, frame resume, and user context switches.

.text;

    li %s0 0x40010000;

    slli %s0 %s0 1;

    sd %sp 0(%s0);

    li %t0 4096;

    sub %t0 %sp %t0;

    sd %t0 8(%s0);

    sd %zero 16(%s0);

    la %t0 @handler;

    sd %t0 24(%s0);

    mv %a0 %s0;

    call @kernel::trap::install;

    addi %s1 %s0 80;

// Enter the vector with a live reservation and the machine trap CSR state.

    la %t0 @resumed;

    csrw mepc %t0;

    li %t0 11;

    csrw mcause %t0;

    li %t0 0x1800;

    csrw mstatus %t0;

    lr.d %t0 (%s1);

    j @kernel::trap::vector;

@resumed

    sc.d %t0 %zero (%s1);

    beqz %t0 @fail;

    ld %t0 0(%s1);

    bnez %t0 @fail;

    li %t0 2;

    sd %t0 88(%s0);

    la %t0 @first;

    csrw mepc %t0;

    csrw mstatus %zero;

    mret;

// The first user context retains its loaded value across a peer's execution.

@first

    lr.d %a2 (%s1);

    ecall;

@firstResume

    sc.d %a3 %a2 (%s1);

    beqz %a3 @fail;

    ld %t0 0(%s1);

    li %t1 1;

    bne %t0 %t1 @fail;

    ecall;

    j @fail;

// The peer changes shared memory and reserves the same location before exit.

@peer

    li %t0 1;

    sd %t0 0(%s1);

    lr.d %t0 (%s1);

    ecall;

    j @fail;

// Inject a live return reservation after trap entry has completed.

@handler

    ld %t0 88(%s0);

    beqz %t0 @entryCheck;

    ld %t1 272(%a0);

    li %t2 8;

    bne %t1 %t2 @fail;

    li %t1 2;

    beq %t0 %t1 @switchPeer;

    li %t1 3;

    beq %t0 %t1 @switchFirst;

    li %t1 4;

    bne %t0 %t1 @fail;

    j @success;

@entryCheck

    sc.d %t0 %zero (%s1);

    beqz %t0 @fail;

    li %t0 1;

    sd %t0 88(%s0);

    lr.d %t0 (%s1);

    ret;

@switchPeer

    li %t0 3;

    sd %t0 88(%s0);

    la %t0 @peer;

    sd %t0 256(%a0);

    ret;

@switchFirst

    li %t0 4;

    sd %t0 88(%s0);

    la %t0 @firstResume;

    sd %t0 256(%a0);

    lr.d %t0 (%s1);

    ret;

@success

    li %t0 0x10001000;

    li %t1 0x5555;

    sw %t1 0(%t0);

@fail

    li %t0 0x10001000;

    li %t1 0x13333;

    sw %t1 0(%t0);

    ebreak;

#!/bin/sh

# Execute the production trap boundary with a deterministic nested timer.

set -eu

emulator=${RAD_EMULATOR:-emulator}

work=$(mktemp -d)

trap 'rm -rf "$work"' EXIT HUP INT TERM

for fixture in machine reservations; do

    cat "test/trap/$fixture.ras" kernel/kernel/trap.ras > "$work/trap.ras"

    "$emulator" -run bin/sync.build.rv64 -- "$work/trap.ras" "$work/trap.rv64"

    "$emulator" -machine -max-steps=1000000 -run "$work/trap.rv64"

done

printf 'trap contexts: machine/user registers, nested timer, illegal instruction, and stacks passed\n'

printf 'trap reservations: entry, resume, and user context isolation passed\n'