test: emulator seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test

# Kernel synchronization machine tests

sync-test: $(BIN_DIR)/sync.build.rv64

	@RAD_EMULATOR="$(EMU)" test/sync/run

$(BIN_DIR)/sync.build.rv64: test/sync/build.rad $(STD_LIB) $(RAD_BIN)

	@$(RADIANCE) $(STD) -pkg build -mod $< -entry build -o $@

.PHONY: test clean default seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test seed \

export mod sync;

//! RV64 synchronization and ordered device access.

/// Allocate one wrapping ticket from a naturally aligned u32 counter.

export fn nextTicket(counter: &mut u32) -> u32;

/// Read a naturally aligned shared word with acquire ordering.

export fn loadAcquire(value: &u64) -> u64;

/// Publish a naturally aligned shared word with release ordering.

export fn storeRelease(value: &mut u64, next: u64);

/// Read a naturally aligned shared 32-bit word with acquire ordering.

export fn loadAcquire32(value: &u32) -> u32;

/// Publish a naturally aligned shared 32-bit word with release ordering.

export fn storeRelease32(value: &mut u32, next: u32);

/// Add to a naturally aligned shared word and return its previous value.

export fn fetchAdd(value: &mut u64, amount: u64) -> u64;

/// Synchronize local instruction fetch after executable bytes become visible.

export fn syncInstructions();

/// Order memory and device accesses in both directions.

export fn deviceFence();

/// Read a 32-bit device register with memory and I/O ordering.

export unsafe fn read32(address: &u32) -> u32;

/// Write a 32-bit device register with memory and I/O ordering.

export unsafe fn write32(address: &mut u32, value: u32);

//! RV64 synchronization boundary. Shared words must be naturally aligned.

.text;

.export @kernel::sync::nextTicket;

.export @kernel::sync::loadAcquire;

.export @kernel::sync::storeRelease;

.export @kernel::sync::loadAcquire32;

.export @kernel::sync::storeRelease32;

.export @kernel::sync::fetchAdd;

.export @kernel::sync::syncInstructions;

.export @kernel::sync::deviceFence;

.export @kernel::sync::read32;

.export @kernel::sync::write32;

// Allocate a wrapping ticket and return its zero-extended u32 value.

@kernel::sync::nextTicket

    li %t0 1;

    amoadd.w.aqrl %a0 %t0 (%a0);

    slli %a0 %a0 32;

    srli %a0 %a0 32;

    ret;

// Order subsequent memory accesses after this load.

@kernel::sync::loadAcquire

    ld %a0 0(%a0);

    fence r rw;

    ret;

// Order preceding memory accesses before this store.

@kernel::sync::storeRelease

    fence rw w;

    sd %a1 0(%a0);

    ret;

// Order subsequent memory accesses after a zero-extended word load.

@kernel::sync::loadAcquire32

    lwu %a0 0(%a0);

    fence r rw;

    ret;

// Order preceding memory accesses before a word store.

@kernel::sync::storeRelease32

    fence rw w;

    sw %a1 0(%a0);

    ret;

// Atomically add with acquire and release ordering.

@kernel::sync::fetchAdd

    amoadd.d.aqrl %a0 %a1 (%a0);

    ret;

// Refresh the local instruction stream after code publication.

@kernel::sync::syncInstructions

    fence.i;

    ret;

// Order both memory and device accesses.

@kernel::sync::deviceFence

    fence iorw iorw;

    ret;

// Read a device word between full I/O barriers.

@kernel::sync::read32

    fence iorw iorw;

    lwu %a0 0(%a0);

    fence iorw iorw;

    ret;

// Write a device word between full I/O barriers.

@kernel::sync::write32

    fence iorw iorw;

    sw %a1 0(%a0);

    fence iorw iorw;

    ret;

export mod atomics;

@test mod atomicTests;

    /// Memory fence with optional predecessor and successor masks.

    Fence,

export constant INSTRUCTIONS: [InstructionEntry; 89] = [

    { name: "fence",  encoder: InstructionEncoder::Fence },

    { name: "fence.i", encoder: InstructionEncoder::NoOperand { enc: encode::fenceI } },

use std::arch::rv64::atomics;

    if let format = atomics::parse(name) {

        let rd = try parseRegister(a);

        let mut rs2 = rv64::ZERO;

        if format.operation <> 2 { set rs2 = try parseRegister(a); }

        let memory = try parseMemory(a);

        if memory.offset <> 0 { throw fail(a, "atomic memory offset must be zero"); }

        try emit::emitText(a, atomics::encode(atomics::Instruction { format, rd, rs1: memory.base, rs2 }));

        return;

    }

        case super::InstructionEncoder::Fence => return try parseFence(a),

/// Parse an access-class mask without duplicate fields.

unsafe fn parseFenceMask(a: &mut super::Assembler) -> u32 throws (super::Error) {

    if a.scan.current.kind == scanner::TokenKind::Number {

        let value = try parseValue(a);

        if value <> 0 { throw fail(a, "numeric fence mask must be zero"); }

        return 0;

    }

    let token = try expectToken(a, scanner::TokenKind::Ident, "expected fence access classes");

    let mut mask: u32 = 0;

    for ch in token.source {

        let mut bit: u32 = 0;

        match ch {

            case 'i' => { set bit = 8; }, case 'o' => { set bit = 4; },

            case 'r' => { set bit = 2; }, case 'w' => { set bit = 1; },

            else => throw failOnToken(token, "invalid fence access class"),

        }

        if (mask & bit) <> 0 { throw failOnToken(token, "duplicate fence access class"); }

        set mask |= bit;

    }

    return mask;

}

/// Parse a full memory fence or a pair of explicit access-class masks.

unsafe fn parseFence(a: &mut super::Assembler) throws (super::Error) {

    if a.scan.current.kind == scanner::TokenKind::Semicolon {

        try emit::emitText(a, encode::fence());

        return;

    }

    let predecessor = try parseFenceMask(a);

    let successor = try parseFenceMask(a);

    try emit::emitText(a, encode::fenceOrder(predecessor, successor));

}

    while let ch = current(s); char::isAlpha(ch) or char::isDigit(ch) or ch == '_' or ch == '.' {

//! RV64 A-extension and fence encoding checks.

use std::testing;

use std::lang::alloc;

use std::lang::strings;

use super::atomics;

use super::encode;

use super::decode;

use super::asm;

/// Assembler workspace for individual atomic instruction fixtures.

static MEMORY: [u8; 65536] = [0; 65536];

/// Interned fixture names.

unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };

/// Output storage for one assembled instruction.

static WORDS: [u32; 1] = [0; 1];

/// Exact unordered word encodings for a0, a1, and a2 operands.

@test fn encodings() throws (testing::TestError) {

    let operations: [u32; 11] = [0, 1, 2, 3, 4, 8, 12, 16, 20, 24, 28];

    let words: [u32; 11] = [

        0x00c5a52f, 0x08c5a52f, 0x1005a52f, 0x18c5a52f, 0x20c5a52f,

        0x40c5a52f, 0x60c5a52f, 0x80c5a52f, 0xa0c5a52f, 0xc0c5a52f, 0xe0c5a52f,

    ];

    for operation, i in &operations[..] {

        for wide in 0..2 {

            for order in 0..4 {

                let item = atomics::Instruction {

                    format: atomics::Format { operation, width: 2 + wide, order },

                    rd: super::A0, rs1: super::A1, rs2: super::ZERO if operation == 2 else super::A2,

                };

                let word = words[i] | (wide << 12) | (order << 25);

                try testing::expect(atomics::encode(item) == word);

                let decoded = atomics::decode(word) else { throw testing::TestError::Failed; };

                try testing::expect(decoded == item);

                let case decode::Instr::Atomic(instruction) = decode::decode(word) else { throw testing::TestError::Failed; };

                try testing::expect(instruction == item);

            }

        }

    }

}

/// Reject reserved opcodes, widths, LR sources, and malformed suffixes.

@test fn invalid() throws (testing::TestError) {

    for word in &[0x2805a52f, 0x00c5852f, 0x10c5a52f, 0x00c5a513] {

        try testing::expect(atomics::decode(word) == nil);

    }

    for name in &["lr", "lr.q", "lr.w.aq.rl", "sc.d.rl.aq", "amoswap.d.bad", "amoadd.wextra"] {

        try testing::expect(atomics::parse(name) == nil);

    }

    let format = atomics::parse("amoswap.d.aqrl") else { throw testing::TestError::Failed; };

    try testing::expect(format.operation == 1 and format.width == 3 and format.order == 3);

}

/// Assemble one instruction and return its exact word.

unsafe fn assemble(source: *[u8]) -> u32 throws (testing::TestError) {

    let mut arena = alloc::new(&mut MEMORY[..]);

    let words = &mut WORDS[..];

    let data: *mut [u8] = &mut [];

    let result = try asm::assemble(asm::scanner::SourceKind::String, source,

        &mut words[..], &mut data[..], &mut arena, &mut STRINGS, 0)

        catch { throw testing::TestError::Failed; };

    try testing::expect(result.text.len == 1);

    return result.text[0];

}

/// Check instruction suffix scanning and assembler operand order.

@test unsafe fn assembly() throws (testing::TestError) {

    try testing::expect((try assemble("amoadd.d.aqrl %a0 %a2 (%a1);")) == 0x06c5b52f);

    try testing::expect((try assemble("amoswap.w.aq %a0 %a2 0(%a1);")) == 0x0cc5a52f);

    try testing::expect((try assemble("lr.d.aq %a0 (%a1);")) == 0x1405b52f);

    try testing::expect((try assemble("sc.w.rl %a0 %a2 (%a1);")) == 0x1ac5a52f);

    try testing::expect((try assemble("fence.i;")) == 0x0000100f);

    try testing::expect((try assemble("fence iorw iorw;")) == 0x0ff0000f);

    try testing::expect((try assemble("fence r rw;")) == 0x0230000f);

    try testing::expect((try assemble("fence rw w;")) == 0x0310000f);

    try testing::expect((try assemble("fence 0 0;")) == 0x0000000f);

}

/// Decode fence classes and local instruction synchronization.

@test fn fences() throws (testing::TestError) {

    try testing::expect(encode::fenceI() == 0x0000100f);

    try testing::expect(decode::decode(encode::fenceI()) == decode::Instr::FenceI);

    let case decode::Instr::Fence { predecessor, successor } = decode::decode(encode::fenceOrder(15, 15))

        else { throw testing::TestError::Failed; };

    try testing::expect(predecessor == 15 and successor == 15);

    let case decode::Instr::Unknown { .. } = decode::decode(0x0000200f)

        else { throw testing::TestError::Failed; };

}

/// Reject invalid atomic operands and fence access classes.

@test unsafe fn invalidAssembly() throws (testing::TestError) {

    for source in &[

        "lr.w %a0 %a2 (%a1);", "sc.d %a0 %a2 8(%a1);",

        "amoadd.d.aq.aq %a0 %a2 (%a1);", "fence rr rw;", "fence rx rw;",

        "fence 1 rw;", "fence.i %a0;",

    ] {

        let mut arena = alloc::new(&mut MEMORY[..]);

        let words = &mut WORDS[..];

        let data: *mut [u8] = &mut [];

        let mut rejected = false;

        try asm::assemble(asm::scanner::SourceKind::String, source,

            &mut words[..], &mut data[..], &mut arena, &mut STRINGS, 0) catch {

            set rejected = true;

        };

        try testing::expect(rejected);

    }

}

//! RV64 atomic instruction fields and canonical assembly names.

use std::mem;

use std::lang::gen;

/// Atomic operation, width, and ordering fields.

export record Format: Copy {

    /// Five-bit operation code from the A extension.

    operation: u32,

    /// Memory width encoding: 2 for a word, 3 for a doubleword.

    width: u32,

    /// Ordering bits: 2 for acquire, 1 for release, 3 for both.

    order: u32,

}

/// Decoded atomic instruction operands.

export record Instruction: Copy {

    /// Operation and ordering fields.

    format: Format,

    /// Destination register.

    rd: gen::Reg,

    /// Address register.

    rs1: gen::Reg,

    /// Source value; zero for load-reserved.

    rs2: gen::Reg,

}

/// An architectural operation and its mnemonic stem.

record Operation: Copy {

    /// Mnemonic without width or ordering suffixes.

    name: *[u8],

    /// Five-bit instruction field.

    code: u32,

}

/// Operations shared by the 32-bit and 64-bit forms.

constant OPERATIONS: [Operation; 11] = [

    { name: "amoadd", code: 0 }, { name: "amoswap", code: 1 },

    { name: "lr", code: 2 }, { name: "sc", code: 3 },

    { name: "amoxor", code: 4 }, { name: "amoor", code: 8 },

    { name: "amoand", code: 12 }, { name: "amomin", code: 16 },

    { name: "amomax", code: 20 }, { name: "amominu", code: 24 },

    { name: "amomaxu", code: 28 },

];

/// Look up an operation's canonical mnemonic stem.

export fn name(code: u32) -> ?*[u8] {

    for operation in &OPERATIONS[..] { if operation.code == code { return operation.name; } }

    return nil;

}

/// Parse an atomic mnemonic with mandatory width and optional ordering suffix.

export fn parse(text: *[u8]) -> ?Format {

    for operation in &OPERATIONS[..] {

        let n = operation.name.len;

        if text.len < n + 2 or not mem::eq(&text[..n], operation.name) or text[n] <> '.' { continue; }

        let mut width: u32 = 2;

        if text[n + 1] == 'd' { set width = 3; }

        else if text[n + 1] <> 'w' { return nil; }

        let suffix = &text[n + 2..];

        let mut order: u32 = 0;

        if mem::eq(suffix, ".aq") { set order = 2; }

        else if mem::eq(suffix, ".rl") { set order = 1; }

        else if mem::eq(suffix, ".aqrl") { set order = 3; }

        else if suffix.len <> 0 { return nil; }

        return Format { operation: operation.code, width, order };

    }

    return nil;

}

/// Encode a validated atomic operation and register operands.

export fn encode(instruction: Instruction) -> u32 {

    let format = instruction.format;

    assert name(format.operation) <> nil and (format.width == 2 or format.width == 3) and format.order <= 3;

    assert format.operation <> 2 or instruction.rs2 == super::ZERO;

    return 0x2f | (*instruction.rd as u32 << 7) | (format.width << 12)

        | (*instruction.rs1 as u32 << 15) | (*instruction.rs2 as u32 << 20)

        | (format.order << 25) | (format.operation << 27);

}

/// Decode a supported atomic word and reject reserved width and LR fields.

export fn decode(word: u32) -> ?Instruction {

    let operation = word >> 27;

    let width = (word >> 12) & 7;

    let rs2 = super::reg(((word >> 20) & 31) as u8);

    if (word & 127) <> 0x2f or name(operation) == nil or (width <> 2 and width <> 3) { return nil; }

    if operation == 2 and rs2 <> super::ZERO { return nil; }

    return Instruction {

        format: Format { operation, width, order: (word >> 25) & 3 },

        rd: super::reg(((word >> 7) & 31) as u8), rs1: super::reg(((word >> 15) & 31) as u8), rs2,

    };

}

use super::atomics;

    /// Atomic memory operation.

    Atomic(atomics::Instruction),

    /// Memory and I/O ordering fence.

    Fence {

        /// Ordered preceding access classes.

        predecessor: u32,

        /// Ordered following access classes.

        successor: u32,

    },

    /// Local instruction-fetch synchronization.

    FenceI,

        case 0x2f => {

            if let atomic = atomics::decode(instr) { return Instr::Atomic(atomic); }

            return Instr::Unknown { bits: instr };

        },

        case 0x0f => {

            if instr == 0x100f { return Instr::FenceI; }

            if (instr & 0xf00fffff) == 0x0f {

                return Instr::Fence { predecessor: (instr >> 24) & 15, successor: (instr >> 20) & 15 };

            }

            return Instr::Unknown { bits: instr };

        },

    return fenceOrder(3, 3);

/// Order the specified predecessor and successor memory or I/O access classes.

/// Each mask uses I=8, O=4, R=2, W=1.

export fn fenceOrder(predecessor: u32, successor: u32) -> u32 {

    assert predecessor <= 15 and successor <= 15;

    return 0x0f | (predecessor << 24) | (successor << 20);

}

/// Synchronize subsequent instruction fetch on the executing hart.

export fn fenceI() -> u32 { return 0x0000100f; }

use super::atomics;

        case decode::Instr::Atomic(instruction) => {

            let stem = atomics::name(instruction.format.operation) else panic "invalid atomic operation";

            write(out, stem);

            write(out, ".w" if instruction.format.width == 2 else ".d");

            match instruction.format.order {

                case 1 => write(out, ".rl"), case 2 => write(out, ".aq"), case 3 => write(out, ".aqrl"),

                else => {},

            }

            write(out, " "); write(out, regNameR(instruction.rd)); write(out, ", ");

            if instruction.format.operation <> 2 { write(out, regNameR(instruction.rs2)); write(out, ", "); }

            write(out, "0("); write(out, regNameR(instruction.rs1)); write(out, ")");

        },

        case decode::Instr::Fence { predecessor, successor } => {

            write(out, "fence "); fenceMask(out, predecessor); write(out, ", "); fenceMask(out, successor);

        },

        case decode::Instr::FenceI => write(out, "fence.i"),

/// Print a memory-ordering mask in canonical order.

unsafe fn fenceMask(out: &mut sexpr::Output, mask: u32) {

    if mask == 0 { write(out, "0"); return; }

    if (mask & 8) <> 0 { write(out, "i"); }

    if (mask & 4) <> 0 { write(out, "o"); }

    if (mask & 2) <> 0 { write(out, "r"); }

    if (mask & 1) <> 0 { write(out, "w"); }

}

lib/std/arch/rv64/atomics.rad

lib/std/arch/rv64/atomicTests.rad

//! Build a native image from the synchronization machine fixture.

use std::sys;

use std::io;

use std::sys::unix;

use std::lang::alloc;

use std::lang::strings;

use std::arch::rv64::asm;

use std::arch::rv64::image;

/// Assembly input storage.

static SOURCE: [u8; 65536] = [0; 65536];

/// Encoded text words.

static TEXT: [u32; 4096] = [0; 4096];

/// Assembly name and fixup storage.

static MEMORY: [u8; 4194304] = [0; 4194304];

/// Interned assembly identifiers.

unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };

/// Assemble startup and synchronization routines into one native code segment.

@default unsafe fn main(env: *sys::Env) -> i32 {

    assert env.args.len == 3;

    let length = unix::readFile(env.args[1], &mut SOURCE[..]) else panic "missing assembly";

    let mut arena = alloc::new(&mut MEMORY[..]);

    let data: *mut [u8] = &mut [];

    let program = try asm::assemble(asm::scanner::SourceKind::String, &SOURCE[..length],

        &mut TEXT[..], &mut data[..], &mut arena, &mut STRINGS, 0) catch err {

        match err {

            case asm::Error::Invalid { offset, message } => {

                io::printU32(offset); io::print(": "); io::printLn(message);

            },

            else => io::printLn("assembly output full"),

        }

        return 1;

    };

    assert program.externalFixups.len == 0;

    let size = program.text.len * 4;

    let header = try! image::header(image::Layout {

        entry: 0x80010000,

        code: image::Segment { address: 0x80010000, initialized: size, memory: size },

        roData: image::Segment { address: 0, initialized: 0, memory: 0 },

        rwData: image::Segment { address: 0x80020000, initialized: 0, memory: 4096 },

    });

    let fd = unix::openOpts(env.args[2], unix::OpenFlags(*unix::O_WRONLY | *unix::O_CREAT | *unix::O_TRUNC), 420);

    assert fd >= 0;

    let written = unix::writeAll(fd, &header[..]) and unix::writeAll(fd, @sliceOf(program.text.ptr as *u8, size));

    let closed = unix::close(fd) == 0;

    assert written and closed;

    return 0;

}

//! Exercise code publication and contended metadata updates on each hart.

.constant EXPECTED HARTS * 100;

.text;

@entry

    csrr %s6 mhartid;

    li %s0 0x40010000;

    slli %s0 %s0 1;

    bnez %s6 @waitCode;

    li %t0 1;

    slli %t0 %t0 32;

    addi %t0 %t0 -4;

    sw %t0 0(%s0);

    sw %t0 4(%s0);

    li %t0 0x02a00513;

    sw %t0 64(%s0);

    li %t0 0x00008067;

    sw %t0 68(%s0);

    li %t0 37;

    sw %t0 44(%s0);

    addi %a0 %s0 40;

    li %a1 -1;

    call @kernel::sync::storeRelease32;

    addi %a0 %s0 24;

    li %a1 1;

    call @kernel::sync::storeRelease;

@waitCode

    addi %a0 %s0 24;

    call @kernel::sync::loadAcquire;

    beqz %a0 @waitCode;

    addi %a0 %s0 40;

    call @kernel::sync::loadAcquire32;

    li %t0 1;

    slli %t0 %t0 32;

    addi %t0 %t0 -1;

    bne %a0 %t0 @fail;

    lwu %t0 44(%s0);

    li %t1 37;

    bne %t0 %t1 @fail;

    call @kernel::sync::syncInstructions;

    addi %t0 %s0 64;

    jalr %ra %t0 0;

    li %t0 42;

    bne %a0 %t0 @fail;

    li %s1 100;

@ticket

    mv %a0 %s0;

    call @kernel::sync::nextTicket;

    mv %s2 %a0;

    li %t0 100;

    bne %s1 %t0 @acquire;

    // Every hart holds its first ticket before any hart can enter the lock.

    addi %a0 %s0 72;

    li %a1 1;

    call @kernel::sync::fetchAdd;

@queued

    addi %a0 %s0 72;

    call @kernel::sync::loadAcquire;

    li %t0 HARTS;

    bne %a0 %t0 @queued;

@acquire

    addi %a0 %s0 4;

    call @kernel::sync::loadAcquire32;

    bne %a0 %s2 @acquire;

    // The protected count must follow ticket order across the u32 wrap.

    addi %t1 %s2 4;

    slli %t1 %t1 32;

    srli %t1 %t1 32;

    ld %t0 8(%s0);

    bne %t0 %t1 @fail;

    addi %t0 %t0 1;

    sd %t0 8(%s0);

    addi %a0 %s0 4;

    addi %a1 %s2 1;

    call @kernel::sync::storeRelease32;

    addi %s1 %s1 -1;

    bnez %s1 @ticket;

    addi %a0 %s0 16;

    li %a1 1;

    call @kernel::sync::fetchAdd;

    addi %a0 %a0 1;

    li %t0 HARTS;

    beq %a0 %t0 @check;

@park

    wfi;

    j @park;

@check

    ld %t0 8(%s0);

    li %t1 EXPECTED;

    bne %t0 %t1 @fail;

    lwu %t0 0(%s0);

    lwu %t2 4(%s0);

    bne %t0 %t2 @fail;

    addi %t1 %t1 -4;

    bne %t0 %t1 @fail;

    addi %t2 %s0 32;

    lr.w.aq %t0 (%t2);

    bnez %t0 @fail;

    li %t1 13;

    sc.w.rl %t0 %t1 (%t2);

    bnez %t0 @fail;

    lr.d.aq %t0 (%t2);

    bne %t0 %t1 @fail;

    sc.d.rl %t0 %zero (%t2);

    bnez %t0 @fail;

    li %a0 0x02000000;

    call @kernel::sync::read32;

    bnez %a0 @fail;

    j @success;

@fail

    li %a0 0x10001000;

    li %a1 0x13333;

    call @kernel::sync::write32;

    ebreak;

@success

    li %a0 0x10001000;

    li %a1 0x5555;

    call @kernel::sync::write32;

    ebreak;

#!/bin/sh

# Execute the synchronization boundary under deterministic hart interleaving.

set -eu

emulator=${RAD_EMULATOR:-emulator}

work=$(mktemp -d)

trap 'rm -rf "$work"' EXIT HUP INT TERM

for harts in 1 2 8; do

    printf '.constant HARTS %s;\n' "$harts" > "$work/sync.ras"

    cat test/sync/machine.ras kernel/kernel/sync.ras >> "$work/sync.ras"

    "$emulator" -run bin/sync.build.rv64 -- "$work/sync.ras" "$work/sync.rv64"

    "$emulator" -machine -harts="$harts" -run "$work/sync.rv64"

done

printf 'synchronization: code publication, locks, atomics, and MMIO passed on 1/2/8 harts\n'