export mod sync;

//! RV64 synchronization and ordered device access.

/// Allocate one wrapping ticket from a naturally aligned u32 counter.

export fn nextTicket(counter: &mut u32) -> u32;

/// Read a naturally aligned shared word with acquire ordering.

export fn loadAcquire(value: &u64) -> u64;

/// Publish a naturally aligned shared word with release ordering.

export fn storeRelease(value: &mut u64, next: u64);

/// Read a naturally aligned shared 32-bit word with acquire ordering.

export fn loadAcquire32(value: &u32) -> u32;

/// Publish a naturally aligned shared 32-bit word with release ordering.

export fn storeRelease32(value: &mut u32, next: u32);

/// Add to a naturally aligned shared word and return its previous value.

export fn fetchAdd(value: &mut u64, amount: u64) -> u64;

/// Synchronize local instruction fetch after executable bytes become visible.

export fn syncInstructions();

/// Order memory and device accesses in both directions.

export fn deviceFence();

/// Read a 32-bit device register with memory and I/O ordering.

export unsafe fn read32(address: &u32) -> u32;

/// Write a 32-bit device register with memory and I/O ordering.

export unsafe fn write32(address: &mut u32, value: u32);

//! RV64 synchronization boundary. Shared words must be naturally aligned.

.text;

.export @kernel::sync::nextTicket;

.export @kernel::sync::loadAcquire;

.export @kernel::sync::storeRelease;

.export @kernel::sync::loadAcquire32;

.export @kernel::sync::storeRelease32;

.export @kernel::sync::fetchAdd;

.export @kernel::sync::syncInstructions;

.export @kernel::sync::deviceFence;

.export @kernel::sync::read32;

.export @kernel::sync::write32;

// Allocate a wrapping ticket and return its zero-extended u32 value.

@kernel::sync::nextTicket

    li %t0 1;

    amoadd.w.aqrl %a0 %t0 (%a0);

    slli %a0 %a0 32;

    srli %a0 %a0 32;

    ret;

// Order subsequent memory accesses after this load.

@kernel::sync::loadAcquire

    ld %a0 0(%a0);

    fence r rw;

    ret;

// Order preceding memory accesses before this store.

@kernel::sync::storeRelease

    fence rw w;

    sd %a1 0(%a0);

    ret;

// Order subsequent memory accesses after a zero-extended word load.

@kernel::sync::loadAcquire32

    lwu %a0 0(%a0);

    fence r rw;

    ret;

// Order preceding memory accesses before a word store.

@kernel::sync::storeRelease32

    fence rw w;

    sw %a1 0(%a0);

    ret;

// Atomically add with acquire and release ordering.

@kernel::sync::fetchAdd

    amoadd.d.aqrl %a0 %a1 (%a0);

    ret;

// Refresh the local instruction stream after code publication.

@kernel::sync::syncInstructions

    fence.i;

    ret;

// Order both memory and device accesses.

@kernel::sync::deviceFence

    fence iorw iorw;

    ret;

// Read a device word between full I/O barriers.

@kernel::sync::read32

    fence iorw iorw;

    lwu %a0 0(%a0);

    fence iorw iorw;

    ret;

// Write a device word between full I/O barriers.

@kernel::sync::write32

    fence iorw iorw;

    sw %a1 0(%a0);

    fence iorw iorw;

    ret;

//! Build a native image from the synchronization machine fixture.

use std::sys;

use std::io;

use std::sys::unix;

use std::lang::alloc;

use std::lang::strings;

use std::arch::rv64::asm;

use std::arch::rv64::image;

/// Assembly input storage.

static SOURCE: [u8; 65536] = [0; 65536];

/// Encoded text words.

static TEXT: [u32; 4096] = [0; 4096];

/// Assembly name and fixup storage.

static MEMORY: [u8; 4194304] = [0; 4194304];

/// Interned assembly identifiers.

unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };

/// Assemble startup and synchronization routines into one native code segment.

@default unsafe fn main(env: *sys::Env) -> i32 {

    assert env.args.len == 3;

    let length = unix::readFile(env.args[1], &mut SOURCE[..]) else panic "missing assembly";

    let mut arena = alloc::new(&mut MEMORY[..]);

    let data: *mut [u8] = &mut [];

    let program = try asm::assemble(asm::scanner::SourceKind::String, &SOURCE[..length],

        &mut TEXT[..], &mut data[..], &mut arena, &mut STRINGS, 0) catch err {

        match err {

            case asm::Error::Invalid { offset, message } => {

                io::printU32(offset); io::print(": "); io::printLn(message);

            },

            else => io::printLn("assembly output full"),

        }

        return 1;

    };

    assert program.externalFixups.len == 0;

    let size = program.text.len * 4;

    let header = try! image::header(image::Layout {

        entry: 0x80010000,

        code: image::Segment { address: 0x80010000, initialized: size, memory: size },

        roData: image::Segment { address: 0, initialized: 0, memory: 0 },

        rwData: image::Segment { address: 0x80020000, initialized: 0, memory: 4096 },

    });

    let fd = unix::openOpts(env.args[2], unix::OpenFlags(*unix::O_WRONLY | *unix::O_CREAT | *unix::O_TRUNC), 420);

    assert fd >= 0;

    let written = unix::writeAll(fd, &header[..]) and unix::writeAll(fd, @sliceOf(program.text.ptr as *u8, size));

    let closed = unix::close(fd) == 0;

    assert written and closed;

    return 0;

}

//! Exercise code publication and contended metadata updates on each hart.

.constant EXPECTED HARTS * 100;

.text;

@entry

    csrr %s6 mhartid;

    li %s0 0x40010000;

    slli %s0 %s0 1;

    bnez %s6 @waitCode;

    li %t0 1;

    slli %t0 %t0 32;

    addi %t0 %t0 -4;

    sw %t0 0(%s0);

    sw %t0 4(%s0);

    li %t0 0x02a00513;

    sw %t0 64(%s0);

    li %t0 0x00008067;

    sw %t0 68(%s0);

    li %t0 37;

    sw %t0 44(%s0);

    addi %a0 %s0 40;

    li %a1 -1;

    call @kernel::sync::storeRelease32;

    addi %a0 %s0 24;

    li %a1 1;

    call @kernel::sync::storeRelease;

@waitCode

    addi %a0 %s0 24;

    call @kernel::sync::loadAcquire;

    beqz %a0 @waitCode;

    addi %a0 %s0 40;

    call @kernel::sync::loadAcquire32;

    li %t0 1;

    slli %t0 %t0 32;

    addi %t0 %t0 -1;

    bne %a0 %t0 @fail;

    lwu %t0 44(%s0);

    li %t1 37;

    bne %t0 %t1 @fail;

    call @kernel::sync::syncInstructions;

    addi %t0 %s0 64;

    jalr %ra %t0 0;

    li %t0 42;

    bne %a0 %t0 @fail;

    li %s1 100;

@ticket

    mv %a0 %s0;

    call @kernel::sync::nextTicket;

    mv %s2 %a0;

    li %t0 100;

    bne %s1 %t0 @acquire;

    // Every hart holds its first ticket before any hart can enter the lock.

    addi %a0 %s0 72;

    li %a1 1;

    call @kernel::sync::fetchAdd;

@queued

    addi %a0 %s0 72;

    call @kernel::sync::loadAcquire;

    li %t0 HARTS;

    bne %a0 %t0 @queued;

@acquire

    addi %a0 %s0 4;

    call @kernel::sync::loadAcquire32;

    bne %a0 %s2 @acquire;

    // The protected count must follow ticket order across the u32 wrap.

    addi %t1 %s2 4;

    slli %t1 %t1 32;

    srli %t1 %t1 32;

    ld %t0 8(%s0);

    bne %t0 %t1 @fail;

    addi %t0 %t0 1;

    sd %t0 8(%s0);

    addi %a0 %s0 4;

    addi %a1 %s2 1;

    call @kernel::sync::storeRelease32;

    addi %s1 %s1 -1;

    bnez %s1 @ticket;

    addi %a0 %s0 16;

    li %a1 1;

    call @kernel::sync::fetchAdd;

    addi %a0 %a0 1;

    li %t0 HARTS;

    beq %a0 %t0 @check;

@park

    wfi;

    j @park;

@check

    ld %t0 8(%s0);

    li %t1 EXPECTED;

    bne %t0 %t1 @fail;

    lwu %t0 0(%s0);

    lwu %t2 4(%s0);

    bne %t0 %t2 @fail;

    addi %t1 %t1 -4;

    bne %t0 %t1 @fail;

    addi %t2 %s0 32;

    lr.w.aq %t0 (%t2);

    bnez %t0 @fail;

    li %t1 13;

    sc.w.rl %t0 %t1 (%t2);

    bnez %t0 @fail;

    lr.d.aq %t0 (%t2);

    bne %t0 %t1 @fail;

    sc.d.rl %t0 %zero (%t2);

    bnez %t0 @fail;

    li %a0 0x02000000;

    call @kernel::sync::read32;

    bnez %a0 @fail;

    j @success;

@fail

    li %a0 0x10001000;

    li %a1 0x13333;

    call @kernel::sync::write32;

    ebreak;

@success

    li %a0 0x10001000;

    li %a1 0x5555;

    call @kernel::sync::write32;

    ebreak;

#!/bin/sh

# Execute the synchronization boundary under deterministic hart interleaving.

set -eu

emulator=${RAD_EMULATOR:-emulator}

work=$(mktemp -d)

trap 'rm -rf "$work"' EXIT HUP INT TERM

for harts in 1 2 8; do

    printf '.constant HARTS %s;\n' "$harts" > "$work/sync.ras"

    cat test/sync/machine.ras kernel/kernel/sync.ras >> "$work/sync.ras"

    "$emulator" -run bin/sync.build.rv64 -- "$work/sync.ras" "$work/sync.rv64"

    "$emulator" -machine -harts="$harts" -run "$work/sync.rv64"

done

printf 'synchronization: code publication, locks, atomics, and MMIO passed on 1/2/8 harts\n'