radiance · commit

compiler: Select inline RV64 instructions in checked code

d3ace8fb8614a750cc1c60a9843aaf65cb01217caf6e9cbe875405b4a3ce5297

Alexis Sellier committed 6 days ago 1 parent 81619cc0

lib/std/arch/rv64/bounds.rad +89 -0


/// Reusable emitter allocation storage.
static MEMORY: [u8; 16777216] = [0; 16777216];
/// Function and liveness test storage.
static SCRATCH: [u8; 65536] = [0; 65536];
/// Code output with guard words for instruction selection checks.
static SELECTION_WORDS: [u32; 130] = [0; 130];

/// Dictionary storage for bounded map tests.
unsafe static ENTRIES: [dict::Entry; 4] = undefined;

/// Build a non-debug generator with a fixed code address.

            throw testing::TestError::Failed;
        };
    try testing::expect(length == 8 and bytes[0] == 1);
}

/// Inline instruction selection respects every shorter output capacity.
@test unsafe fn inlineSelectionCapacity() throws (testing::TestError) {
    let dst = il::Reg { n: 3 };
    let first = il::Reg { n: 0 };
    let second = il::Reg { n: 1 };
    let a = il::Val::Reg(first);
    let b = il::Val::Reg(second);
    let instructions = [
        il::Instr::BinOp { op: il::BinOp::Add, typ: il::Type::W64, dst, a, b },
        il::Instr::UnOp { op: il::UnOp::Neg, typ: il::Type::W32, dst, a },
        il::Instr::Load { typ: il::Type::W8, dst, src: first, offset: 4096 },
        il::Instr::Sload { typ: il::Type::W16, dst, src: first, offset: -4096 },
        il::Instr::Store { typ: il::Type::W32, src: a, dst: second, offset: 4096 },
        il::Instr::Copy { dst, val: il::Val::Imm(0x123456789abcdef) },
        il::Instr::Reserve { dst, size: il::Val::Imm(32), alignment: 16 },
        il::Instr::Reserve { dst, size: a, alignment: 16 },
        il::Instr::Blit { dst: first, src: second, size: il::Val::Imm(0) },
        il::Instr::Blit { dst: first, src: second, size: il::Val::Imm(8) },
        il::Instr::Blit { dst: first, src: second, size: il::Val::Imm(40) },
        il::Instr::Zext { typ: il::Type::W16, dst, val: a },
        il::Instr::Sext { typ: il::Type::W8, dst, val: a },
        il::Instr::Ret { val: a },
        il::Instr::Unreachable,
        il::Instr::Ecall { dst, num: a, a0: b, a1: a, a2: b, a3: a },
        il::Instr::DeviceRead { typ: il::Type::W64, dst, handle: a, offset: b },
        il::Instr::DeviceWrite { typ: il::Type::W8, handle: a, offset: b, value: a },
        il::Instr::Ebreak,
        il::Instr::MemoryFence,
    ];
    for instr in instructions {
        try checkSelectionCapacity(instr);
    }
}

/// Verify exact-fit output and canaries around every shorter code buffer.
unsafe fn checkSelectionCapacity(instr: il::Instr) throws (testing::TestError) {
    let mut body = [instr, il::Instr::Ret { val: il::Val::Imm(0) }];
    let mut count: u32 = 2;
    if let case il::Instr::Ret { .. } = instr {
        set count = 1;
    }
    let params = [
        il::Param { value: il::Reg { n: 0 }, type: il::Type::W64 },
        il::Param { value: il::Reg { n: 1 }, type: il::Type::W64 },
    ];
    let func = il::Fn {
        name: "p::inline", params: &params[..], returnType: il::Type::W64,
        isExtern: false, isLeaf: not il::isCall(instr),
        blocks: &[il::Block { label: "entry", params: &[], instrs: &mut body[..count], locs: &[], preds: &[], loopDepth: 0 }],
    };
    let mut arena = alloc::new(&mut MEMORY[..]);
    let mut gen = generator(&mut arena);
    let mut scratch = alloc::new(&mut SCRATCH[..]);
    super::generateFunction(&mut gen, &func, &mut scratch);
    assert gen.e.error == nil;
    let length = gen.e.codeLen;
    let mut expected: [u32; 128] = [0; 128];
    assert length > 0 and length <= expected.len;
    for i in 0..length {
        set expected[i] = gen.e.code[i];
    }
    for capacity in 0..(length + 1) {
        let words = &mut SELECTION_WORDS[..];
        for i in 0..words.len {
            set words[i] = 0xdeadbeef;
        }
        alloc::reset(&mut arena);
        set gen = generator(&mut arena);
        set gen.e.code = &mut words[1..capacity + 1];
        super::generateFunction(&mut gen, &func, &mut scratch);
        assert gen.e.codeLen <= capacity;
        assert words[0] == 0xdeadbeef;
        for i in (capacity + 1)..words.len {
            assert words[i] == 0xdeadbeef;
        }
        if capacity == length {
            assert gen.e.error == nil;
            assert gen.e.codeLen == length;
            for i in 0..length {
                assert words[i + 1] == expected[i];
            }
        } else {
            assert gen.e.error == super::Error::Capacity;
        }
    }
}

/// Name the failed invariant before returning to the test runner.
fn check(condition: bool, name: *[u8]) throws (testing::TestError) {
    if not condition {
        io::printLn(name);
        throw testing::TestError::Failed;

lib/std/arch/rv64/isel.rad +139 -129

        // Record debug location before emitting machine instructions.
        if hasLocs {
            emit::recordSrcLoc(s.e, block.locs[i]);
        }
        set s.pendingSpill = nil;
        selectInstr(s, blockIdx, instr, frame, func);
        selectInstr(s, blockIdx, &instr, frame, func);

        // Flush the pending spill store, if any.
        if let p = s.pendingSpill {
            if let slot = regalloc::spill::spillSlot(&s.ralloc.spill, p.ssa) {
                emit::emitSd(s.e, p.rd, spillBase(s), spillOffset(s, slot));

        }
    }
}

/// Select instructions for a single IL instruction.
unsafe fn selectInstr 'scratch 'selection (s: &mut Selector 'scratch 'selection, blockIdx: u32, instr: il::Instr, frame: &emit::Frame, func: &il::Fn) where 'scratch: 'selection {
    match instr {
        case il::Instr::BinOp { op, typ, dst, a, b } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs1 = resolveVal(s, super::SCRATCH1, a);
            selectAluBinOp(s, op, typ, rd, rs1, b);
        },
        case il::Instr::UnOp { op, typ, dst, a } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs = resolveVal(s, super::SCRATCH1, a);
            selectAluUnOp(s, op, typ, rd, rs);
        },
        case il::Instr::Load { typ, dst, src, offset } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let base = getSrcReg(s, src, super::SCRATCH2);
            emit::emitLoad(s.e, rd, base, offset, typ);
        },
        case il::Instr::Sload { typ, dst, src, offset } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let base = getSrcReg(s, src, super::SCRATCH2);
            emit::emitSload(s.e, rd, base, offset, typ);
        },
        case il::Instr::Store { typ, src, dst, offset } => {
            let base = getSrcReg(s, dst, super::SCRATCH2);
            let rs = resolveVal(s, super::SCRATCH1, src);
            emit::emitStore(s.e, rs, base, offset, typ);
        },
        case il::Instr::Copy { dst, val } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs = resolveVal(s, super::SCRATCH1, val);
            emitMv(s, rd, rs);
        },
        case il::Instr::Reserve { dst, size, alignment } => {
            match size {
                case il::Val::Imm(sz) => {
                    // Constant-sized reserve: use pre-allocated frame slot.
                    let rd = getDstReg(s, dst, super::SCRATCH1);
                    let aligned: i32 = mem::alignUpI32(s.reserveOffset, alignment as i32);
                    let base = spillBase(s);
                    let offset = s.ralloc.spill.frameSize + aligned
                        - (s.frameSize if s.isDynamic else 0);

                    emit::emitAddImm(s.e, rd, base, offset);
                    set s.reserveOffset = aligned + (sz as i32);
                },
                case il::Val::Reg(r) => {
                    // Dynamic-sized reserve: runtime SP adjustment.
                    let rd = getDstReg(s, dst, super::SCRATCH1);
                    let rs = getSrcReg(s, r, super::SCRATCH2);

                    emit::emit(s.e, encode::sub(super::SP, super::SP, rs));

                    if alignment > 1 {
                        let mask = 0 - alignment as i32;
                        assert encode::isSmallImm(mask);

                        emit::emit(s.e, encode::andi(super::SP, super::SP, mask));
                    }
                    emit::emit(s.e, encode::mv(rd, super::SP));
                },
                else =>
                    panic "selectInstr: invalid reserve operand",
            }
        },
        case il::Instr::Blit { dst, src, size } => {
            let case il::Val::Imm(staticSize) = size else {
                set s.e.error = super::Error::Capacity; return;
            };
            if staticSize < 0 or staticSize > 0x7fffffff {
                set s.e.error = super::Error::Capacity; return;
            }
            if staticSize == 0 {
                return;
            }
            let rdst = getSrcReg(s, dst, super::SCRATCH2);
            let rsrc = getSrcReg(s, src, super::SCRATCH1);
            // Blit addresses have byte alignment. Small copies need no loop state.
            if staticSize < super::BLIT_LOOP_THRESHOLD as i64 {
                for offset in 0..staticSize as u32 {
                    emit::emitLb(s.e, super::ADDR_SCRATCH, rsrc, offset as i32);
                    emit::emitSb(s.e, super::ADDR_SCRATCH, rdst, offset as i32);
                }
            } else {
                // Private cursors preserve both input pointers. Save the count
                // register because it can hold a live allocated value.
                emit::emit(s.e, encode::addi(super::ADDR_SCRATCH, rsrc, 0));
                emit::emit(s.e, encode::addi(super::SCRATCH2, rdst, 0));
                emit::emit(s.e, encode::addi(super::SP, super::SP, -16));
                emit::emitSd(s.e, super::T3, super::SP, 0);
                emit::loadImm(s.e, super::T3, staticSize);
                let start = s.e.codeLen;
                emit::emitLb(s.e, super::SCRATCH1, super::ADDR_SCRATCH, 0);
                emit::emitSb(s.e, super::SCRATCH1, super::SCRATCH2, 0);
                emit::emit(s.e, encode::addi(super::ADDR_SCRATCH, super::ADDR_SCRATCH, 1));
                emit::emit(s.e, encode::addi(super::SCRATCH2, super::SCRATCH2, 1));
                emit::emit(s.e, encode::addi(super::T3, super::T3, -1));
                let offset = (start as i32 - s.e.codeLen as i32) * super::INSTR_SIZE;
                emit::emit(s.e, encode::bne(super::T3, super::ZERO, offset));
                emit::emitLd(s.e, super::T3, super::SP, 0);
                emit::emit(s.e, encode::addi(super::SP, super::SP, 16));
            }
        },
        case il::Instr::Zext { typ, dst, val } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs = resolveVal(s, super::SCRATCH1, val);
            emitZext(s.e, rd, rs, typ);
        },
        case il::Instr::Sext { typ, dst, val } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs = resolveVal(s, super::SCRATCH1, val);
            emitSext(s.e, rd, rs, typ);
        },
        case il::Instr::Ret { val } => {
            if let v = val {
                let rs = resolveVal(s, super::SCRATCH1, v);
                emitMv(s, super::A0, rs);
            }
            // Skip the jump to epilogue if this RET is in the last block,
            // since the epilogue immediately follows.
            if frame.totalSize <> 0 and blockIdx + 1 == frame.epilogueBlock {
                // Epilogue is the next block; fallthrough is sufficient.
            } else {
                emit::emitReturn(s.e, frame);
            }
        },
unsafe fn selectInstr 'scratch 'selection (s: &mut Selector 'scratch 'selection, blockIdx: u32, instr: &il::Instr, frame: &emit::Frame, func: &il::Fn) where 'scratch: 'selection {
    match *instr {
        case il::Instr::Jmp { target, args } => {
            // Move arguments to target block's parameter registers.
            emitBlockArgs(s, func.blocks[target].params, args);
            // Skip branch if target is the next block (fallthrough).
            if target <> blockIdx + 1 {

            }
            // Fall through to default.
            emitBlockArgs(s, func.blocks[defaultTarget].params, defaultArgs);
            emit::recordBranch(s.e, defaultTarget, emit::BranchKind::Jump);
        },
        case il::Instr::Unreachable => {
            emit::emit(s.e, encode::ebreak());
        },
        case il::Instr::Call { retTy, dst, func, args } => {
            // For indirect calls, save target to scratch register before arg
            // setup can clobber it.
            if let case il::Val::Reg(r) = func {
                let target = getSrcReg(s, r, super::SCRATCH2);

            if let d = dst {
                let rd = getDstReg(s, d, super::SCRATCH1);
                emitMv(s, rd, super::A0);
            }
        },
        else => selectFixedInstr(s, blockIdx, instr, frame),
    }
}

/// Select RV64 instructions for inline IL operands.
fn selectFixedInstr 'scratch 'selection (s: &mut Selector 'scratch 'selection, blockIdx: u32, instr: &il::Instr, frame: &emit::Frame) where 'scratch: 'selection {
    match *instr {
        case il::Instr::BinOp { op, typ, dst, a, b } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs1 = resolveVal(s, super::SCRATCH1, a);
            selectAluBinOp(s, op, typ, rd, rs1, b);
        },
        case il::Instr::UnOp { op, typ, dst, a } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs = resolveVal(s, super::SCRATCH1, a);
            selectAluUnOp(s, op, typ, rd, rs);
        },
        case il::Instr::Load { typ, dst, src, offset } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let base = getSrcReg(s, src, super::SCRATCH2);
            emit::emitLoad(s.e, rd, base, offset, typ);
        },
        case il::Instr::Sload { typ, dst, src, offset } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let base = getSrcReg(s, src, super::SCRATCH2);
            emit::emitSload(s.e, rd, base, offset, typ);
        },
        case il::Instr::Store { typ, src, dst, offset } => {
            let base = getSrcReg(s, dst, super::SCRATCH2);
            let rs = resolveVal(s, super::SCRATCH1, src);
            emit::emitStore(s.e, rs, base, offset, typ);
        },
        case il::Instr::Copy { dst, val } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs = resolveVal(s, super::SCRATCH1, val);
            emitMv(s, rd, rs);
        },
        case il::Instr::Reserve { dst, size, alignment } => {
            match size {
                case il::Val::Imm(sz) => {
                    // Constant-sized reserve: use pre-allocated frame slot.
                    let rd = getDstReg(s, dst, super::SCRATCH1);
                    let aligned: i32 = mem::alignUpI32(s.reserveOffset, alignment as i32);
                    let base = spillBase(s);
                    let offset = s.ralloc.spill.frameSize + aligned
                        - (s.frameSize if s.isDynamic else 0);

                    emit::emitAddImm(s.e, rd, base, offset);
                    set s.reserveOffset = aligned + (sz as i32);
                },
                case il::Val::Reg(r) => {
                    // Dynamic-sized reserve: runtime SP adjustment.
                    let rd = getDstReg(s, dst, super::SCRATCH1);
                    let rs = getSrcReg(s, r, super::SCRATCH2);

                    emit::emit(s.e, encode::sub(super::SP, super::SP, rs));

                    if alignment > 1 {
                        let mask = 0 - alignment as i32;
                        assert encode::isSmallImm(mask);

                        emit::emit(s.e, encode::andi(super::SP, super::SP, mask));
                    }
                    emit::emit(s.e, encode::mv(rd, super::SP));
                },
                else =>
                    panic "selectFixedInstr: invalid reserve operand",
            }
        },
        case il::Instr::Blit { dst, src, size } => {
            let case il::Val::Imm(staticSize) = size else {
                set s.e.error = super::Error::Capacity; return;
            };
            if staticSize < 0 or staticSize > 0x7fffffff {
                set s.e.error = super::Error::Capacity; return;
            }
            if staticSize == 0 {
                return;
            }
            let rdst = getSrcReg(s, dst, super::SCRATCH2);
            let rsrc = getSrcReg(s, src, super::SCRATCH1);
            // Blit addresses have byte alignment. Small copies need no loop state.
            if staticSize < super::BLIT_LOOP_THRESHOLD as i64 {
                for offset in 0..staticSize as u32 {
                    emit::emitLb(s.e, super::ADDR_SCRATCH, rsrc, offset as i32);
                    emit::emitSb(s.e, super::ADDR_SCRATCH, rdst, offset as i32);
                }
            } else {
                // Private cursors preserve both input pointers. Save the count
                // register because it can hold a live allocated value.
                emit::emit(s.e, encode::addi(super::ADDR_SCRATCH, rsrc, 0));
                emit::emit(s.e, encode::addi(super::SCRATCH2, rdst, 0));
                emit::emit(s.e, encode::addi(super::SP, super::SP, -16));
                emit::emitSd(s.e, super::T3, super::SP, 0);
                emit::loadImm(s.e, super::T3, staticSize);
                let start = s.e.codeLen;
                emit::emitLb(s.e, super::SCRATCH1, super::ADDR_SCRATCH, 0);
                emit::emitSb(s.e, super::SCRATCH1, super::SCRATCH2, 0);
                emit::emit(s.e, encode::addi(super::ADDR_SCRATCH, super::ADDR_SCRATCH, 1));
                emit::emit(s.e, encode::addi(super::SCRATCH2, super::SCRATCH2, 1));
                emit::emit(s.e, encode::addi(super::T3, super::T3, -1));
                let offset = (start as i32 - s.e.codeLen as i32) * super::INSTR_SIZE;
                emit::emit(s.e, encode::bne(super::T3, super::ZERO, offset));
                emit::emitLd(s.e, super::T3, super::SP, 0);
                emit::emit(s.e, encode::addi(super::SP, super::SP, 16));
            }
        },
        case il::Instr::Zext { typ, dst, val } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs = resolveVal(s, super::SCRATCH1, val);
            emitZext(s.e, rd, rs, typ);
        },
        case il::Instr::Sext { typ, dst, val } => {
            let rd = getDstReg(s, dst, super::SCRATCH1);
            let rs = resolveVal(s, super::SCRATCH1, val);
            emitSext(s.e, rd, rs, typ);
        },
        case il::Instr::Ret { val } => {
            if let v = val {
                let rs = resolveVal(s, super::SCRATCH1, v);
                emitMv(s, super::A0, rs);
            }
            // Skip the jump to epilogue if this RET is in the last block,
            // since the epilogue immediately follows.
            if frame.totalSize <> 0 and blockIdx + 1 == frame.epilogueBlock {
                // Epilogue is the next block; fallthrough is sufficient.
            } else {
                emit::emitReturn(s.e, frame);
            }
        },
        case il::Instr::Unreachable => {
            emit::emit(s.e, encode::ebreak());
        },
        case il::Instr::Ecall { dst, num, a0, a1, a2, a3 } => {
            // Move arguments using parallel move.
            // TODO: Can't use slice literals here because the lowerer doesn't
            // support constant-evaluating struct/union values in them.
            let ecallDsts: [gen::Reg; 5] = [super::A7, super::A0, super::A1, super::A2, super::A3];

            emit::emit(s.e, encode::ebreak());
        },
        case il::Instr::MemoryFence => {
            emit::emit(s.e, encode::fence());
        },
        case il::Instr::Call { .. }, il::Instr::Jmp { .. },
             il::Instr::Br { .. }, il::Instr::Switch { .. } =>
            panic "selectFixedInstr: expected inline operands",
    }
}

/// Choose the cheapest canonical representation that preserves the comparison.
/// RV64 word operations naturally sign-extend, and sign-extension preserves

196	198		throw testing::TestError::Failed;
197	199		};
198	200		try testing::expect(length == 8 and bytes[0] == 1);
199	201		}
200	202
	203	+	/// Inline instruction selection respects every shorter output capacity.
	204	+	@test unsafe fn inlineSelectionCapacity() throws (testing::TestError) {
	205	+	let dst = il::Reg { n: 3 };
	206	+	let first = il::Reg { n: 0 };
	207	+	let second = il::Reg { n: 1 };
	208	+	let a = il::Val::Reg(first);
	209	+	let b = il::Val::Reg(second);
	210	+	let instructions = [
	211	+	il::Instr::BinOp { op: il::BinOp::Add, typ: il::Type::W64, dst, a, b },
	212	+	il::Instr::UnOp { op: il::UnOp::Neg, typ: il::Type::W32, dst, a },
	213	+	il::Instr::Load { typ: il::Type::W8, dst, src: first, offset: 4096 },
	214	+	il::Instr::Sload { typ: il::Type::W16, dst, src: first, offset: -4096 },
	215	+	il::Instr::Store { typ: il::Type::W32, src: a, dst: second, offset: 4096 },
	216	+	il::Instr::Copy { dst, val: il::Val::Imm(0x123456789abcdef) },
	217	+	il::Instr::Reserve { dst, size: il::Val::Imm(32), alignment: 16 },
	218	+	il::Instr::Reserve { dst, size: a, alignment: 16 },
	219	+	il::Instr::Blit { dst: first, src: second, size: il::Val::Imm(0) },
	220	+	il::Instr::Blit { dst: first, src: second, size: il::Val::Imm(8) },
	221	+	il::Instr::Blit { dst: first, src: second, size: il::Val::Imm(40) },
	222	+	il::Instr::Zext { typ: il::Type::W16, dst, val: a },
	223	+	il::Instr::Sext { typ: il::Type::W8, dst, val: a },
	224	+	il::Instr::Ret { val: a },
	225	+	il::Instr::Unreachable,
	226	+	il::Instr::Ecall { dst, num: a, a0: b, a1: a, a2: b, a3: a },
	227	+	il::Instr::DeviceRead { typ: il::Type::W64, dst, handle: a, offset: b },
	228	+	il::Instr::DeviceWrite { typ: il::Type::W8, handle: a, offset: b, value: a },
	229	+	il::Instr::Ebreak,
	230	+	il::Instr::MemoryFence,
	231	+	];
	232	+	for instr in instructions {
	233	+	try checkSelectionCapacity(instr);
	234	+	}
	235	+	}
	236	+
	237	+	/// Verify exact-fit output and canaries around every shorter code buffer.
	238	+	unsafe fn checkSelectionCapacity(instr: il::Instr) throws (testing::TestError) {
	239	+	let mut body = [instr, il::Instr::Ret { val: il::Val::Imm(0) }];
	240	+	let mut count: u32 = 2;
	241	+	if let case il::Instr::Ret { .. } = instr {
	242	+	set count = 1;
	243	+	}
	244	+	let params = [
	245	+	il::Param { value: il::Reg { n: 0 }, type: il::Type::W64 },
	246	+	il::Param { value: il::Reg { n: 1 }, type: il::Type::W64 },
	247	+	];
	248	+	let func = il::Fn {
	249	+	name: "p::inline", params: &params[..], returnType: il::Type::W64,
	250	+	isExtern: false, isLeaf: not il::isCall(instr),
	251	+	blocks: &[il::Block { label: "entry", params: &[], instrs: &mut body[..count], locs: &[], preds: &[], loopDepth: 0 }],
	252	+	};
	253	+	let mut arena = alloc::new(&mut MEMORY[..]);
	254	+	let mut gen = generator(&mut arena);
	255	+	let mut scratch = alloc::new(&mut SCRATCH[..]);
	256	+	super::generateFunction(&mut gen, &func, &mut scratch);
	257	+	assert gen.e.error == nil;
	258	+	let length = gen.e.codeLen;
	259	+	let mut expected: [u32; 128] = [0; 128];
	260	+	assert length > 0 and length <= expected.len;
	261	+	for i in 0..length {
	262	+	set expected[i] = gen.e.code[i];
	263	+	}
	264	+	for capacity in 0..(length + 1) {
	265	+	let words = &mut SELECTION_WORDS[..];
	266	+	for i in 0..words.len {
	267	+	set words[i] = 0xdeadbeef;
	268	+	}
	269	+	alloc::reset(&mut arena);
	270	+	set gen = generator(&mut arena);
	271	+	set gen.e.code = &mut words[1..capacity + 1];
	272	+	super::generateFunction(&mut gen, &func, &mut scratch);
	273	+	assert gen.e.codeLen <= capacity;
	274	+	assert words[0] == 0xdeadbeef;
	275	+	for i in (capacity + 1)..words.len {
	276	+	assert words[i] == 0xdeadbeef;
	277	+	}
	278	+	if capacity == length {
	279	+	assert gen.e.error == nil;
	280	+	assert gen.e.codeLen == length;
	281	+	for i in 0..length {
	282	+	assert words[i + 1] == expected[i];
	283	+	}
	284	+	} else {
	285	+	assert gen.e.error == super::Error::Capacity;
	286	+	}
	287	+	}
	288	+	}
	289	+
201	290		/// Name the failed invariant before returning to the test runner.
202	291		fn check(condition: bool, name: *[u8]) throws (testing::TestError) {
203	292		if not condition {
204	293		io::printLn(name);
205	294		throw testing::TestError::Failed;

398	398		// Record debug location before emitting machine instructions.
399	399		if hasLocs {
400	400		emit::recordSrcLoc(s.e, block.locs[i]);
401	401		}
402	402		set s.pendingSpill = nil;
403		-	selectInstr(s, blockIdx, instr, frame, func);
	403	+	selectInstr(s, blockIdx, &instr, frame, func);
404	404
405	405		// Flush the pending spill store, if any.
406	406		if let p = s.pendingSpill {
407	407		if let slot = regalloc::spill::spillSlot(&s.ralloc.spill, p.ssa) {
408	408		emit::emitSd(s.e, p.rd, spillBase(s), spillOffset(s, slot));

411	411		}
412	412		}
413	413		}
414	414
415	415		/// Select instructions for a single IL instruction.
416		-	unsafe fn selectInstr 'scratch 'selection (s: &mut Selector 'scratch 'selection, blockIdx: u32, instr: il::Instr, frame: &emit::Frame, func: &il::Fn) where 'scratch: 'selection {
417		-	match instr {
418		-	case il::Instr::BinOp { op, typ, dst, a, b } => {
419		-	let rd = getDstReg(s, dst, super::SCRATCH1);
420		-	let rs1 = resolveVal(s, super::SCRATCH1, a);
421		-	selectAluBinOp(s, op, typ, rd, rs1, b);
422		-	},
423		-	case il::Instr::UnOp { op, typ, dst, a } => {
424		-	let rd = getDstReg(s, dst, super::SCRATCH1);
425		-	let rs = resolveVal(s, super::SCRATCH1, a);
426		-	selectAluUnOp(s, op, typ, rd, rs);
427		-	},
428		-	case il::Instr::Load { typ, dst, src, offset } => {
429		-	let rd = getDstReg(s, dst, super::SCRATCH1);
430		-	let base = getSrcReg(s, src, super::SCRATCH2);
431		-	emit::emitLoad(s.e, rd, base, offset, typ);
432		-	},
433		-	case il::Instr::Sload { typ, dst, src, offset } => {
434		-	let rd = getDstReg(s, dst, super::SCRATCH1);
435		-	let base = getSrcReg(s, src, super::SCRATCH2);
436		-	emit::emitSload(s.e, rd, base, offset, typ);
437		-	},
438		-	case il::Instr::Store { typ, src, dst, offset } => {
439		-	let base = getSrcReg(s, dst, super::SCRATCH2);
440		-	let rs = resolveVal(s, super::SCRATCH1, src);
441		-	emit::emitStore(s.e, rs, base, offset, typ);
442		-	},
443		-	case il::Instr::Copy { dst, val } => {
444		-	let rd = getDstReg(s, dst, super::SCRATCH1);
445		-	let rs = resolveVal(s, super::SCRATCH1, val);
446		-	emitMv(s, rd, rs);
447		-	},
448		-	case il::Instr::Reserve { dst, size, alignment } => {
449		-	match size {
450		-	case il::Val::Imm(sz) => {
451		-	// Constant-sized reserve: use pre-allocated frame slot.
452		-	let rd = getDstReg(s, dst, super::SCRATCH1);
453		-	let aligned: i32 = mem::alignUpI32(s.reserveOffset, alignment as i32);
454		-	let base = spillBase(s);
455		-	let offset = s.ralloc.spill.frameSize + aligned
456		-	- (s.frameSize if s.isDynamic else 0);
457		-
458		-	emit::emitAddImm(s.e, rd, base, offset);
459		-	set s.reserveOffset = aligned + (sz as i32);
460		-	},
461		-	case il::Val::Reg(r) => {
462		-	// Dynamic-sized reserve: runtime SP adjustment.
463		-	let rd = getDstReg(s, dst, super::SCRATCH1);
464		-	let rs = getSrcReg(s, r, super::SCRATCH2);
465		-
466		-	emit::emit(s.e, encode::sub(super::SP, super::SP, rs));
467		-
468		-	if alignment > 1 {
469		-	let mask = 0 - alignment as i32;
470		-	assert encode::isSmallImm(mask);
471		-
472		-	emit::emit(s.e, encode::andi(super::SP, super::SP, mask));
473		-	}
474		-	emit::emit(s.e, encode::mv(rd, super::SP));
475		-	},
476		-	else =>
477		-	panic "selectInstr: invalid reserve operand",
478		-	}
479		-	},
480		-	case il::Instr::Blit { dst, src, size } => {
481		-	let case il::Val::Imm(staticSize) = size else {
482		-	set s.e.error = super::Error::Capacity; return;
483		-	};
484		-	if staticSize < 0 or staticSize > 0x7fffffff {
485		-	set s.e.error = super::Error::Capacity; return;
486		-	}
487		-	if staticSize == 0 {
488		-	return;
489		-	}
490		-	let rdst = getSrcReg(s, dst, super::SCRATCH2);
491		-	let rsrc = getSrcReg(s, src, super::SCRATCH1);
492		-	// Blit addresses have byte alignment. Small copies need no loop state.
493		-	if staticSize < super::BLIT_LOOP_THRESHOLD as i64 {
494		-	for offset in 0..staticSize as u32 {
495		-	emit::emitLb(s.e, super::ADDR_SCRATCH, rsrc, offset as i32);
496		-	emit::emitSb(s.e, super::ADDR_SCRATCH, rdst, offset as i32);
497		-	}
498		-	} else {
499		-	// Private cursors preserve both input pointers. Save the count
500		-	// register because it can hold a live allocated value.
501		-	emit::emit(s.e, encode::addi(super::ADDR_SCRATCH, rsrc, 0));
502		-	emit::emit(s.e, encode::addi(super::SCRATCH2, rdst, 0));
503		-	emit::emit(s.e, encode::addi(super::SP, super::SP, -16));
504		-	emit::emitSd(s.e, super::T3, super::SP, 0);
505		-	emit::loadImm(s.e, super::T3, staticSize);
506		-	let start = s.e.codeLen;
507		-	emit::emitLb(s.e, super::SCRATCH1, super::ADDR_SCRATCH, 0);
508		-	emit::emitSb(s.e, super::SCRATCH1, super::SCRATCH2, 0);
509		-	emit::emit(s.e, encode::addi(super::ADDR_SCRATCH, super::ADDR_SCRATCH, 1));
510		-	emit::emit(s.e, encode::addi(super::SCRATCH2, super::SCRATCH2, 1));
511		-	emit::emit(s.e, encode::addi(super::T3, super::T3, -1));
512		-	let offset = (start as i32 - s.e.codeLen as i32) * super::INSTR_SIZE;
513		-	emit::emit(s.e, encode::bne(super::T3, super::ZERO, offset));
514		-	emit::emitLd(s.e, super::T3, super::SP, 0);
515		-	emit::emit(s.e, encode::addi(super::SP, super::SP, 16));
516		-	}
517		-	},
518		-	case il::Instr::Zext { typ, dst, val } => {
519		-	let rd = getDstReg(s, dst, super::SCRATCH1);
520		-	let rs = resolveVal(s, super::SCRATCH1, val);
521		-	emitZext(s.e, rd, rs, typ);
522		-	},
523		-	case il::Instr::Sext { typ, dst, val } => {
524		-	let rd = getDstReg(s, dst, super::SCRATCH1);
525		-	let rs = resolveVal(s, super::SCRATCH1, val);
526		-	emitSext(s.e, rd, rs, typ);
527		-	},
528		-	case il::Instr::Ret { val } => {
529		-	if let v = val {
530		-	let rs = resolveVal(s, super::SCRATCH1, v);
531		-	emitMv(s, super::A0, rs);
532		-	}
533		-	// Skip the jump to epilogue if this RET is in the last block,
534		-	// since the epilogue immediately follows.
535		-	if frame.totalSize <> 0 and blockIdx + 1 == frame.epilogueBlock {
536		-	// Epilogue is the next block; fallthrough is sufficient.
537		-	} else {
538		-	emit::emitReturn(s.e, frame);
539		-	}
540		-	},
	416	+	unsafe fn selectInstr 'scratch 'selection (s: &mut Selector 'scratch 'selection, blockIdx: u32, instr: &il::Instr, frame: &emit::Frame, func: &il::Fn) where 'scratch: 'selection {
	417	+	match *instr {
541	418		case il::Instr::Jmp { target, args } => {
542	419		// Move arguments to target block's parameter registers.
543	420		emitBlockArgs(s, func.blocks[target].params, args);
544	421		// Skip branch if target is the next block (fallthrough).
545	422		if target <> blockIdx + 1 {

630	507		}
631	508		// Fall through to default.
632	509		emitBlockArgs(s, func.blocks[defaultTarget].params, defaultArgs);
633	510		emit::recordBranch(s.e, defaultTarget, emit::BranchKind::Jump);
634	511		},
635		-	case il::Instr::Unreachable => {
636		-	emit::emit(s.e, encode::ebreak());
637		-	},
638	512		case il::Instr::Call { retTy, dst, func, args } => {
639	513		// For indirect calls, save target to scratch register before arg
640	514		// setup can clobber it.
641	515		if let case il::Val::Reg(r) = func {
642	516		let target = getSrcReg(s, r, super::SCRATCH2);

665	539		if let d = dst {
666	540		let rd = getDstReg(s, d, super::SCRATCH1);
667	541		emitMv(s, rd, super::A0);
668	542		}
669	543		},
	544	+	else => selectFixedInstr(s, blockIdx, instr, frame),
	545	+	}
	546	+	}
	547	+
	548	+	/// Select RV64 instructions for inline IL operands.
	549	+	fn selectFixedInstr 'scratch 'selection (s: &mut Selector 'scratch 'selection, blockIdx: u32, instr: &il::Instr, frame: &emit::Frame) where 'scratch: 'selection {
	550	+	match *instr {
	551	+	case il::Instr::BinOp { op, typ, dst, a, b } => {
	552	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	553	+	let rs1 = resolveVal(s, super::SCRATCH1, a);
	554	+	selectAluBinOp(s, op, typ, rd, rs1, b);
	555	+	},
	556	+	case il::Instr::UnOp { op, typ, dst, a } => {
	557	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	558	+	let rs = resolveVal(s, super::SCRATCH1, a);
	559	+	selectAluUnOp(s, op, typ, rd, rs);
	560	+	},
	561	+	case il::Instr::Load { typ, dst, src, offset } => {
	562	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	563	+	let base = getSrcReg(s, src, super::SCRATCH2);
	564	+	emit::emitLoad(s.e, rd, base, offset, typ);
	565	+	},
	566	+	case il::Instr::Sload { typ, dst, src, offset } => {
	567	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	568	+	let base = getSrcReg(s, src, super::SCRATCH2);
	569	+	emit::emitSload(s.e, rd, base, offset, typ);
	570	+	},
	571	+	case il::Instr::Store { typ, src, dst, offset } => {
	572	+	let base = getSrcReg(s, dst, super::SCRATCH2);
	573	+	let rs = resolveVal(s, super::SCRATCH1, src);
	574	+	emit::emitStore(s.e, rs, base, offset, typ);
	575	+	},
	576	+	case il::Instr::Copy { dst, val } => {
	577	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	578	+	let rs = resolveVal(s, super::SCRATCH1, val);
	579	+	emitMv(s, rd, rs);
	580	+	},
	581	+	case il::Instr::Reserve { dst, size, alignment } => {
	582	+	match size {
	583	+	case il::Val::Imm(sz) => {
	584	+	// Constant-sized reserve: use pre-allocated frame slot.
	585	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	586	+	let aligned: i32 = mem::alignUpI32(s.reserveOffset, alignment as i32);
	587	+	let base = spillBase(s);
	588	+	let offset = s.ralloc.spill.frameSize + aligned
	589	+	- (s.frameSize if s.isDynamic else 0);
	590	+
	591	+	emit::emitAddImm(s.e, rd, base, offset);
	592	+	set s.reserveOffset = aligned + (sz as i32);
	593	+	},
	594	+	case il::Val::Reg(r) => {
	595	+	// Dynamic-sized reserve: runtime SP adjustment.
	596	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	597	+	let rs = getSrcReg(s, r, super::SCRATCH2);
	598	+
	599	+	emit::emit(s.e, encode::sub(super::SP, super::SP, rs));
	600	+
	601	+	if alignment > 1 {
	602	+	let mask = 0 - alignment as i32;
	603	+	assert encode::isSmallImm(mask);
	604	+
	605	+	emit::emit(s.e, encode::andi(super::SP, super::SP, mask));
	606	+	}
	607	+	emit::emit(s.e, encode::mv(rd, super::SP));
	608	+	},
	609	+	else =>
	610	+	panic "selectFixedInstr: invalid reserve operand",
	611	+	}
	612	+	},
	613	+	case il::Instr::Blit { dst, src, size } => {
	614	+	let case il::Val::Imm(staticSize) = size else {
	615	+	set s.e.error = super::Error::Capacity; return;
	616	+	};
	617	+	if staticSize < 0 or staticSize > 0x7fffffff {
	618	+	set s.e.error = super::Error::Capacity; return;
	619	+	}
	620	+	if staticSize == 0 {
	621	+	return;
	622	+	}
	623	+	let rdst = getSrcReg(s, dst, super::SCRATCH2);
	624	+	let rsrc = getSrcReg(s, src, super::SCRATCH1);
	625	+	// Blit addresses have byte alignment. Small copies need no loop state.
	626	+	if staticSize < super::BLIT_LOOP_THRESHOLD as i64 {
	627	+	for offset in 0..staticSize as u32 {
	628	+	emit::emitLb(s.e, super::ADDR_SCRATCH, rsrc, offset as i32);
	629	+	emit::emitSb(s.e, super::ADDR_SCRATCH, rdst, offset as i32);
	630	+	}
	631	+	} else {
	632	+	// Private cursors preserve both input pointers. Save the count
	633	+	// register because it can hold a live allocated value.
	634	+	emit::emit(s.e, encode::addi(super::ADDR_SCRATCH, rsrc, 0));
	635	+	emit::emit(s.e, encode::addi(super::SCRATCH2, rdst, 0));
	636	+	emit::emit(s.e, encode::addi(super::SP, super::SP, -16));
	637	+	emit::emitSd(s.e, super::T3, super::SP, 0);
	638	+	emit::loadImm(s.e, super::T3, staticSize);
	639	+	let start = s.e.codeLen;
	640	+	emit::emitLb(s.e, super::SCRATCH1, super::ADDR_SCRATCH, 0);
	641	+	emit::emitSb(s.e, super::SCRATCH1, super::SCRATCH2, 0);
	642	+	emit::emit(s.e, encode::addi(super::ADDR_SCRATCH, super::ADDR_SCRATCH, 1));
	643	+	emit::emit(s.e, encode::addi(super::SCRATCH2, super::SCRATCH2, 1));
	644	+	emit::emit(s.e, encode::addi(super::T3, super::T3, -1));
	645	+	let offset = (start as i32 - s.e.codeLen as i32) * super::INSTR_SIZE;
	646	+	emit::emit(s.e, encode::bne(super::T3, super::ZERO, offset));
	647	+	emit::emitLd(s.e, super::T3, super::SP, 0);
	648	+	emit::emit(s.e, encode::addi(super::SP, super::SP, 16));
	649	+	}
	650	+	},
	651	+	case il::Instr::Zext { typ, dst, val } => {
	652	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	653	+	let rs = resolveVal(s, super::SCRATCH1, val);
	654	+	emitZext(s.e, rd, rs, typ);
	655	+	},
	656	+	case il::Instr::Sext { typ, dst, val } => {
	657	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	658	+	let rs = resolveVal(s, super::SCRATCH1, val);
	659	+	emitSext(s.e, rd, rs, typ);
	660	+	},
	661	+	case il::Instr::Ret { val } => {
	662	+	if let v = val {
	663	+	let rs = resolveVal(s, super::SCRATCH1, v);
	664	+	emitMv(s, super::A0, rs);
	665	+	}
	666	+	// Skip the jump to epilogue if this RET is in the last block,
	667	+	// since the epilogue immediately follows.
	668	+	if frame.totalSize <> 0 and blockIdx + 1 == frame.epilogueBlock {
	669	+	// Epilogue is the next block; fallthrough is sufficient.
	670	+	} else {
	671	+	emit::emitReturn(s.e, frame);
	672	+	}
	673	+	},
	674	+	case il::Instr::Unreachable => {
	675	+	emit::emit(s.e, encode::ebreak());
	676	+	},
670	677		case il::Instr::Ecall { dst, num, a0, a1, a2, a3 } => {
671	678		// Move arguments using parallel move.
672	679		// TODO: Can't use slice literals here because the lowerer doesn't
673	680		// support constant-evaluating struct/union values in them.
674	681		let ecallDsts: [gen::Reg; 5] = [super::A7, super::A0, super::A1, super::A2, super::A3];

14	14
15	15		/// Reusable emitter allocation storage.
16	16		static MEMORY: [u8; 16777216] = [0; 16777216];
17	17		/// Function and liveness test storage.
18	18		static SCRATCH: [u8; 65536] = [0; 65536];
	19	+	/// Code output with guard words for instruction selection checks.
	20	+	static SELECTION_WORDS: [u32; 130] = [0; 130];
19	21
20	22		/// Dictionary storage for bounded map tests.
21	23		unsafe static ENTRIES: [dict::Entry; 4] = undefined;
22	24
23	25		/// Build a non-debug generator with a fixed code address.

706	713		emit::emit(s.e, encode::ebreak());
707	714		},
708	715		case il::Instr::MemoryFence => {
709	716		emit::emit(s.e, encode::fence());
710	717		},
	718	+	case il::Instr::Call { .. }, il::Instr::Jmp { .. },
	719	+	il::Instr::Br { .. }, il::Instr::Switch { .. } =>
	720	+	panic "selectFixedInstr: expected inline operands",
711	721		}
712	722		}
713	723
714	724		/// Choose the cheapest canonical representation that preserves the comparison.
715	725		/// RV64 word operations naturally sign-extend, and sign-extension preserves