radiance.s0 · Radiance bootstrapping compiler

gen/ data.c 10.9 KiB data.h 2.5 KiB emit.c 37.2 KiB emit.h 4.6 KiB .clang-format 570 B .gitignore 30 B .gitsigners 112 B LICENSE 1.1 KiB Makefile 911 B README 1.8 KiB ast.c 5.0 KiB ast.h 15.1 KiB desugar.c 23.1 KiB desugar.h 286 B gen.c 108.5 KiB gen.h 4.9 KiB io.c 1.1 KiB io.h 444 B limits.h 1.3 KiB module.c 10.0 KiB module.h 2.2 KiB options.c 1.4 KiB options.h 472 B parser.c 68.3 KiB parser.h 942 B radiance.c 3.7 KiB ralloc.c 2.0 KiB ralloc.h 1.1 KiB resolver.c 109.7 KiB resolver.h 5.6 KiB riscv.c 12.0 KiB riscv.h 12.0 KiB scanner.c 10.2 KiB scanner.h 3.2 KiB strings.c 2.6 KiB strings.h 407 B symtab.c 5.7 KiB symtab.h 4.6 KiB types.h 1.0 KiB util.h 1.5 KiB
gen/emit.c 37.2 KiB raw
#include <stdio.h>
#include <stdlib.h>

#include "emit.h"

void split_imm(i32 imm, i32 *hi, i32 *lo) {
    /* Split immediate into upper 20 bits and lower 12 bits */
    *hi = ((imm + 0x800) >> 12) & 0xFFFFF; /* Add 0x800 for proper rounding */
    *lo = imm & 0xFFF;
    if (*lo & 0x800) {
        /* If the highest bit of the lower 12 bits is set,
           it will be sign-extended, so adjust upper part */
        *lo = *lo | ~0xFFF; /* Sign extend lower */
    }
}

void emit_li(gen_t *g, reg_t rd, i32 imm) {
    if (is_small(imm)) {
        emit(g, instr(I_ADDI, rd, 0, 0, imm));
        return;
    }
    i32 hi, lo;
    split_imm(imm, &hi, &lo);

    emit(g, instr(I_LUI, rd, 0, 0, hi));
    /* Use ADDIW to sign-extend the 32-bit result on RV64, otherwise
     * LUI's upper-bit sign-extension leaves garbage in bits 63:32. */
    if (lo != 0) {
        emit(g, instr(I_ADDIW, rd, rd, 0, lo));
    } else {
        /* Even with lo == 0, LUI sign-extends bit 31 into 63:32.
         * Use SEXT.W (ADDIW rd, rd, 0) to canonicalize. */
        emit(g, instr(I_ADDIW, rd, rd, 0, 0));
    }
}

void emit_mv(gen_t *g, reg_t dst, reg_t src) {
    if (dst != src) {
        emit(g, instr(I_MV, dst, src, 0, 0));
    }
}

usize emit_jump(gen_t *g, usize offset) {
    return emit(g, JMP(jump_offset(g->ninstrs, offset)));
}

/* Compute hi/lo split for PC-relative offset to target address. */
static void pc_rel_offset(gen_t *g, usize addr, i32 *hi, i32 *lo) {
    i32 target_addr  = (i32)(addr * INSTR_SIZE);
    i32 current_addr = (i32)(g->ninstrs * INSTR_SIZE);
    i32 offset       = target_addr - current_addr;
    split_imm(offset, hi, lo);
}

void emit_pc_rel_addr(gen_t *g, reg_t rd, usize addr) {
    i32 hi, lo;
    pc_rel_offset(g, addr, &hi, &lo);
    emit(g, AUIPC(rd, hi));
    emit(g, ADDI(rd, rd, lo));
}

static usize emit_call_far(gen_t *g, usize addr, reg_t scratch) {
    i32 hi, lo;
    pc_rel_offset(g, addr, &hi, &lo);
    usize pc = emit(g, AUIPC(scratch, hi));
    emit(g, JALR(RA, scratch, lo));
    return pc;
}

usize emit_call(gen_t *g, usize addr) {
    i32 offset = jump_offset(g->ninstrs, addr);
    if (is_jump_imm(offset))
        return emit(g, JAL(RA, offset));

    reg_t scratch = nextreg(g);
    usize pc      = emit_call_far(g, addr, scratch);
    freereg(g, scratch);
    return pc;
}

void emit_record_copy(gen_t *g, offset_t src, offset_t dst, type_t *ty) {
    for (usize i = 0; i < ty->info.srt.nfields; i++) {
        symbol_t *field     = ty->info.srt.fields[i];
        type_t   *field_typ = field->e.field.typ;
        i32       field_off = field->e.field.offset;
        offset_t  field_src = OFFSET(src.base, src.offset + field_off);
        offset_t  field_dst = OFFSET(dst.base, dst.offset + field_off);

        emit_memcopy(g, field_src, field_dst, field_typ);
    }
}

static value_t emit_field_get(value_t sval, i32 off, type_t *typ) {
    switch (sval.loc) {
    case LOC_REG:
        return value_stack(OFFSET(sval.as.reg, off), typ);
    case LOC_STACK:
        return value_stack(
            OFFSET(sval.as.off.base, sval.as.off.offset + off), typ
        );
    case LOC_ADDR:
        return value_addr(sval.as.adr.base, sval.as.adr.offset + off, typ);
    case LOC_NONE:
    case LOC_IMM:
        break;
    }
    abort();
}

/* RISC-V load/store immediates are limited to signed 12 bits. This helper folds
 * large displacements into a temporary register so the generated instruction
 * still uses the small-immediate forms, keeping the addressing logic in callers
 * simple. */
static addr_adj_t adjust_addr_avoid(
    gen_t *g, reg_t base, i32 *offset, reg_t avoid
) {
    if (is_small(*offset))
        return (addr_adj_t){ base, false };

    reg_t tmp = avoid ? nextreg_except(g, avoid) : nextreg(g);

    emit_li(g, tmp, *offset);
    emit(g, ADD(tmp, base, tmp));
    *offset = 0;

    return (addr_adj_t){ tmp, true };
}

static addr_adj_t adjust_addr(gen_t *g, reg_t base, i32 *offset) {
    return adjust_addr_avoid(g, base, offset, 0);
}

/* Release any temporary register created by `adjust_addr`. */
static void release_addr(gen_t *g, addr_adj_t adj) {
    if (adj.temp)
        freereg(g, adj.base);
}

void emit_addr_offset(gen_t *g, reg_t dst, reg_t base, i32 offset) {
    if (is_small(offset)) {
        emit(g, ADDI(dst, base, offset));
        return;
    }
    reg_t tmp = nextreg(g);
    emit_li(g, tmp, offset);
    emit(g, ADD(dst, base, tmp));
    freereg(g, tmp);
}

value_t emit_slice_lit(
    gen_t *g, i32 offset, usize ptr, usize len, type_t *typ
) {
    static type_t ptr_type = { .cls = TYPE_PTR };
    imm_t         imm_ptr  = (imm_t){ .u = ptr }; /* Slice pointer */
    imm_t         imm_len  = (imm_t){ .u = len }; /* Slice length */

    emit_store(
        g, value_imm(imm_ptr, &ptr_type), FP, offset + SLICE_FIELD_PTR_OFFSET
    );
    emit_store(
        g, value_imm(imm_len, &ptr_type), FP, offset + SLICE_FIELD_LEN_OFFSET
    );
    return value_stack(OFFSET(FP, offset), typ);
}

value_t emit_record_field_get(value_t sval, symbol_t *field) {
    i32     foff  = field->e.field.offset;
    type_t *ftype = field->node->type;

    return emit_field_get(sval, foff, ftype);
}

void emit_record_field_set(
    gen_t *g, value_t val, reg_t base, i32 record_offset, symbol_t *field
) {
    i32 field_offset  = field->e.field.offset;
    i32 target_offset = record_offset + field_offset;

    value_t dest = value_stack(OFFSET(base, target_offset), field->e.field.typ);

    emit_replace(g, dest, val);
}

void emit_memzero(gen_t *g, offset_t dst, i32 size) {
    if (size == 0) /* Nothing to do for zero-sized regions */
        return;

    reg_t cursor = nextreg(g);
    emit_addr_offset(g, cursor, dst.base, dst.offset);

    /* Calculate word-aligned size and remainder */
    i32 aligned_size = align_stack(size, WORD_SIZE);
    i32 remainder    = size - aligned_size;

    reg_t end = ZERO;

    /* Only use the word-based loop if we have at least one complete word */
    if (aligned_size > 0) {
        end = nextreg(g);
        emit_addr_offset(g, end, cursor, aligned_size);

        usize loop_start = g->ninstrs;
        usize branch_end = emit(g, NOP);

        /* Store zero to current address and increment by word size */
        emit(g, SD(ZERO, cursor, 0));
        emit(g, ADDI(cursor, cursor, WORD_SIZE));
        emit(g, JMP(jump_offset(g->ninstrs, loop_start))); /* Loop back */

        g->instrs[branch_end] =
            BGE(cursor, end, jump_offset(branch_end, g->ninstrs));
    }

    /* At least four bytes left */
    if (remainder >= 4) { /* Store a word (4 bytes) */
        emit(g, SW(ZERO, cursor, 0));
        emit(g, ADDI(cursor, cursor, 4));
        remainder -= 4;
    }
    /* At least two bytes left */
    if (remainder >= 2) { /* Store a halfword (2 bytes) */
        emit(g, SH(ZERO, cursor, 0));
        emit(g, ADDI(cursor, cursor, 2));
        remainder -= 2;
    }
    /* One byte left */
    if (remainder == 1) {
        emit(g, SB(ZERO, cursor, 0));
    }
    freereg(g, cursor);
    if (aligned_size > 0)
        freereg(g, end);
}

void emit_replace(gen_t *g, value_t old, value_t new) {
    if (old.type->cls == TYPE_OPT) {
        if (new.type->cls == TYPE_OPT) {
            switch (old.loc) {
            case LOC_STACK:
                emit_memcopy(g, new.as.off, old.as.off, old.type);
                break;
            case LOC_ADDR: {
                /* Handle assignment to LOC_ADDR optional */
                reg_t base = nextreg(g);
                emit_li(g, base, old.as.adr.base);
                emit_store(g, new, base, old.as.adr.offset);
                freereg(g, base);
                break;
            }
            default:
                bail(
                    "can't replace tagged value with storage location %d",
                    old.loc
                );
            }
        } else if (new.type->cls == old.type->info.opt.elem->cls) {
            /* T -> ?T coercion: create some value */
            tval_store(g, old, new, 1);
        } else {
            bail(
                "cannot assign %s to %s; type mismatch",
                type_names[new.type->cls],
                type_names[old.type->cls]
            );
        }
    } else if (old.type->cls == TYPE_RESULT) {
        type_t *payload = old.type->info.res.payload;
        type_t *err     = old.type->info.res.err;

        if (new.type->cls == TYPE_RESULT) {
            switch (old.loc) {
            case LOC_STACK:
                emit_memcopy(g, new.as.off, old.as.off, old.type);
                break;
            case LOC_ADDR: {
                /* Handle assignment to LOC_ADDR result */
                reg_t base = nextreg(g);
                emit_li(g, base, old.as.adr.base);
                emit_store(g, new, base, old.as.adr.offset);
                freereg(g, base);
                break;
            }
            default:
                bail(
                    "can't replace tagged value with storage location %d",
                    old.loc
                );
            }
        } else if (new.type == payload) {
            emit_result_store_success(g, old, new);
        } else if (new.type == err) {
            emit_result_store_error(g, old, new);
        } else {
            bail(
                "cannot assign %s to %s; type mismatch",
                type_names[new.type->cls],
                type_names[old.type->cls]
            );
        }
    } else {
        /* Non-optional assignments (original logic) */
        switch (old.loc) {
        case LOC_REG:
            /* Load the new value directly into the register of
             * the old value. */
            emit_load_into(g, old.as.reg, new);
            break;
        case LOC_STACK:
            emit_store(g, new, old.as.off.base, old.as.off.offset);
            break;
        case LOC_ADDR: {
            reg_t base = usereg(g, nextreg(g));
            emit_li(g, base, old.as.adr.base);
            emit_store(g, new, base, old.as.adr.offset);
            freereg(g, base);
            break;
        }
        default:
            bail("can't replace variable with storage location %d", old.loc);
        }
    }

    /* Free the new location and update the value, since we don't
     * need two copies of the value. Only free temporaries so we don't
     * invalidate live values that are intentionally kept in registers
     * (eg. function parameters). */
    if (new.loc == LOC_REG && new.temp) {
        freereg(g, new.as.reg);
    }
}

void emit_array_copy(gen_t *g, offset_t src, offset_t dst, type_t *ty) {
    type_t *elem_type = ty->info.ary.elem;
    usize   length    = ty->info.ary.length;

    for (usize i = 0; i < length; i++) {
        i32      elem_off = (i32)(i * elem_type->size);
        offset_t elem_src = OFFSET(src.base, src.offset + elem_off);
        offset_t elem_dst = OFFSET(dst.base, dst.offset + elem_off);

        emit_memcopy(g, elem_src, elem_dst, elem_type);
    }
}

/* Copy single value between offsets, via register */
static void emit_offset_copy(gen_t *g, offset_t src, offset_t dst, type_t *ty) {
    reg_t rs = emit_load(g, value_stack(src, ty));
    emit_regstore(g, rs, dst.base, dst.offset, ty);
    freereg(g, rs);
}

/* Copy a full machine word (WORD_SIZE bytes) using LD/SD. */
static void emit_dword_copy(gen_t *g, offset_t src, offset_t dst) {
    reg_t      tmp     = nextreg(g);
    i32        src_off = src.offset;
    i32        dst_off = dst.offset;
    addr_adj_t src_adj = adjust_addr(g, src.base, &src_off);
    emit(g, LD(tmp, src_adj.base, src_off));
    release_addr(g, src_adj);
    addr_adj_t dst_adj = adjust_addr(g, dst.base, &dst_off);
    emit(g, SD(tmp, dst_adj.base, dst_off));
    release_addr(g, dst_adj);
    freereg(g, tmp);
}

/* Copy tagged values (optional and payload unions) */
static void emit_tval_copy(
    gen_t   *g,
    offset_t src,
    offset_t dst,
    usize    size,
    i32      val_offset,
    type_t  *value_type
) {
    /* Copy tag byte */
    emit_offset_copy(g, src, dst, g->types->type_u8);

    /* Zero padding between tag (1 byte) and payload start, so that
     * byte-level equality comparisons of tagged values work correctly
     * even when the destination was previously uninitialized. */
    if (val_offset > TAG_SIZE) {
        emit_memzero(
            g, OFFSET(dst.base, dst.offset + TAG_SIZE), val_offset - TAG_SIZE
        );
    }

    if (size == 0)
        return;

    offset_t val_src = OFFSET(src.base, src.offset + val_offset);
    offset_t val_dst = OFFSET(dst.base, dst.offset + val_offset);

    if (value_type) {
        /* Use recursive memcopy for typed data (optionals) */
        emit_memcopy(g, val_src, val_dst, value_type);
        return;
    }
    /* Copy raw bytes for untyped data (payload unions) */
    usize copied = 0;

    /* Copy whole dwords (8 bytes) */
    while (copied + WORD_SIZE <= size) {
        emit_dword_copy(
            g,
            OFFSET(val_src.base, val_src.offset + (i32)copied),
            OFFSET(val_dst.base, val_dst.offset + (i32)copied)
        );
        copied += WORD_SIZE;
    }
    /* Copy remaining word (4 bytes) if present */
    if (size - copied >= 4) {
        emit_offset_copy(
            g,
            OFFSET(val_src.base, val_src.offset + (i32)copied),
            OFFSET(val_dst.base, val_dst.offset + (i32)copied),
            g->types->type_i32
        );
        copied += 4;
    }
    /* Copy remaining halfword if present */
    if (size - copied >= 2) {
        emit_offset_copy(
            g,
            OFFSET(val_src.base, val_src.offset + (i32)copied),
            OFFSET(val_dst.base, val_dst.offset + (i32)copied),
            g->types->type_u16
        );
        copied += 2;
    }
    /* Copy remaining byte if present */
    if (size - copied == 1) {
        emit_offset_copy(
            g,
            OFFSET(val_src.base, val_src.offset + (i32)copied),
            OFFSET(val_dst.base, val_dst.offset + (i32)copied),
            g->types->type_u8
        );
    }
}

void emit_memcopy(gen_t *g, offset_t src, offset_t dst, type_t *ty) {
    if (src.base == dst.base && src.offset == dst.offset)
        return; /* Nothing to do. */

    switch (ty->cls) {
    case TYPE_RECORD:
        emit_record_copy(g, src, dst, ty);
        return;
    case TYPE_ARRAY:
        emit_array_copy(g, src, dst, ty);
        return;
    case TYPE_OPT: {
        /* For optional types, copy tag and typed value */
        i32 val_off = align(TAG_SIZE, ty->info.opt.elem->align);
        emit_tval_copy(
            g, src, dst, ty->info.opt.elem->size, val_off, ty->info.opt.elem
        );
        return;
    }
    case TYPE_UNION:
        if (ty->info.uni.has_payload) {
            /* Copy the full payload area including alignment padding,
             * so that byte-level equality comparisons work correctly. */
            i32   val_off      = align(TAG_SIZE, ty->align);
            usize payload_size = ty->size - val_off;
            emit_tval_copy(g, src, dst, payload_size, val_off, NULL);
            return;
        }
        break;
    case TYPE_SLICE: {
        /* For slice types, copy both pointer (8 bytes) and length (8 bytes) */
        emit_dword_copy(g, src, dst);
        emit_dword_copy(
            g,
            OFFSET(src.base, src.offset + WORD_SIZE),
            OFFSET(dst.base, dst.offset + WORD_SIZE)
        );
        return;
    }
    case TYPE_RESULT: {
        bail("result types are never materialized");
    }
    default:
        break;
    }
    /* For primitive types, just copy via a register. */
    emit_offset_copy(g, src, dst, ty);
}

value_t emit_store(gen_t *g, value_t v, reg_t base, int offset) {
    switch (v.loc) {
    case LOC_IMM: {
        /* Load, store, free. */
        reg_t rd = nextreg(g);
        emit_load_into(g, rd, v);
        emit_regstore(g, rd, base, offset, v.type);
        freereg(g, rd);

        break;
    }
    case LOC_REG:
        if (type_is_passed_by_ref(v.type)) {
            emit_memcopy(g, OFFSET(v.as.reg, 0), OFFSET(base, offset), v.type);
        } else {
            emit_regstore(g, v.as.reg, base, offset, v.type);
        }
        break;
    case LOC_STACK:
        emit_memcopy(g, v.as.off, OFFSET(base, offset), v.type);
        break;
    case LOC_ADDR: {
        /* Copy from data section into stack */
        reg_t addr = nextreg(g);
        emit_li(g, addr, v.as.adr.base);
        emit_memcopy(
            g, OFFSET(addr, v.as.adr.offset), OFFSET(base, offset), v.type
        );
        freereg(g, addr);

        break;
    }
    case LOC_NONE:
        break;
    }
    return value_stack(OFFSET(base, offset), v.type);
}

reg_t emit_load(gen_t *g, value_t v) {
    if (v.loc == LOC_REG && v.temp) {
        return v.as.reg;
    } else {
        return emit_load_into(g, nextreg(g), v);
    }
}

/* Load a full machine dword (WORD_SIZE = 8 bytes) from a value. */
reg_t emit_load_dword(gen_t *g, value_t v) {
    /* Use TYPE_PTR to trigger LD (8-byte load). */
    type_t ptr_type = { .cls = TYPE_PTR };
    return emit_load(
        g, (value_t){ .loc = v.loc, .as = v.as, .type = &ptr_type }
    );
}

reg_t emit_load_offset(gen_t *g, value_t v, i32 offset) {
    reg_t rd = nextreg(g);
    switch (v.loc) {
    case LOC_REG:
        emit(g, LD(rd, v.as.reg, offset));
        break;
    case LOC_STACK: {
        i32        combined_offset = v.as.off.offset + offset;
        addr_adj_t adj = adjust_addr(g, v.as.off.base, &combined_offset);
        emit(g, LD(rd, adj.base, combined_offset));
        release_addr(g, adj);
        break;
    }
    case LOC_ADDR: {
        reg_t base = nextreg(g);
        emit_li(g, base, v.as.adr.base);
        i32        combined_offset = v.as.adr.offset + offset;
        addr_adj_t adj             = adjust_addr(g, base, &combined_offset);
        emit(g, LD(rd, adj.base, combined_offset));
        release_addr(g, adj);
        freereg(g, base);
        break;
    }
    case LOC_IMM:
    case LOC_NONE:
        abort();
    }
    return rd;
}

value_t emit_push(gen_t *g, value_t v) {
    /* Always allocate new stack space - each variable should have its own
     * location */
    int offset = reserve(g, v.type);
    return emit_store(g, v, FP, offset);
}

value_t emit_array_index(gen_t *g, value_t array_val, value_t index, bool ref) {
    reg_t   elem_siz    = nextreg(g);
    reg_t   data_adr    = ZERO;
    reg_t   base_reg    = ZERO;
    reg_t   base_alloc  = ZERO;
    i32     base_offset = 0;
    type_t *elem_type;
    type_t *arr_type = array_val.type;
    if (arr_type->cls == TYPE_PTR) {
        arr_type = arr_type->info.ptr.target;
    }

    /* Handle different storage locations */
    if (array_val.type->cls == TYPE_PTR) {
        /* Dereference pointers up front to get the actual base address. */
        base_reg    = emit_load_dword(g, array_val);
        base_offset = 0;
    } else if (array_val.loc == LOC_REG) {
        base_reg    = array_val.as.reg;
        base_offset = 0;
    } else if (array_val.loc == LOC_STACK) {
        base_reg    = array_val.as.off.base;
        base_offset = array_val.as.off.offset;
    } else if (array_val.loc == LOC_ADDR) {
        /* For constants in the data section, load the address but don't
         * dereference it. This way we get the actual array base address for
         * indexing. */
        base_reg = nextreg(g);
        emit_li(g, base_reg, array_val.as.adr.base);
        base_offset = array_val.as.adr.offset;
        base_alloc  = base_reg;
    } else {
        bail("cannot index array/slice at this location");
    }
    /* Load index into a register. Will hold final output */
    reg_t rd = emit_load(g, index);

    if (arr_type->cls == TYPE_SLICE) {
        /* Adjust base_offset for large offsets before loading slice fields */
        i32        ptr_offset = base_offset;
        addr_adj_t adj        = adjust_addr(g, base_reg, &ptr_offset);

        /* Load data pointer (first dword of slice) */
        /* and use it as our new base. */
        data_adr = nextreg(g);
        emit(g, LD(data_adr, adj.base, ptr_offset));

        /* Load slice length (second dword of slice) for bounds checking */
        reg_t len = nextreg(g);
        emit(g, LD(len, adj.base, ptr_offset + WORD_SIZE));

        release_addr(g, adj);

        /* Bounds check: if index >= length, emit EBREAK */
        /* Skip EBREAK if index < length (jump 2 instructions) */
        emit(g, BLTU(rd, len, INSTR_SIZE * 2));
        emit(g, EBREAK);

        freereg(g, len);

        base_reg    = data_adr;
        base_offset = 0;
        elem_type   = arr_type->info.slc.elem;
    } else {
        elem_type = arr_type->info.ary.elem;
    }

    /* Get element size */
    emit_li(g, elem_siz, elem_type->size);
    emit(g, MUL(rd, rd, elem_siz)); /* Relative offset. */
    emit(g, ADD(rd, rd, base_reg));

    freereg(g, elem_siz);
    freereg(g, data_adr);
    if (base_alloc)
        freereg(g, base_alloc);

    if (base_offset != 0 && !is_small(base_offset)) {
        emit_addr_offset(g, rd, rd, base_offset);
        base_offset = 0;
    }

    if (ref) {
        return value_stack(OFFSET(rd, base_offset), elem_type);
    } else {
        /* Reserve space on stack for the element */
        i32 stack_offset = reserve(g, elem_type);

        /* Copy element from array to stack using memcopy */
        offset_t src = OFFSET(rd, base_offset);  /* Source: element in array */
        offset_t dst = OFFSET(FP, stack_offset); /* Destination: stack */
        emit_memcopy(g, src, dst, elem_type);

        freereg(g, rd);

        /* Return a stack-based value pointing to the array element. */
        return value_stack(dst, elem_type);
    }
}

usize emit_regstore(gen_t *g, reg_t src, reg_t base, i32 offset, type_t *ty) {
    reg_t      orig_base   = base;
    i32        orig_offset = offset;
    addr_adj_t adj         = adjust_addr_avoid(g, base, &offset, src);
    reg_t      addr        = adj.base;
    usize      idx         = 0;

    switch (ty->cls) {
    case TYPE_BOOL:
    case TYPE_I8:
    case TYPE_U8:
        idx = emit(g, SB(src, addr, offset));
        break;
    case TYPE_I16:
    case TYPE_U16:
        idx = emit(g, SH(src, addr, offset));
        break;
    case TYPE_I32:
    case TYPE_U32:
        idx = emit(g, SW(src, addr, offset));
        break;
    case TYPE_PTR: /* References are pointers, so store as a dword. */
    case TYPE_FN:  /* Function pointers are addresses, so store as a dword. */
        idx = emit(g, SD(src, addr, offset));
        break;
    case TYPE_UNION:
        if (ty->info.uni.has_payload) {
            /* Tag is 1 byte. */
            idx = emit(g, SB(src, addr, offset));
            break;
        }
        release_addr(g, adj);
        return emit_regstore(g, src, orig_base, orig_offset, ty->info.uni.base);
    case TYPE_ARRAY:
    case TYPE_RECORD:
    case TYPE_OPT:
        /* Structs, arrays, optional types are stored by reference, so
         * just store the address (pointer). */
        idx = emit(g, SD(src, addr, offset));
        break;
    case TYPE_SLICE:
        release_addr(g, adj);
        bail("storing slices via register store is unsupported");
    default:
        bail("storing unsupported type `%s`", type_names[ty->cls]);
    }
    release_addr(g, adj);

    return idx;
}

void emit_store_tag(gen_t *g, tval_t tv, reg_t tag_reg) {
    i32        off = tv.tag.as.off.offset;
    addr_adj_t adj = adjust_addr(g, tv.tag.as.off.base, &off);

    emit(g, SB(tag_reg, adj.base, off));
    release_addr(g, adj);
}

usize emit_regload(gen_t *g, reg_t dst, reg_t base, i32 offset, type_t *ty) {
    reg_t      orig_base   = base;
    i32        orig_offset = offset;
    addr_adj_t adj         = adjust_addr(g, base, &offset);
    reg_t      addr        = adj.base;
    usize      idx         = 0;

    switch (ty->cls) {
    case TYPE_BOOL:
    case TYPE_U8:
        idx = emit(g, LBU(dst, addr, offset));
        break;
    case TYPE_I8:
        idx = emit(g, LB(dst, addr, offset));
        break;
    case TYPE_U16:
        idx = emit(g, LHU(dst, addr, offset));
        break;
    case TYPE_I16:
        idx = emit(g, LH(dst, addr, offset));
        break;
    case TYPE_I32:
        idx = emit(g, LW(dst, addr, offset));
        break;
    case TYPE_U32:
        idx = emit(g, LWU(dst, addr, offset));
        break;
    case TYPE_PTR: /* Raw pointer values occupy one 64-bit dword. */
    case TYPE_FN:  /* Function pointers are addresses, so load as a dword. */
        idx = emit(g, LD(dst, addr, offset));
        break;
    case TYPE_UNION:
        if (ty->info.uni.has_payload) {
            idx = emit(g, ADDI(dst, addr, offset));
            break;
        }
        release_addr(g, adj);
        return emit_regload(g, dst, orig_base, orig_offset, ty->info.uni.base);
    case TYPE_ARRAY:
    case TYPE_RECORD:
    case TYPE_SLICE:
    case TYPE_OPT:
        /* For records, arrays, optional types, we load the address in the
         * register. */
        idx = emit(g, ADDI(dst, addr, offset));
        break;
    default:
        release_addr(g, adj);
        bail("loading unsupported type `%s`", type_names[ty->cls]);
    }
    release_addr(g, adj);

    return idx;
}

int emit_regpush(gen_t *g, reg_t src, type_t *ty) {
    /* Store the register to the stack. */
    int offset = reserve(g, ty);
    emit_regstore(g, src, FP, offset, ty);

    return offset;
}

i32 reserve_aligned(gen_t *g, type_t *ty, i32 align) {
    frame_t *frame = &g->fn.current->e.fn.frame;

    /* Zero-sized types (e.g. empty arrays) don't need stack space. */
    if (ty->size == 0) {
        return frame->sp;
    }
    frame->sp = align_stack(frame->sp - ty->size, align);

    if (-frame->sp >= MAX_FRAME_SIZE)
        bail("stack frame overflow");
    if (-frame->sp < 0)
        bail("stack frame underflow");

    if (-frame->sp > frame->size)
        frame->size = -frame->sp;

    /* Zero memory for non-packed types to ensure clean initialization.
     * Packed types are skipped as they are densely packed without padding. */
    if (!type_is_packed(ty)) {
        emit_memzero(g, OFFSET(FP, frame->sp), ty->size);
    }

    return frame->sp;
}

reg_t emit_load_into(gen_t *g, reg_t dst, value_t src) {
    switch (src.loc) {
    case LOC_IMM:
        switch (src.type->cls) {
        case TYPE_UNION: /* Unions default to i32 base type. */
        case TYPE_I8:
        case TYPE_I16:
        case TYPE_I32:
            emit_li(g, dst, src.as.imm.i);
            break;
        case TYPE_U8:
        case TYPE_U16:
        case TYPE_U32:
        case TYPE_PTR:
        case TYPE_FN:
            emit_li(g, dst, src.as.imm.u);
            break;
        case TYPE_BOOL:
            emit_li(g, dst, src.as.imm.b);
            break;
        default:
            bail("unsupported type `%s`", type_names[src.type->cls]);
        }
        break;
    case LOC_STACK:
        /* For types passed by reference, load the address
         * instead of the value. */
        if (type_is_passed_by_ref(src.type)) {
            i32        off = src.as.off.offset;
            addr_adj_t adj = adjust_addr(g, src.as.off.base, &off);
            emit(g, ADDI(dst, adj.base, off));
            release_addr(g, adj);
        } else {
            emit_regload(g, dst, src.as.off.base, src.as.off.offset, src.type);
        }
        break;
    case LOC_REG: {
        reg_t rs = src.as.reg;
        if (rs == dst) {
            break;
        }
        if (src.temp)
            freereg(g, rs);

        emit(g, MV(dst, rs));
        break;
    }
    case LOC_ADDR: {
        /* Start by loading the address into the register */
        emit_li(g, dst, src.as.adr.base);

        /* For non-compound types, we need to load the value from the address.
         * For compound types, we keep the address itself. */
        if (!type_is_passed_by_ref(src.type)) {
            emit_regload(g, dst, dst, src.as.adr.offset, src.type);
        } else {
            /* For compound types passed by reference, add the offset to get
             * the actual address. */
            if (src.as.adr.offset != 0) {
                emit(g, ADDI(dst, dst, src.as.adr.offset));
            }
        }
        break;
    }
    case LOC_NONE:
        break;
    }
    return dst;
}

/* Compare values at two memory addresses and accumulate result.
 * Loads values from memory, compares them, and ANDs the comparison result
 * with the accumulating result register. */
static void emit_cmp_step(
    gen_t  *g,
    reg_t   left_val,   /* Register to hold left value during comparison */
    reg_t   right_val,  /* Register to hold right value during comparison */
    reg_t   left_addr,  /* Base address register for left operand */
    reg_t   right_addr, /* Base address register for right operand */
    usize   offset,     /* Byte offset from base addresses to load from */
    reg_t   result,     /* Register that accumulates comparison results */
    type_t *val_typ     /* Type information for loading value */
) {
    /* Load values from both memory addresses at the given offset */
    emit_regload(g, left_val, left_addr, offset, val_typ);
    emit_regload(g, right_val, right_addr, offset, val_typ);

    /* XOR the two loaded values: left_val = left_val ^ right_val
     * If values are equal, result will be 0 (equal values XOR to 0)
     * If values differ, result will be non-zero */
    emit(g, XOR(left_val, left_val, right_val));
    /* Convert XOR result to 1 (equal) or 0 (not equal) */
    emit(g, SLTIU(left_val, left_val, 1));
    /* Accumulate the result with a previous result.
     * If any comparison fails, the final result becomes 0 */
    emit(g, AND(result, result, left_val));
}

/* Compare raw bytes at two memory addresses.
 * Sets result = 1 if all bytes match. */
void emit_bytes_equal(
    gen_t *g, reg_t left, reg_t right, usize size, reg_t result
) {
    /* Start assuming they're equal */
    emit_li(g, result, 1);

    if (size == 0)
        return; /* Zero bytes are always equal */

    reg_t left_val  = nextreg(g);
    reg_t right_val = nextreg(g);

    /* Compare dword by dword (8 bytes) */
    usize i, remaining = size;

    for (i = 0; i + WORD_SIZE <= size; i += WORD_SIZE) {
        /* Load 8-byte dwords directly with LD */
        i32        off_l = (i32)i, off_r = (i32)i;
        addr_adj_t adj_l = adjust_addr(g, left, &off_l);
        emit(g, LD(left_val, adj_l.base, off_l));
        release_addr(g, adj_l);
        addr_adj_t adj_r = adjust_addr(g, right, &off_r);
        emit(g, LD(right_val, adj_r.base, off_r));
        release_addr(g, adj_r);
        emit(g, XOR(left_val, left_val, right_val));
        emit(g, SLTIU(left_val, left_val, 1));
        emit(g, AND(result, result, left_val));
    }
    remaining -= i;

    if (remaining >= 4) {
        emit_cmp_step(
            g, left_val, right_val, left, right, i, result, g->types->type_u32
        );
        i         += 4;
        remaining -= 4;
    }

    if (remaining >= 2) {
        emit_cmp_step(
            g, left_val, right_val, left, right, i, result, g->types->type_u16
        );
        i         += 2;
        remaining -= 2;
    }
    if (remaining == 1) {
        emit_cmp_step(
            g, left_val, right_val, left, right, i, result, g->types->type_u8
        );
    }
    freereg(g, left_val);
    freereg(g, right_val);
}

void emit_memequal(
    gen_t *g, reg_t left, reg_t right, type_t *ty, reg_t result
) {
    switch (ty->cls) {
    case TYPE_OPT: { /* For optional types, compare tag and value */
        reg_t left_tag  = nextreg(g);
        reg_t right_tag = nextreg(g);

        /* Load tags (first byte) */
        emit(g, LBU(left_tag, left, 0));
        emit(g, LBU(right_tag, right, 0));

        /* Compare tags directly - if different, optionals are not equal */
        emit_li(g, result, 0); /* Assume not equal */
        usize jump_to_end = emit(g, NOP);

        /* If both are nil (tag == 0), they're equal */
        emit_li(g, result, 1); /* Set equal */
        usize skip_value_check = emit(g, NOP);

        /* Compare values (past tag) */
        type_t *inner_type = ty->info.opt.elem;
        i32     val_off    = align(TAG_SIZE, inner_type->align);
        reg_t   left_val   = nextreg(g);
        reg_t   right_val  = nextreg(g);

        /* Calculate value addresses (skip tag) */
        emit(g, ADDI(left_val, left, val_off));
        emit(g, ADDI(right_val, right, val_off));

        /* Load values if primitive type */
        if (type_is_primitive(inner_type)) {
            emit_regload(g, left_val, left_val, 0, inner_type);
            emit_regload(g, right_val, right_val, 0, inner_type);
        }

        /* Compare the values recursively */
        emit_memequal(g, left_val, right_val, inner_type, result);

        /* Patch skip_value_check: jump here if both tags are 0 (nil) */
        g->instrs[skip_value_check] =
            BEQ(left_tag, ZERO, jump_offset(skip_value_check, g->ninstrs));

        /* Patch jump_to_end: jump here if tags are different */
        g->instrs[jump_to_end] =
            BNE(left_tag, right_tag, jump_offset(jump_to_end, g->ninstrs));

        freereg(g, left_tag);
        freereg(g, right_tag);
        freereg(g, left_val);
        freereg(g, right_val);

        break;
    }
    case TYPE_I8:
    case TYPE_I16:
    case TYPE_I32:
        /* For primitive types, compare directly */
        emit(g, SUB(result, left, right));
        emit(g, SLTIU(result, result, 1));
        break;
    case TYPE_U8:
    case TYPE_U16:
    case TYPE_U32:
    case TYPE_BOOL:
    case TYPE_PTR:
        /* For primitive types, compare directly */
        emit(g, XOR(result, left, right));
        emit(g, SLTIU(result, result, 1));
        break;
    case TYPE_UNION:
        if (!ty->info.uni.has_payload) {
            type_t *base =
                ty->info.uni.base ? ty->info.uni.base : g->types->type_i32;
            emit_memequal(g, left, right, base, result);
        } else {
            emit_bytes_equal(g, left, right, ty->size, result);
        }
        break;
    case TYPE_ARRAY:
    case TYPE_RECORD:
    case TYPE_SLICE:
        emit_bytes_equal(g, left, right, ty->size, result);
        break;
    default:
        bail("equality is not supported for type `%s`", ty->name);
    }
}

void emit_copy_by_ref(gen_t *g, value_t src, value_t dst) {
    static type_t ptr_type = { .cls = TYPE_PTR };

    if (src.loc == LOC_REG && dst.loc == LOC_REG) {
        emit_mv(g, dst.as.reg, src.as.reg);
    } else if (src.loc == LOC_REG && dst.loc == LOC_STACK) {
        i32     dst_off  = dst.as.off.offset;
        type_t *store_ty = dst.type;

        if (dst.type->cls == TYPE_SLICE) {
            /* Slice fat pointers live on the stack; only copy the address. */
            dst_off  += SLICE_FIELD_PTR_OFFSET;
            store_ty  = &ptr_type;
        }
        emit_regstore(g, src.as.reg, dst.as.off.base, dst_off, store_ty);
    } else if (src.loc == LOC_STACK && dst.loc == LOC_REG) {
        type_t *load_ty = dst.type;
        i32     src_off = src.as.off.offset;

        if (dst.type->cls == TYPE_SLICE) {
            load_ty  = &ptr_type;
            src_off += SLICE_FIELD_PTR_OFFSET;
        }
        emit_regload(g, dst.as.reg, src.as.off.base, src_off, load_ty);
    } else if (src.loc == LOC_STACK && dst.loc == LOC_STACK) {
        i32        src_off = src.as.off.offset;
        addr_adj_t src_adj = adjust_addr(g, src.as.off.base, &src_off);
        reg_t      adr     = nextreg(g);

        emit(g, ADDI(adr, src_adj.base, src_off));

        i32        dst_off = dst.as.off.offset;
        addr_adj_t dst_adj = adjust_addr(g, dst.as.off.base, &dst_off);

        if (dst.type->cls == TYPE_SLICE)
            dst_off += SLICE_FIELD_PTR_OFFSET;

        emit(g, SD(adr, dst_adj.base, dst_off));

        release_addr(g, dst_adj);
        release_addr(g, src_adj);
        freereg(g, adr);
    } else if (src.loc == LOC_ADDR && dst.loc == LOC_STACK) {
        reg_t adr = nextreg(g);
        /* Load the absolute address into a register. */
        emit_li(g, adr, (i32)(src.as.adr.base + src.as.adr.offset));
        i32     dst_off  = dst.as.off.offset;
        type_t *store_ty = dst.type;

        if (dst.type->cls == TYPE_SLICE) {
            dst_off  += SLICE_FIELD_PTR_OFFSET;
            store_ty  = &ptr_type;
        }
        emit_regstore(g, adr, dst.as.off.base, dst_off, store_ty);
        freereg(g, adr);
    } else {
        bail("don't know how to copy between these slots");
    }
}

/* Write a successful result tag (0) and copy the payload if present. */
void emit_result_store_success(gen_t *g, value_t dest, value_t value) {
    tval_t tv  = tval_from_val(g, dest);
    reg_t  tag = nextreg(g);

    emit_li(g, tag, 0);
    emit_store_tag(g, tv, tag);
    freereg(g, tag);

    type_t *payload = dest.type->info.res.payload;

    /* Nb. We don't memzero, since result types are always unwrapped to
     * one of their payloads. */

    if (payload->size > 0) {
        /* Check if we need to wrap the value in an optional. */
        if (payload->cls == TYPE_OPT && value.type->cls != TYPE_OPT) {
            /* Wrap non-optional value in an optional */
            value_t payload_val = value_stack(
                OFFSET(tv.val.as.off.base, tv.val.as.off.offset), payload
            );
            tval_store(g, payload_val, value, 1);
        } else {
            emit_store(g, value, tv.val.as.off.base, tv.val.as.off.offset);
        }
    }
}

/* Write an error Result tag (1) and copy the error payload. */
void emit_result_store_error(gen_t *g, value_t dest, value_t err) {
    tval_t tv  = tval_from_val(g, dest);
    reg_t  tag = nextreg(g);

    emit_li(g, tag, 1);
    emit_store_tag(g, tv, tag);
    freereg(g, tag);

    /* Nb. We don't memzero, since result types are always unwrapped to
     * one of their payloads. */

    if (err.type->cls != TYPE_VOID) {
        emit_store(g, err, tv.val.as.off.base, tv.val.as.off.offset);
    }
}