radiance · commit

Stream function lowering into RV64 codegen

e69f75fc4fd7ec3a4ef3ab0d933e466dc7caa43a80d6cd14ef91e66b1f7f12ef

Lower functions into a per-function arena and emit them to RV64 as they
are produced, allowing function-local IL to be reclaimed after each
function.

Split RV64 generation into begin, per-function, and finish phases so
data layout and address patching can happen after all functions are
generated. Track the default entry through codegen entry patching
instead of storing it on the IL program.

Alexis Sellier committed 1 month ago 1 parent 64585227

compiler/radiance.rad +151 -30

constant MAX_TESTS: u32 = 1024;

/// Temporary arena size (32 MB) - retains all parsed AST until resolution.
/// Used for: AST during parsing, then codegen scratch space.
constant TEMP_ARENA_SIZE: u32 = 33554432;
/// Main arena size (128 MB) - lives throughout compilation.
/// Used for: resolver data, types, symbols, lowered IL.
constant MAIN_ARENA_SIZE: u32 = 134217728;
/// Main arena size (64 MB) - lives throughout compilation.
/// Used for: resolver data, types, symbols, global IL data, and codegen output.
constant MAIN_ARENA_SIZE: u32 = 67108864;

/// Temporary storage arena - reusable between phases.
static TEMP_ARENA: [u8; TEMP_ARENA_SIZE] = undefined;
/// Main storage arena - persists throughout compilation.
static MAIN_ARENA: [u8; MAIN_ARENA_SIZE] = undefined;

record RootModule {
    entry: *module::ModuleEntry,
    ast: *mut ast::Node,
}

/// State carried by the streaming lowerer/codegen callback.
record CodegenSinkContext {
    /// RV64 generator receiving lowered functions.
    generator: *mut rv64::Generator,
    /// Arena used for codegen scratch allocations.
    codegenArena: *mut alloc::Arena,
    /// Arena holding the current function's lowered IL.
    fnArena: *mut alloc::Arena,
}

/// Entry handling for streamed code generation.
union CodegenEntryMode {
    /// Do not reserve an entry jump.
    None,
    /// Reserve and patch an entry jump to the `@default` function.
    DefaultEntry,
}

/// Options controlling streamed lowering and code generation.
record CodegenOptions {
    /// Optional output path used for progress logging.
    logPath: ?*[u8],
    /// Whether to emit debug source locations.
    debug: bool,
    /// How the generated program should handle entry.
    entryMode: CodegenEntryMode,
}

/// Print a log line for the given package.
fn pkgLog(pkg: *package::Package, msg: *[*[u8]]) {
    io::printError("radiance: ");
    io::printError(pkg.name);
    io::printError(": ");

    };
    let entryPkg = &ctx.packages[entryIdx];

    // Create the lowerer accumulator using entry package's name.
    let options = lower::LowerOptions { debug: ctx.debug, buildTest: ctx.config.buildTest };
    let mut low = lower::lowerer(res, &ctx.graph, entryPkg.name, &mut res.arena, options);
    let mut low = lower::lowerer(
        res, &ctx.graph, entryPkg.name, &mut res.arena, &mut res.arena, options
    );
    try lowerAllPackagesInto(ctx, res, &mut low);

    // Finalize and return the unified program.
    return lower::finalize(&low);
}

/// Lower all packages into an existing lowerer.
fn lowerAllPackagesInto(
    ctx: *mut CompileContext,
    res: *mut resolver::Resolver,
    low: *mut lower::Lowerer
) throws (Error) {
    let entryIdx = ctx.entryPkgIdx else {
        panic "lowerAllPackagesInto: no entry package";
    };
    // Lower all packages except entry.
    for i in 0..ctx.packageCount {
        if i <> entryIdx {
            try lowerPackage(ctx, res, &mut low, &mut ctx.packages[i], false);
            try lowerPackage(ctx, res, low, &mut ctx.packages[i], false);
        }
    }
    // Lower entry package.
    let defaultFn = try lowerPackage(ctx, res, &mut low, &mut ctx.packages[entryIdx], true);

    // Finalize and return the unified program.
    return lower::finalize(&low, defaultFn);
    try lowerPackage(ctx, res, low, &mut ctx.packages[entryIdx], true);
}

/// Lower all modules in a package into the lowerer accumulator.
fn lowerPackage(
    ctx: *CompileContext,
    res: *mut resolver::Resolver,
    low: *mut lower::Lowerer,
    pkg: *mut package::Package,
    isEntry: bool
) -> ?u32 throws (Error) {
) throws (Error) {
    let rootId = pkg.rootModuleId else {
        io::printError("radiance: no root module found\n");
        throw Error::Other;
    };
    // Set lowerer's package context for qualified name generation.
    // TODO: We shouldn't have to call this manually.
    lower::setPackage(low, &ctx.graph, pkg.name);

    return try lowerModuleTreeInto(ctx, low, &ctx.graph, rootId, isEntry, pkg);
    try lowerModuleTreeInto(ctx, low, &ctx.graph, rootId, isEntry, pkg);
}

/// Recursively lower a module and all its children into the accumulator.
fn lowerModuleTreeInto(
    ctx: *CompileContext,
    low: *mut lower::Lowerer,
    graph: *module::ModuleGraph,
    modId: u16,
    isRoot: bool,
    pkg: *package::Package
) -> ?u32 throws (Error) {
) throws (Error) {
    let entry = module::get(graph, modId) else {
        io::printError("radiance: module entry not found\n");
        throw Error::Other;
    };
    let modAst = entry.ast else {
        io::printError("radiance: module has no AST\n");
        throw Error::Other;
    };
    pkgLog(pkg, &["lowering", "(", entry.filePath, ")", ".."]);

    let defaultFn = try lower::lowerModule(low, modId, modAst, isRoot) catch err {
    try lower::lowerModule(low, modId, modAst, isRoot) catch err {
        io::printError("radiance: internal error during lowering: ");
        lower::printError(err);
        io::printError("\n");
        throw Error::Other;
    };
    // Recurse into children.
    for i in 0..entry.childrenLen {
        let childId = module::childAt(entry, i);
        try lowerModuleTreeInto(ctx, low, graph, childId, false, pkg);
    }
    return defaultFn;
}

/// Build a scope access chain: a::b::c from a slice of identifiers.
fn synthScopeAccess(arena: *mut ast::NodeArena, path: *[*[u8]]) -> *ast::Node {
    let mut result = ast::synthNode(

        throw Error::Other;
    }
    return res;
}

/// Emit one lowered function to RV64 codegen and reclaim its IL arena.
fn generateLoweredFn(ctxPtr: *mut opaque, func: *il::Fn, role: lower::FnRole) {
    let ctx = ctxPtr as *mut CodegenSinkContext;

    match role {
        case lower::FnRole::Default => {
            match ctx.generator.entryPatch {
                case rv64::EntryPatch::Reserved(_) => {
                    set ctx.generator.entryPatch = rv64::EntryPatch::Reserved(func.name);
                }
                else => panic "generateLoweredFn: entry jump was not reserved",
            }
        }
        else => {}
    }
    rv64::generateFunction(ctx.generator, func, ctx.codegenArena);
    alloc::reset(ctx.fnArena);
}

/// Lower all packages while streaming each lowered function into RV64 codegen.
fn lowerAndGenerateAllPackages(
    ctx: *mut CompileContext,
    res: *mut resolver::Resolver,
    fnArena: *mut alloc::Arena,
    codegenOptions: CodegenOptions
) -> rv64::Program throws (Error) {
    let entryIdx = ctx.entryPkgIdx else {
        panic "lowerAndGenerateAllPackages: no entry package";
    };
    let entryPkg = &ctx.packages[entryIdx];
    let options = lower::LowerOptions { debug: ctx.debug, buildTest: ctx.config.buildTest };
    let storage = rv64::Storage {
        dataSyms: &mut CODEGEN_DATA_SYMS[..],
        dataSymEntries: &mut CODEGEN_DATA_SYM_ENTRIES[..],
    };
    let mut entryPatch = rv64::EntryPatch::None;
    match codegenOptions.entryMode {
        case CodegenEntryMode::DefaultEntry => {
            set entryPatch = rv64::EntryPatch::Reserved(nil);
        }
        else => {}
    }
    let mut generator = rv64::beginProgram(
        rv64::ProgramOptions { entryPatch, debug: codegenOptions.debug },
        &mut res.arena
    );
    let mut codegenCtx = CodegenSinkContext {
        generator: &mut generator,
        codegenArena: &mut res.arena,
        fnArena,
    };
    let mut low = lower::lowerer(
        res, &ctx.graph, entryPkg.name, &mut res.arena, fnArena, options
    );
    set low.output = lower::FnOutput::Stream(lower::FnSink {
        ctx: &mut codegenCtx as *mut opaque,
        emitFn: generateLoweredFn,
    });
    try lowerAllPackagesInto(ctx, res, &mut low);

    match generator.entryPatch {
        case rv64::EntryPatch::Reserved(targetName) => {
            if targetName == nil {
                io::printError("radiance: fatal: no default function found\n");
                throw Error::Other;
            }
        }
        else => {}
    }
    if let path = codegenOptions.logPath {
        pkgLog(entryPkg, &["generating code", "(", path, ")", ".."]);
    }
    return rv64::finishProgram(&mut generator, &low.data[..], storage, &mut RO_DATA_BUF[..], &mut RW_DATA_BUF[..]);
}

/// Lower, optionally dump, and optionally generate binary output.
fn compile(ctx: *mut CompileContext, res: *mut resolver::Resolver) throws (Error) {
fn compile(
    ctx: *mut CompileContext,
    res: *mut resolver::Resolver,
    fnArena: *mut alloc::Arena
) throws (Error) {
    let entryPkg = try getEntryPackage(ctx);

    // Lower all packages into a single unified IL program.
    // Dependencies must be lowered before the packages that use them.
    let program = try lowerAllPackages(ctx, res);
    let mut out = sexpr::Output::Stdout;

    if ctx.dump == Dump::Il {
        // Lower all packages into a single unified IL program for dumping.
        let program = try lowerAllPackages(ctx, res);
        il::printer::printProgram(&mut out, &mut res.arena, &program);
        io::print("\n");
        return;
    }
    if ctx.dump == Dump::Asm {
        // TODO: Why do we generate code in two places?
        let result = rv64::generate(&program, rv64::Storage { dataSyms: &mut CODEGEN_DATA_SYMS[..], dataSymEntries: &mut CODEGEN_DATA_SYM_ENTRIES[..] }, &mut RO_DATA_BUF[..], &mut RW_DATA_BUF[..], &mut res.arena, false);
        let result = try lowerAndGenerateAllPackages(ctx, res, fnArena, CodegenOptions {
            logPath: nil,
            debug: false,
            entryMode: CodegenEntryMode::None,
        });
        printer::printCodeTo(&mut out, entryPkg.name, result.code, result.funcs, &mut res.arena);
        io::print("\n");

        return;
    }
    // Generate binary output if path specified.
    let outPath = ctx.outputPath else {
        try lowerAllPackages(ctx, res);
        return;
    };
    // Check that we have an entry point.
    if program.defaultFnIdx == nil {
        io::printError("radiance: fatal: no default function found\n");
        throw Error::Other;
    }
    pkgLog(entryPkg, &["generating code", "(", outPath, ")", ".."]);
    let result = try lowerAndGenerateAllPackages(ctx, res, fnArena, CodegenOptions {
        logPath: outPath,
        debug: ctx.debug,
        entryMode: CodegenEntryMode::DefaultEntry,
    });

    let storage = rv64::Storage { dataSyms: &mut CODEGEN_DATA_SYMS[..], dataSymEntries: &mut CODEGEN_DATA_SYM_ENTRIES[..] };
    let result = rv64::generate(&program, storage, &mut RO_DATA_BUF[..], &mut RW_DATA_BUF[..], &mut res.arena, ctx.debug);
    if not writeCode(result.code, outPath) {
        io::printError("radiance: fatal: failed to write output file\n");
        throw Error::Other;
    }
    // Write data files.

    }
    // Run resolution phase.
    let mut res = try runResolver(&mut ctx, arena.nextId) catch {
        return 1;
    };
    let mut fnArena = alloc::new(alloc::remainingBuf(&mut arena.arena));

    // Lower, dump, and/or generate output.
    try compile(&mut ctx, &mut res) catch {
    try compile(&mut ctx, &mut res, &mut fnArena) catch {
        return 1;
    };
    return 0;
}

lib/std/arch/rv64.rad +97 -51

    rwDataSize: u32,
    /// Debug entries mapping PCs to source locations. Empty when debug is off.
    debugEntries: *[types::DebugEntry],
}

/// Generate code for an IL program.
export fn generate(
    program: *il::Program,
/// Entry jump patching requested for the generated program.
export union EntryPatch {
    /// No entry jump is emitted.
    None,
    /// Reserve code slot zero for a jump to the default function.
    Reserved(?*[u8]),
}

/// Options controlling incremental RV64 program generation.
export record ProgramOptions {
    /// Entry jump patching mode.
    entryPatch: EntryPatch,
    /// Whether to emit debug source locations.
    debug: bool,
}

/// State for incremental RV64 program generation.
///
/// The generator owns global codegen state that must survive across function
/// emission. Function-local scratch stays outside this record so callers can
/// reclaim it after each function.
export record Generator {
    /// Binary emitter and relocation state.
    e: emit::Emitter,
    /// Entry jump patching state.
    entryPatch: EntryPatch,
}

/// Begin RV64 code generation for a program's global state.
export fn beginProgram(
    options: ProgramOptions,
    arena: *mut alloc::Arena
) -> Generator {
    let mut e = try! emit::emitter(arena, options.debug);

    // Emit placeholder entry jump when requested.
    // We'll patch this at the end once we know where the function is.
    match options.entryPatch {
        case EntryPatch::Reserved(_) => {
            emit::emit(&mut e, encode::nop()); // Placeholder for two-instruction jump.
            emit::emit(&mut e, encode::nop()); //
        }
        else => {}
    }

    return Generator {
        e,
        entryPatch: options.entryPatch,
    };
}

/// Generate code for one IL function.
export fn generateFunction(
    generator: *mut Generator,
    func: *il::Fn,
    arena: *mut alloc::Arena
) {
    if func.isExtern {
        return;
    }
    let checkpoint = alloc::save(arena);
    let config = targetConfig();
    let ralloc = try! regalloc::allocate(func, &config, arena);

    isel::selectFn(&mut generator.e, &ralloc, func);

    // Reclaim unused memory after instruction selection.
    alloc::restore(arena, checkpoint);
}

/// Finish RV64 code generation and return the emitted program.
export fn finishProgram(
    generator: *mut Generator,
    globalData: *[il::Data],
    storage: Storage,
    roDataBuf: *mut [u8],
    rwDataBuf: *mut [u8],
    arena: *mut alloc::Arena,
    debug: bool
    rwDataBuf: *mut [u8]
) -> Program {
    let mut e = try! emit::emitter(arena, debug);
    let config = targetConfig();

    // Build data map.
    // Build data map after function lowering. Function-local literals can add
    // global data while functions are lowered, so final layout belongs here.
    let mut dataSymCount: u32 = 0;
    let roLayoutSize = data::layoutSection(globalData, storage.dataSyms, &mut dataSymCount, RO_DATA_BASE, true);
    data::layoutSection(globalData, storage.dataSyms, &mut dataSymCount, RW_DATA_BASE, false);

    let roLayoutSize = data::layoutSection(program, storage.dataSyms, &mut dataSymCount, RO_DATA_BASE, true);
    data::layoutSection(program, storage.dataSyms, &mut dataSymCount, RW_DATA_BASE, false);

    // Build hash-indexed data symbol map for O(1) lookups.
    let dataSyms = &storage.dataSyms[..dataSymCount];
    let mut dataSymMap = data::buildMap(dataSyms, storage.dataSymEntries);

    // Code base address: code follows read-only data, aligned to dword boundary.
    let dataSymMap = data::buildMap(dataSyms, storage.dataSymEntries);
    let codeBase = mem::alignUp(RO_DATA_BASE + roLayoutSize, DWORD_SIZE as u32);

    // Emit placeholder entry jump to default function if there is one.
    // We'll patch this at the end once we know where the function is.
    let mut defaultName: ?*[u8] = nil;
    if let defIdx = program.defaultFnIdx {
        set defaultName = program.fns[defIdx].name;
        emit::emit(&mut e, encode::nop()); // Placeholder for two-instruction jump.
        emit::emit(&mut e, encode::nop()); //
    }

    // Generate code for all functions.
    for i in 0..program.fns.len {
        let func = program.fns[i];
        if not func.isExtern {
            let checkpoint = alloc::save(arena);
            let ralloc = try! regalloc::allocate(func, &config, arena);
            isel::selectFn(&mut e, &dataSymMap, &ralloc, func);
    match generator.entryPatch {
        case EntryPatch::Reserved(targetName) => {
            let target = targetName else {
                panic "finishProgram: entry jump reserved without default function";
            };
            let offset = emit::branchOffsetToFunc(&generator.e, 0, target);
            let s = emit::splitImm(offset);

            // Reclaim unused memory after instruction selection.
            alloc::restore(arena, checkpoint);
            emit::patch(&mut generator.e, 0, encode::auipc(SCRATCH1, s.hi));
            emit::patch(&mut generator.e, 1, encode::jalr(ZERO, SCRATCH1, s.lo));
        }
    }
    // Patch entry jump now that we know where the default function is.
    // Nb. we use a two-instruction jump even if it isn't always needed.
    if let target = defaultName {
        let offset = emit::branchOffsetToFunc(&e, 0, target);
        let s = emit::splitImm(offset);

        emit::patch(&mut e, 0, encode::auipc(SCRATCH1, s.hi));
        emit::patch(&mut e, 1, encode::jalr(ZERO, SCRATCH1, s.lo));
        else => {}
    }
    // Patch function calls and address loads now that all functions are emitted.
    emit::patchCalls(&mut e);
    emit::patchAddrLoads(&mut e);
    emit::patchCalls(&mut generator.e);
    emit::patchAddrLoads(&mut generator.e, &dataSymMap);

    // Emit data sections.
    let roDataSize = data::emitSection(program, &dataSymMap, &e.labels, codeBase, roDataBuf, true);
    let rwDataSize = data::emitSection(program, &dataSymMap, &e.labels, codeBase, rwDataBuf, false);
    let roDataSize = data::emitSection(globalData, &dataSymMap, &generator.e.labels, codeBase, roDataBuf, true);
    let rwDataSize = data::emitSection(globalData, &dataSymMap, &generator.e.labels, codeBase, rwDataBuf, false);

    return Program {
        code: emit::getCode(&e),
        funcs: emit::getFuncs(&e),
        code: emit::getCode(&generator.e),
        funcs: emit::getFuncs(&generator.e),
        roDataSize,
        rwDataSize,
        debugEntries: emit::getDebugEntries(&e),
        debugEntries: emit::getDebugEntries(&generator.e),
    };
}

lib/std/arch/rv64/emit.rad +40 -8

//! Emits RV64 machine code as `u32` list.

use std::lang::il;
use std::lang::alloc;
use std::lang::gen;
use std::lang::gen::data;
use std::lang::gen::labels;
use std::lang::gen::types;
use std::collections::dict;
use std::mem;



/// Maximum number of instructions in code buffer.
constant MAX_INSTRS: u32 = 2097152;
/// Maximum code length before byte offset overflows signed 32-bits.
constant MAX_CODE_LEN: u32 = 0x7FFFFFFF / super::INSTR_SIZE as u32;
/// Maximum positive value encodable by a signed 32-bit address calculation.
constant MAX_I32_ADDR: u32 = 0x7FFFFFFF;
/// Maximum number of pending branches awaiting patching.
constant MAX_PENDING: u32 = 65536;
/// Maximum number of function entries.
constant MAX_FUNCS: u32 = 4096;
/// Maximum number of debug entries.

    index: u32,
    /// Target function name.
    target: *[u8],
}

/// Function address load that needs offset patching.
/// Used when taking a function's address as a value.
/// Address load that needs patching after layout is known.
export record PendingAddrLoad {
    /// Index in code buffer where the load was emitted.
    index: u32,
    /// Target function name.
    /// Target function or data symbol name.
    target: *[u8],
    /// Destination register.
    rd: gen::Reg,
    /// Whether this is an absolute data symbol address.
    isData: bool,
}

/// Adjusted base register and offset for addressing.
export record AdjustedOffset {
    /// Base register.

    assert e.pendingAddrLoadsLen < e.pendingAddrLoads.len, "recordAddrLoad: buffer full";
    set e.pendingAddrLoads[e.pendingAddrLoadsLen] = PendingAddrLoad {
        index: e.codeLen,
        target,
        rd: rd,
        isData: false,
    };
    set e.pendingAddrLoadsLen += 1;

    emit(e, encode::nop()); // Placeholder for AUIPC.
    emit(e, encode::nop()); // Placeholder for ADDI.
}

/// Record a data address load needing later patching.
/// Uses an absolute 32-bit load sequence matching the current data memory map.
export fn recordDataAddrLoad(e: *mut Emitter, target: *[u8], rd: gen::Reg) {
    assert e.pendingAddrLoadsLen < e.pendingAddrLoads.len, "recordDataAddrLoad: buffer full";
    set e.pendingAddrLoads[e.pendingAddrLoadsLen] = PendingAddrLoad {
        index: e.codeLen,
        target,
        rd: rd,
        isData: true,
    };
    set e.pendingAddrLoadsLen += 1;

    emit(e, encode::nop()); // Placeholder for LUI.
    emit(e, encode::nop()); // Placeholder for ADDIW.
}

/// Patch local branches and clear the pending list.
///
/// Called after each function.
///
/// Uses two-instruction sequences: short branches use `branch` and `nop`,

        // `JALR ra, scratch, lo(offset)`.
        patch(e, p.index + 1, encode::jalr(super::RA, super::SCRATCH1, s.lo));
    }
}

/// Patch all pending function address loads.
/// Called after all functions have been generated.
/// Uses PC-relative addresses up to 2GB away.
export fn patchAddrLoads(e: *mut Emitter) {
/// Patch all pending function and data address loads.
/// Called after all functions have been generated and data layout is known.
export fn patchAddrLoads(e: *mut Emitter, dataSymMap: *data::DataSymMap) {
    for i in 0..e.pendingAddrLoadsLen {
        let p = e.pendingAddrLoads[i];
        if p.isData {
            let addr = data::lookupAddr(dataSymMap, p.target) else {
                panic "patchAddrLoads: data symbol not found";
            };
            assert addr <= MAX_I32_ADDR, "patchAddrLoads: data address too large";
            let s = splitImm(addr as i32);

            patch(e, p.index, encode::lui(p.rd, s.hi));
            patch(e, p.index + 1, encode::addiw(p.rd, p.rd, s.lo));

            continue;
        }

        let offset = branchOffsetToFunc(e, p.index, p.target);
        let s = splitImm(offset);

        // `AUIPC rd, hi(offset)`.
        patch(e, p.index, encode::auipc(p.rd, s.hi));
        // `ADDI rd, rd, lo(offset)`.
        patch(e, p.index + 1, encode::addi(p.rd, p.rd, s.lo));
    }

lib/std/arch/rv64/isel.rad +2 -22

use std::mem;
use std::lang::il;
use std::lang::gen;
use std::lang::gen::regalloc;
use std::lang::gen::labels;
use std::lang::gen::data;

use super::encode;
use super::emit;

///////////////

/// Shift operation.
union ShiftOp { Sll, Srl, Sra }
/// Compare operation.
union CmpOp { Slt, Ult }

/// Selector errors.
export union SelectorError {
    Internal,
}

/// A pending spill store to be flushed after instruction selection.
record PendingSpill {
    /// The SSA register that was spilled.
    ssa: il::Reg,
    /// The physical register holding the value to store.

export record Selector {
    /// Emitter for outputting instructions.
    e: *mut emit::Emitter,
    /// Register allocation result.
    ralloc: *regalloc::AllocResult,
    /// Hash-indexed data symbol map.
    dataSymMap: *data::DataSymMap,
    /// Total stack frame size.
    frameSize: i32,
    /// Running offset into the reserve region of the frame.
    /// Tracks current position within the pre-allocated reserve slots.
    reserveOffset: i32,

        return scratch;
    }
    return getReg(s, ssa);
}

/// Look up symbol address in data map.
fn lookupDataSym(s: *Selector, name: *[u8]) -> u32 throws (SelectorError) {
    let addr = data::lookupAddr(s.dataSymMap, name) else {
        throw SelectorError::Internal;
    };
    return addr;
}

/// Resolve an IL value to the physical register holding it.
/// For non-spilled register values, returns the physical register directly.
/// For immediates, symbols, and spilled registers, materializes into `scratch`.
fn resolveVal(s: *mut Selector, scratch: gen::Reg, val: il::Val) -> gen::Reg {
    match val {

            }
            emit::loadImm(s.e, scratch, imm);
            return scratch;
        },
        case il::Val::DataSym(name) => {
            let addr = try lookupDataSym(s, name) catch {
                panic "resolveVal: data symbol not found";
            };
            emit::loadImm(s.e, scratch, addr as i64);
            emit::recordDataAddrLoad(s.e, name, scratch);
            return scratch;
        },
        case il::Val::FnAddr(name) => {
            emit::recordAddrLoad(s.e, name, scratch);
            return scratch;

}

/// Select instructions for a function.
export fn selectFn(
    e: *mut emit::Emitter,
    dataSymMap: *data::DataSymMap,
    ralloc: *regalloc::AllocResult,
    func: *il::Fn
) {
    // Reset block offsets for this function.
    labels::resetBlocks(&mut e.labels);

        isLeaf,
        reserveInfo.isDynamic
    );
    // Synthetic block indices start after real blocks and the epilogue block.
    let mut s = Selector {
        e, ralloc, dataSymMap, frameSize: frame.totalSize,
        e, ralloc, frameSize: frame.totalSize,
        reserveOffset: 0, pendingSpill: nil,
        nextSynthBlock: func.blocks.len + 1,
        isDynamic: frame.isDynamic,
    };
    // Record function name for printing.

lib/std/lang/gen/data.rad +8 -8

/// Lay out data symbols for a single section.
/// Initialized data is placed first, then uninitialized, so that only
/// initialized data needs to be written to the output file.
/// Returns the updated offset past all placed symbols.
export fn layoutSection(
    program: *il::Program,
    items: *[il::Data],
    syms: *mut [DataSym],
    count: *mut u32,
    base: u32,
    readOnly: bool
) -> u32 {
    let mut offset: u32 = 0;

    // Initialized data first.
    for i in 0..program.data.len {
        let data = &program.data[i];
    for i in 0..items.len {
        let data = &items[i];
        if data.readOnly == readOnly and not data.isUndefined {
            set offset = mem::alignUp(offset, data.alignment);
            set syms[*count] = DataSym { name: data.name, addr: base + offset };
            set *count += 1;
            set offset += data.size;
        }
    }
    // Uninitialized data after.
    for i in 0..program.data.len {
        let data = &program.data[i];
    for i in 0..items.len {
        let data = &items[i];
        if data.readOnly == readOnly and data.isUndefined {
            set offset = mem::alignUp(offset, data.alignment);
            set syms[*count] = DataSym { name: data.name, addr: base + offset };
            set *count += 1;
            set offset += data.size;


/// Emit data bytes for a single section (read-only or read-write) into `buf`.
/// Iterates initialized data in the IL program, serializing each data item.
/// Returns the total number of bytes written.
export fn emitSection(
    program: *il::Program,
    items: *[il::Data],
    dataSymMap: *DataSymMap,
    fnLabels: *labels::Labels,
    codeBase: u32,
    buf: *mut [u8],
    readOnly: bool
) -> u32 {
    let mut offset: u32 = 0;

    for i in 0..program.data.len {
        let data = &program.data[i];
    for i in 0..items.len {
        let data = &items[i];
        if data.readOnly == readOnly and not data.isUndefined {
            set offset = mem::alignUp(offset, data.alignment);
            assert offset + data.size <= buf.len, "emitSection: buffer overflow";
            for j in 0..data.values.len {
                let v = &data.values[j];

lib/std/lang/il.rad +0 -2

export record Program {
    /// Global data.
    data: *[Data],
    /// Functions.
    fns: *[*Fn],
    /// Index of entry point function, if any.
    defaultFnIdx: ?u32,
}

///////////////////////
// Utility Functions //
///////////////////////

lib/std/lang/lower.rad +88 -37

    debug: bool,
    /// Whether to lower `@test` functions.
    buildTest: bool,
}

/// Role of a lowered function in the final program.
export union FnRole {
    /// Regular function.
    Normal,
    /// Program entry function marked with `@default`.
    Default,
}

/// Function sink used by lowerers that consume functions as they are produced.
export record FnSink {
    /// Opaque context passed to the sink callback.
    ctx: *mut opaque,
    /// Callback invoked for each lowered function.
    emitFn: fn(*mut opaque, *il::Fn, FnRole),
}

/// Destination for functions produced by the lowerer.
export union FnOutput {
    /// Store lowered functions in the provided slice.
    Accumulate(*mut [*il::Fn]),
    /// Send lowered functions to an external consumer immediately.
    Stream(FnSink),
}

/// Module-level lowering context. Shared across all function lowerings.
/// Holds global state like the data section (strings, constants) and provides
/// access to the resolver for type queries.
export record Lowerer {
    /// Arena for IL allocations. All IL nodes are allocated here.
    /// Arena for persistent lowering state, including data and symbol names.
    arena: *mut alloc::Arena,
    /// Arena for allocations owned by the function currently being lowered.
    fnArena: *mut alloc::Arena,
    /// Allocator backed by the arena.
    allocator: alloc::Allocator,
    /// Resolver for type information. Used to query types, symbols, and
    /// compile-time constant values during lowering.
    resolver: *resolver::Resolver,

    /// Current module being lowered.
    currentMod: ?u16,
    /// Global data items (string literals, constants, static arrays).
    /// These become the data sections in the final binary.
    data: *mut [il::Data],
    /// Lowered functions.
    fns: *mut [*il::Fn],
    /// Destination for lowered functions.
    output: FnOutput,
    /// Map of function symbols to qualified names.
    fnSyms: *mut [FnSymEntry],
    /// Global error type tag table. Maps nominal types to unique tags.
    errTags: *mut [ErrTagEntry],
    /// Next error tag to assign (starts at 1; 0 = success).

    self.errTags.append(ErrTagEntry { ty: errType, tag }, self.allocator);

    return tag;
}

/// Emit one function to the active output.
fn emitFunction(self: *mut Lowerer, func: *il::Fn, role: FnRole) {
    match self.output {
        case FnOutput::Accumulate(accumulated) => {
            let mut fns = accumulated;
            fns.append(func, self.allocator);
            set self.output = FnOutput::Accumulate(fns);
        }
        case FnOutput::Stream(sink) => {
            sink.emitFn(sink.ctx, func, role);
        }
    }
}

/// Get the function role for a top-level function declaration.
fn lowerFnRole(isRoot: bool, attrs: ?ast::Attributes) -> FnRole {
    if isRoot and checkAttr(attrs, ast::Attribute::Default) {
        return FnRole::Default;
    }
    return FnRole::Normal;
}

/// Builder for accumulating data values during constant lowering.
record DataValueBuilder {
    allocator: alloc::Allocator,
    values: *mut [il::DataValue],
    /// Whether all values pushed are undefined.

    pkgName: *[u8],
    arena: *mut alloc::Arena
) -> il::Program throws (LowerError) {
    let mut low = Lowerer {
        arena: arena,
        fnArena: arena,
        allocator: alloc::arenaAllocator(arena),
        resolver: res,
        moduleGraph: nil,
        pkgName,
        currentMod: nil,
        data: &mut [],
        fns: &mut [],
        output: FnOutput::Accumulate(&mut []),
        fnSyms: &mut [],
        errTags: &mut [],
        errTagCounter: 1,
        options: LowerOptions { debug: false, buildTest: false },
    };
    let defaultFnIdx = try lowerDecls(&mut low, root, true);
    try lowerDecls(&mut low, root, true);

    return il::Program {
        data: &low.data[..],
        fns: &low.fns[..],
        defaultFnIdx,
    };
    return finalize(&low);
}

/////////////////////////////////
// Multi-Module Lowering API   //
/////////////////////////////////

export fn lowerer(
    res: *resolver::Resolver,
    graph: *module::ModuleGraph,
    pkgName: *[u8],
    arena: *mut alloc::Arena,
    fnArena: *mut alloc::Arena,
    options: LowerOptions
) -> Lowerer {
    return Lowerer {
        arena,
        fnArena,
        allocator: alloc::arenaAllocator(arena),
        resolver: res,
        moduleGraph: graph,
        pkgName,
        currentMod: nil,
        data: &mut [],
        fns: &mut [],
        output: FnOutput::Accumulate(&mut []),
        fnSyms: &mut [],
        errTags: &mut [],
        errTagCounter: 1,
        options,
    };
}

/// Lower a module's AST into the lowerer accumulator.
/// Lower a module's AST into the lowerer.
/// Call this for each module in the package, then use `finalize` to get the program.
export fn lowerModule(
    low: *mut Lowerer,
    moduleId: u16,
    root: *ast::Node,
    isRoot: bool
) -> ?u32 throws (LowerError) {
) throws (LowerError) {
    set low.currentMod = moduleId;
    return try lowerDecls(low, root, isRoot);
    try lowerDecls(low, root, isRoot);
}

/// Lower all top-level declarations in a block.
fn lowerDecls(low: *mut Lowerer, root: *ast::Node, isRoot: bool) -> ?u32 throws (LowerError) {
fn lowerDecls(low: *mut Lowerer, root: *ast::Node, isRoot: bool) throws (LowerError) {
    let case ast::NodeValue::Block(block) = root.value else {
        throw LowerError::ExpectedBlock(root);
    };
    let stmtsList = block.statements;
    let mut defaultFnIdx: ?u32 = nil;

    for node in stmtsList {
        match node.value {
            case ast::NodeValue::FnDecl(decl) => {
                if let f = try lowerFnDecl(low, node, decl) {
                    if isRoot and checkAttr(decl.attrs, ast::Attribute::Default) {
                        set defaultFnIdx = low.fns.len;
                    }
                    low.fns.append(f, low.allocator);
                    let role = lowerFnRole(isRoot, decl.attrs);
                    emitFunction(low, f, role);
                }
            }
            case ast::NodeValue::ConstDecl(decl) => {
                try lowerDataDecl(low, node, decl.value, true);
            }

            case ast::NodeValue::InstanceDecl { traitName, targetType, methods } => {
                try lowerInstanceDecl(low, node, traitName, targetType, methods);
            }
            case ast::NodeValue::MethodDecl { name, receiverName, receiverType, sig, body, .. } => {
                if let f = try lowerMethodDecl(low, node, name, receiverName, sig, body) {
                    low.fns.append(f, low.allocator);
                    emitFunction(low, f, FnRole::Normal);
                }
            }
            else => {},
        }
    }
    return defaultFnIdx;
}

/// Finalize lowering and return the unified IL program.
export fn finalize(low: *Lowerer, defaultFnIdx: ?u32) -> il::Program {
export fn finalize(low: *Lowerer) -> il::Program {
    let mut fns: *mut [*il::Fn] = undefined;
    match low.output {
        case FnOutput::Accumulate(accumulated) => {
            set fns = accumulated;
        }
        case FnOutput::Stream(_) => {
            panic "finalize: cannot finalize streaming lowerer";
        }
    }
    return il::Program {
        data: &low.data[..],
        fns: &low.fns[..],
        defaultFnIdx,
        fns: &fns[..],
    };
}

/////////////////////////////////
// Qualified Name Construction //

    self: *mut Lowerer,
    node: *ast::Node,
    fnType: *resolver::FnType,
    qualName: *[u8]
) -> FnLowerer {
    let loopStack = try! alloc::allocSlice(self.arena, @sizeOf(LoopCtx), @alignOf(LoopCtx), MAX_LOOP_DEPTH) as *mut [LoopCtx];
    let loopStack = try! alloc::allocSlice(self.fnArena, @sizeOf(LoopCtx), @alignOf(LoopCtx), MAX_LOOP_DEPTH) as *mut [LoopCtx];

    let mut fnLow = FnLowerer {
        low: self,
        allocator: alloc::arenaAllocator(self.arena),
        allocator: alloc::arenaAllocator(self.fnArena),
        fnType: fnType,
        fnName: qualName,
        vars: &mut [],
        params: &mut [],
        blockData: &mut [],

    // as the first argument; the callee writes the return value into it.
    if requiresReturnParam(fnType) and not isExtern {
        set fnLow.returnReg = nextReg(&mut fnLow);
    }
    let lowParams = try lowerParams(&mut fnLow, *fnType, decl.sig.params, nil);
    let func = try! alloc::alloc(self.arena, @sizeOf(il::Fn), @alignOf(il::Fn)) as *mut il::Fn;
    let func = try! alloc::alloc(self.fnArena, @sizeOf(il::Fn), @alignOf(il::Fn)) as *mut il::Fn;

    set *func = il::Fn {
        name: qualName,
        params: lowParams,
        returnType: undefined,

            throw LowerError::ExpectedIdentifier;
        };
        let qualName = instanceMethodName(self, nil, typeName, mName);
        let func = try lowerMethod(self, methodNode, qualName, receiverName, sig, body)
            else continue;
        self.fns.append(func, self.allocator);
        emitFunction(self, func, FnRole::Normal);

        let method = resolver::findTraitMethod(traitInfo, mName)
            else panic "lowerInstanceDecl: method not found in trait";

        set methodNames[method.index] = qualName;

    let mut fnLow = fnLowerer(self, node, fnType, qualName);
    if requiresReturnParam(fnType) {
        set fnLow.returnReg = nextReg(&mut fnLow);
    }
    let lowParams = try lowerParams(&mut fnLow, *fnType, sig.params, receiverName);
    let func = try! alloc::alloc(self.arena, @sizeOf(il::Fn), @alignOf(il::Fn)) as *mut il::Fn;
    let func = try! alloc::alloc(self.fnArena, @sizeOf(il::Fn), @alignOf(il::Fn)) as *mut il::Fn;

    set *func = il::Fn {
        name: qualName,
        params: lowParams,
        returnType: ilType(self, *fnType.returnType),

/// This ensures unique labels like `@then0`, `@then1`, etc.
fn labelWithSuffix(self: *mut FnLowerer, base: *[u8], suffix: u32) -> *[u8] throws (LowerError) {
    let mut digits: [u8; fmt::U32_STR_LEN] = undefined;
    let suffixText = fmt::formatU32(suffix, &mut digits[..]);
    let totalLen = base.len + suffixText.len;
    let buf = try! alloc::allocSlice(self.low.arena, 1, 1, totalLen) as *mut [u8];
    let buf = try! alloc::allocSlice(self.low.fnArena, 1, 1, totalLen) as *mut [u8];

    try! mem::copy(&mut buf[..base.len], base);
    try! mem::copy(&mut buf[base.len..totalLen], suffixText);

    return &buf[..totalLen];

/// be switched to via [`switchToBlock`] before instructions can be emitted.
fn createBlock(self: *mut FnLowerer, labelBase: *[u8]) -> BlockId throws (LowerError) {
    let label = try nextLabel(self, labelBase);
    let id = BlockId(self.blockData.len);
    let varCount = self.fnType.localCount;
    let vars = try! alloc::allocSlice(self.low.arena, @sizeOf(?il::Val), @alignOf(?il::Val), varCount) as *mut [?il::Val];
    let vars = try! alloc::allocSlice(self.low.fnArena, @sizeOf(?il::Val), @alignOf(?il::Val), varCount) as *mut [?il::Val];

    for i in 0..varCount {
        set vars[i] = nil;
    }
    self.blockData.append(BlockData {

}

/// Finalize all blocks and return the block array.
fn finalizeBlocks(self: *mut FnLowerer) -> *[il::Block] throws (LowerError) {
    let blocks = try! alloc::allocSlice(
        self.low.arena, @sizeOf(il::Block), @alignOf(il::Block), self.blockData.len
        self.low.fnArena, @sizeOf(il::Block), @alignOf(il::Block), self.blockData.len
    ) as *mut [il::Block];

    for i in 0..self.blockData.len {
        let data = &self.blockData[i];


    return block;
}

/// Allocate a slice of values in the lowering arena.
fn allocVals(self: *mut FnLowerer, len: u32) -> *mut [il::Val] throws (LowerError) {
    return try! alloc::allocSlice(self.low.arena, @sizeOf(il::Val), @alignOf(il::Val), len) as *mut [il::Val];
    return try! alloc::allocSlice(self.low.fnArena, @sizeOf(il::Val), @alignOf(il::Val), len) as *mut [il::Val];
}

/// Allocate a single-value slice in the lowering arena.
fn allocVal(self: *mut FnLowerer, val: il::Val) -> *mut [il::Val] throws (LowerError) {
    let args = try allocVals(self, 1);

fn growArgs(self: *mut FnLowerer, args: *mut [il::Val], capacity: u32) -> *mut [il::Val] {
    if args.len >= capacity {
        return args;
    }
    let newArgs = try! alloc::allocSlice(
        self.low.arena, @sizeOf(il::Val), @alignOf(il::Val), capacity
        self.low.fnArena, @sizeOf(il::Val), @alignOf(il::Val), capacity
    ) as *mut [il::Val];

    for arg, i in args {
        set newArgs[i] = arg;
    }

        return &[];
    }
    assert fnType.paramTypes.len as u32 <= resolver::MAX_FN_PARAMS;

    let params = try! alloc::allocSlice(
        self.low.arena, @sizeOf(il::Param), @alignOf(il::Param), totalLen
        self.low.fnArena, @sizeOf(il::Param), @alignOf(il::Param), totalLen
    ) as *mut [il::Param];

    if let reg = self.returnReg {
        set params[0] = il::Param { value: reg, type: il::Type::W64 };
    }


    // Create comparison blocks for each non-void variant and build switch cases.
    // Void variants jump directly to merge with `true`.
    let trueArgs = try allocVal(self, il::Val::Imm(1));
    let cases = try! alloc::allocSlice(
        self.low.arena, @sizeOf(il::SwitchCase), @alignOf(il::SwitchCase), unionInfo.variants.len as u32
        self.low.fnArena, @sizeOf(il::SwitchCase), @alignOf(il::SwitchCase), unionInfo.variants.len as u32
    ) as *mut [il::SwitchCase];

    let mut caseBlocks: [?BlockId; resolver::MAX_UNION_VARIANTS] = undefined;
    for variant, i in unionInfo.variants {
        if variant.valueType == resolver::Type::Void {

134	134		record RootModule {
135	135		entry: *module::ModuleEntry,
136	136		ast: *mut ast::Node,
137	137		}
138	138
	139	+	/// State carried by the streaming lowerer/codegen callback.
	140	+	record CodegenSinkContext {
	141	+	/// RV64 generator receiving lowered functions.
	142	+	generator: *mut rv64::Generator,
	143	+	/// Arena used for codegen scratch allocations.
	144	+	codegenArena: *mut alloc::Arena,
	145	+	/// Arena holding the current function's lowered IL.
	146	+	fnArena: *mut alloc::Arena,
	147	+	}
	148	+
	149	+	/// Entry handling for streamed code generation.
	150	+	union CodegenEntryMode {
	151	+	/// Do not reserve an entry jump.
	152	+	None,
	153	+	/// Reserve and patch an entry jump to the `@default` function.
	154	+	DefaultEntry,
	155	+	}
	156	+
	157	+	/// Options controlling streamed lowering and code generation.
	158	+	record CodegenOptions {
	159	+	/// Optional output path used for progress logging.
	160	+	logPath: ?*[u8],
	161	+	/// Whether to emit debug source locations.
	162	+	debug: bool,
	163	+	/// How the generated program should handle entry.
	164	+	entryMode: CodegenEntryMode,
	165	+	}
	166	+
139	167		/// Print a log line for the given package.
140	168		fn pkgLog(pkg: package::Package, msg: [*[u8]]) {
141	169		io::printError("radiance: ");
142	170		io::printError(pkg.name);
143	171		io::printError(": ");

398	426		};
399	427		let entryPkg = &ctx.packages[entryIdx];
400	428
401	429		// Create the lowerer accumulator using entry package's name.
402	430		let options = lower::LowerOptions { debug: ctx.debug, buildTest: ctx.config.buildTest };
403		-	let mut low = lower::lowerer(res, &ctx.graph, entryPkg.name, &mut res.arena, options);
	431	+	let mut low = lower::lowerer(
	432	+	res, &ctx.graph, entryPkg.name, &mut res.arena, &mut res.arena, options
	433	+	);
	434	+	try lowerAllPackagesInto(ctx, res, &mut low);
	435	+
	436	+	// Finalize and return the unified program.
	437	+	return lower::finalize(&low);
	438	+	}
404	439
	440	+	/// Lower all packages into an existing lowerer.
	441	+	fn lowerAllPackagesInto(
	442	+	ctx: *mut CompileContext,
	443	+	res: *mut resolver::Resolver,
	444	+	low: *mut lower::Lowerer
	445	+	) throws (Error) {
	446	+	let entryIdx = ctx.entryPkgIdx else {
	447	+	panic "lowerAllPackagesInto: no entry package";
	448	+	};
405	449		// Lower all packages except entry.
406	450		for i in 0..ctx.packageCount {
407	451		if i <> entryIdx {
408		-	try lowerPackage(ctx, res, &mut low, &mut ctx.packages[i], false);
	452	+	try lowerPackage(ctx, res, low, &mut ctx.packages[i], false);
409	453		}
410	454		}
411	455		// Lower entry package.
412		-	let defaultFn = try lowerPackage(ctx, res, &mut low, &mut ctx.packages[entryIdx], true);
413		-
414		-	// Finalize and return the unified program.
415		-	return lower::finalize(&low, defaultFn);
	456	+	try lowerPackage(ctx, res, low, &mut ctx.packages[entryIdx], true);
416	457		}
417	458
418	459		/// Lower all modules in a package into the lowerer accumulator.
419	460		fn lowerPackage(
420	461		ctx: *CompileContext,
421	462		res: *mut resolver::Resolver,
422	463		low: *mut lower::Lowerer,
423	464		pkg: *mut package::Package,
424	465		isEntry: bool
425		-	) -> ?u32 throws (Error) {
	466	+	) throws (Error) {
426	467		let rootId = pkg.rootModuleId else {
427	468		io::printError("radiance: no root module found\n");
428	469		throw Error::Other;
429	470		};
430	471		// Set lowerer's package context for qualified name generation.
431	472		// TODO: We shouldn't have to call this manually.
432	473		lower::setPackage(low, &ctx.graph, pkg.name);
433	474
434		-	return try lowerModuleTreeInto(ctx, low, &ctx.graph, rootId, isEntry, pkg);
	475	+	try lowerModuleTreeInto(ctx, low, &ctx.graph, rootId, isEntry, pkg);
435	476		}
436	477
437	478		/// Recursively lower a module and all its children into the accumulator.
438	479		fn lowerModuleTreeInto(
439	480		ctx: *CompileContext,
440	481		low: *mut lower::Lowerer,
441	482		graph: *module::ModuleGraph,
442	483		modId: u16,
443	484		isRoot: bool,
444	485		pkg: *package::Package
445		-	) -> ?u32 throws (Error) {
	486	+	) throws (Error) {
446	487		let entry = module::get(graph, modId) else {
447	488		io::printError("radiance: module entry not found\n");
448	489		throw Error::Other;
449	490		};
450	491		let modAst = entry.ast else {
451	492		io::printError("radiance: module has no AST\n");
452	493		throw Error::Other;
453	494		};
454	495		pkgLog(pkg, &["lowering", "(", entry.filePath, ")", ".."]);
455	496
456		-	let defaultFn = try lower::lowerModule(low, modId, modAst, isRoot) catch err {
	497	+	try lower::lowerModule(low, modId, modAst, isRoot) catch err {
457	498		io::printError("radiance: internal error during lowering: ");
458	499		lower::printError(err);
459	500		io::printError("\n");
460	501		throw Error::Other;
461	502		};
462	503		// Recurse into children.
463	504		for i in 0..entry.childrenLen {
464	505		let childId = module::childAt(entry, i);
465	506		try lowerModuleTreeInto(ctx, low, graph, childId, false, pkg);
466	507		}
467		-	return defaultFn;
468	508		}
469	509
470	510		/// Build a scope access chain: a::b::c from a slice of identifiers.
471	511		fn synthScopeAccess(arena: mut ast::NodeArena, path: [[u8]]) -> ast::Node {
472	512		let mut result = ast::synthNode(

768	808		throw Error::Other;
769	809		}
770	810		return res;
771	811		}
772	812
	813	+	/// Emit one lowered function to RV64 codegen and reclaim its IL arena.
	814	+	fn generateLoweredFn(ctxPtr: mut opaque, func: il::Fn, role: lower::FnRole) {
	815	+	let ctx = ctxPtr as *mut CodegenSinkContext;
	816	+
	817	+	match role {
	818	+	case lower::FnRole::Default => {
	819	+	match ctx.generator.entryPatch {
	820	+	case rv64::EntryPatch::Reserved(_) => {
	821	+	set ctx.generator.entryPatch = rv64::EntryPatch::Reserved(func.name);
	822	+	}
	823	+	else => panic "generateLoweredFn: entry jump was not reserved",
	824	+	}
	825	+	}
	826	+	else => {}
	827	+	}
	828	+	rv64::generateFunction(ctx.generator, func, ctx.codegenArena);
	829	+	alloc::reset(ctx.fnArena);
	830	+	}
	831	+
	832	+	/// Lower all packages while streaming each lowered function into RV64 codegen.
	833	+	fn lowerAndGenerateAllPackages(
	834	+	ctx: *mut CompileContext,
	835	+	res: *mut resolver::Resolver,
	836	+	fnArena: *mut alloc::Arena,
	837	+	codegenOptions: CodegenOptions
	838	+	) -> rv64::Program throws (Error) {
	839	+	let entryIdx = ctx.entryPkgIdx else {
	840	+	panic "lowerAndGenerateAllPackages: no entry package";
	841	+	};
	842	+	let entryPkg = &ctx.packages[entryIdx];
	843	+	let options = lower::LowerOptions { debug: ctx.debug, buildTest: ctx.config.buildTest };
	844	+	let storage = rv64::Storage {
	845	+	dataSyms: &mut CODEGEN_DATA_SYMS[..],
	846	+	dataSymEntries: &mut CODEGEN_DATA_SYM_ENTRIES[..],
	847	+	};
	848	+	let mut entryPatch = rv64::EntryPatch::None;
	849	+	match codegenOptions.entryMode {
	850	+	case CodegenEntryMode::DefaultEntry => {
	851	+	set entryPatch = rv64::EntryPatch::Reserved(nil);
	852	+	}
	853	+	else => {}
	854	+	}
	855	+	let mut generator = rv64::beginProgram(
	856	+	rv64::ProgramOptions { entryPatch, debug: codegenOptions.debug },
	857	+	&mut res.arena
	858	+	);
	859	+	let mut codegenCtx = CodegenSinkContext {
	860	+	generator: &mut generator,
	861	+	codegenArena: &mut res.arena,
	862	+	fnArena,
	863	+	};
	864	+	let mut low = lower::lowerer(
	865	+	res, &ctx.graph, entryPkg.name, &mut res.arena, fnArena, options
	866	+	);
	867	+	set low.output = lower::FnOutput::Stream(lower::FnSink {
	868	+	ctx: &mut codegenCtx as *mut opaque,
	869	+	emitFn: generateLoweredFn,
	870	+	});
	871	+	try lowerAllPackagesInto(ctx, res, &mut low);
	872	+
	873	+	match generator.entryPatch {
	874	+	case rv64::EntryPatch::Reserved(targetName) => {
	875	+	if targetName == nil {
	876	+	io::printError("radiance: fatal: no default function found\n");
	877	+	throw Error::Other;
	878	+	}
	879	+	}
	880	+	else => {}
	881	+	}
	882	+	if let path = codegenOptions.logPath {
	883	+	pkgLog(entryPkg, &["generating code", "(", path, ")", ".."]);
	884	+	}
	885	+	return rv64::finishProgram(&mut generator, &low.data[..], storage, &mut RO_DATA_BUF[..], &mut RW_DATA_BUF[..]);
	886	+	}
	887	+
773	888		/// Lower, optionally dump, and optionally generate binary output.
774		-	fn compile(ctx: mut CompileContext, res: mut resolver::Resolver) throws (Error) {
	889	+	fn compile(
	890	+	ctx: *mut CompileContext,
	891	+	res: *mut resolver::Resolver,
	892	+	fnArena: *mut alloc::Arena
	893	+	) throws (Error) {
775	894		let entryPkg = try getEntryPackage(ctx);
776		-
777		-	// Lower all packages into a single unified IL program.
778		-	// Dependencies must be lowered before the packages that use them.
779		-	let program = try lowerAllPackages(ctx, res);
780	895		let mut out = sexpr::Output::Stdout;
781	896
782	897		if ctx.dump == Dump::Il {
	898	+	// Lower all packages into a single unified IL program for dumping.
	899	+	let program = try lowerAllPackages(ctx, res);
783	900		il::printer::printProgram(&mut out, &mut res.arena, &program);
784	901		io::print("\n");
785	902		return;
786	903		}
787	904		if ctx.dump == Dump::Asm {
788		-	// TODO: Why do we generate code in two places?
789		-	let result = rv64::generate(&program, rv64::Storage { dataSyms: &mut CODEGEN_DATA_SYMS[..], dataSymEntries: &mut CODEGEN_DATA_SYM_ENTRIES[..] }, &mut RO_DATA_BUF[..], &mut RW_DATA_BUF[..], &mut res.arena, false);
	905	+	let result = try lowerAndGenerateAllPackages(ctx, res, fnArena, CodegenOptions {
	906	+	logPath: nil,
	907	+	debug: false,
	908	+	entryMode: CodegenEntryMode::None,
	909	+	});
790	910		printer::printCodeTo(&mut out, entryPkg.name, result.code, result.funcs, &mut res.arena);
791	911		io::print("\n");
	912	+
792	913		return;
793	914		}
794	915		// Generate binary output if path specified.
795	916		let outPath = ctx.outputPath else {
	917	+	try lowerAllPackages(ctx, res);
796	918		return;
797	919		};
798		-	// Check that we have an entry point.
799		-	if program.defaultFnIdx == nil {
800		-	io::printError("radiance: fatal: no default function found\n");
801		-	throw Error::Other;
802		-	}
803		-	pkgLog(entryPkg, &["generating code", "(", outPath, ")", ".."]);
	920	+	let result = try lowerAndGenerateAllPackages(ctx, res, fnArena, CodegenOptions {
	921	+	logPath: outPath,
	922	+	debug: ctx.debug,
	923	+	entryMode: CodegenEntryMode::DefaultEntry,
	924	+	});
804	925
805		-	let storage = rv64::Storage { dataSyms: &mut CODEGEN_DATA_SYMS[..], dataSymEntries: &mut CODEGEN_DATA_SYM_ENTRIES[..] };
806		-	let result = rv64::generate(&program, storage, &mut RO_DATA_BUF[..], &mut RW_DATA_BUF[..], &mut res.arena, ctx.debug);
807	926		if not writeCode(result.code, outPath) {
808	927		io::printError("radiance: fatal: failed to write output file\n");
809	928		throw Error::Other;
810	929		}
811	930		// Write data files.

845	964		}
846	965		// Run resolution phase.
847	966		let mut res = try runResolver(&mut ctx, arena.nextId) catch {
848	967		return 1;
849	968		};
	969	+	let mut fnArena = alloc::new(alloc::remainingBuf(&mut arena.arena));
	970	+
850	971		// Lower, dump, and/or generate output.
851		-	try compile(&mut ctx, &mut res) catch {
	972	+	try compile(&mut ctx, &mut res, &mut fnArena) catch {
852	973		return 1;
853	974		};
854	975		return 0;
855	976		}

168	168		rwDataSize: u32,
169	169		/// Debug entries mapping PCs to source locations. Empty when debug is off.
170	170		debugEntries: *[types::DebugEntry],
171	171		}
172	172
173		-	/// Generate code for an IL program.
174		-	export fn generate(
175		-	program: *il::Program,
	173	+	/// Entry jump patching requested for the generated program.
	174	+	export union EntryPatch {
	175	+	/// No entry jump is emitted.
	176	+	None,
	177	+	/// Reserve code slot zero for a jump to the default function.
	178	+	Reserved(?*[u8]),
	179	+	}
	180	+
	181	+	/// Options controlling incremental RV64 program generation.
	182	+	export record ProgramOptions {
	183	+	/// Entry jump patching mode.
	184	+	entryPatch: EntryPatch,
	185	+	/// Whether to emit debug source locations.
	186	+	debug: bool,
	187	+	}
	188	+
	189	+	/// State for incremental RV64 program generation.
	190	+	///
	191	+	/// The generator owns global codegen state that must survive across function
	192	+	/// emission. Function-local scratch stays outside this record so callers can
	193	+	/// reclaim it after each function.
	194	+	export record Generator {
	195	+	/// Binary emitter and relocation state.
	196	+	e: emit::Emitter,
	197	+	/// Entry jump patching state.
	198	+	entryPatch: EntryPatch,
	199	+	}
	200	+
	201	+	/// Begin RV64 code generation for a program's global state.
	202	+	export fn beginProgram(
	203	+	options: ProgramOptions,
	204	+	arena: *mut alloc::Arena
	205	+	) -> Generator {
	206	+	let mut e = try! emit::emitter(arena, options.debug);
	207	+
	208	+	// Emit placeholder entry jump when requested.
	209	+	// We'll patch this at the end once we know where the function is.
	210	+	match options.entryPatch {
	211	+	case EntryPatch::Reserved(_) => {
	212	+	emit::emit(&mut e, encode::nop()); // Placeholder for two-instruction jump.
	213	+	emit::emit(&mut e, encode::nop()); //
	214	+	}
	215	+	else => {}
	216	+	}
	217	+
	218	+	return Generator {
	219	+	e,
	220	+	entryPatch: options.entryPatch,
	221	+	};
	222	+	}
	223	+
	224	+	/// Generate code for one IL function.
	225	+	export fn generateFunction(
	226	+	generator: *mut Generator,
	227	+	func: *il::Fn,
	228	+	arena: *mut alloc::Arena
	229	+	) {
	230	+	if func.isExtern {
	231	+	return;
	232	+	}
	233	+	let checkpoint = alloc::save(arena);
	234	+	let config = targetConfig();
	235	+	let ralloc = try! regalloc::allocate(func, &config, arena);
	236	+
	237	+	isel::selectFn(&mut generator.e, &ralloc, func);
	238	+
	239	+	// Reclaim unused memory after instruction selection.
	240	+	alloc::restore(arena, checkpoint);
	241	+	}
	242	+
	243	+	/// Finish RV64 code generation and return the emitted program.
	244	+	export fn finishProgram(
	245	+	generator: *mut Generator,
	246	+	globalData: *[il::Data],
176	247		storage: Storage,
177	248		roDataBuf: *mut [u8],
178		-	rwDataBuf: *mut [u8],
179		-	arena: *mut alloc::Arena,
180		-	debug: bool
	249	+	rwDataBuf: *mut [u8]
181	250		) -> Program {
182		-	let mut e = try! emit::emitter(arena, debug);
183		-	let config = targetConfig();
184		-
185		-	// Build data map.
	251	+	// Build data map after function lowering. Function-local literals can add
	252	+	// global data while functions are lowered, so final layout belongs here.
186	253		let mut dataSymCount: u32 = 0;
	254	+	let roLayoutSize = data::layoutSection(globalData, storage.dataSyms, &mut dataSymCount, RO_DATA_BASE, true);
	255	+	data::layoutSection(globalData, storage.dataSyms, &mut dataSymCount, RW_DATA_BASE, false);
187	256
188		-	let roLayoutSize = data::layoutSection(program, storage.dataSyms, &mut dataSymCount, RO_DATA_BASE, true);
189		-	data::layoutSection(program, storage.dataSyms, &mut dataSymCount, RW_DATA_BASE, false);
190		-
191		-	// Build hash-indexed data symbol map for O(1) lookups.
192	257		let dataSyms = &storage.dataSyms[..dataSymCount];
193		-	let mut dataSymMap = data::buildMap(dataSyms, storage.dataSymEntries);
194		-
195		-	// Code base address: code follows read-only data, aligned to dword boundary.
	258	+	let dataSymMap = data::buildMap(dataSyms, storage.dataSymEntries);
196	259		let codeBase = mem::alignUp(RO_DATA_BASE + roLayoutSize, DWORD_SIZE as u32);
197	260
198		-	// Emit placeholder entry jump to default function if there is one.
199		-	// We'll patch this at the end once we know where the function is.
200		-	let mut defaultName: ?*[u8] = nil;
201		-	if let defIdx = program.defaultFnIdx {
202		-	set defaultName = program.fns[defIdx].name;
203		-	emit::emit(&mut e, encode::nop()); // Placeholder for two-instruction jump.
204		-	emit::emit(&mut e, encode::nop()); //
205		-	}
206		-
207		-	// Generate code for all functions.
208		-	for i in 0..program.fns.len {
209		-	let func = program.fns[i];
210		-	if not func.isExtern {
211		-	let checkpoint = alloc::save(arena);
212		-	let ralloc = try! regalloc::allocate(func, &config, arena);
213		-	isel::selectFn(&mut e, &dataSymMap, &ralloc, func);
	261	+	match generator.entryPatch {
	262	+	case EntryPatch::Reserved(targetName) => {
	263	+	let target = targetName else {
	264	+	panic "finishProgram: entry jump reserved without default function";
	265	+	};
	266	+	let offset = emit::branchOffsetToFunc(&generator.e, 0, target);
	267	+	let s = emit::splitImm(offset);
214	268
215		-	// Reclaim unused memory after instruction selection.
216		-	alloc::restore(arena, checkpoint);
	269	+	emit::patch(&mut generator.e, 0, encode::auipc(SCRATCH1, s.hi));
	270	+	emit::patch(&mut generator.e, 1, encode::jalr(ZERO, SCRATCH1, s.lo));
217	271		}
218		-	}
219		-	// Patch entry jump now that we know where the default function is.
220		-	// Nb. we use a two-instruction jump even if it isn't always needed.
221		-	if let target = defaultName {
222		-	let offset = emit::branchOffsetToFunc(&e, 0, target);
223		-	let s = emit::splitImm(offset);
224		-
225		-	emit::patch(&mut e, 0, encode::auipc(SCRATCH1, s.hi));
226		-	emit::patch(&mut e, 1, encode::jalr(ZERO, SCRATCH1, s.lo));
	272	+	else => {}
227	273		}
228	274		// Patch function calls and address loads now that all functions are emitted.
229		-	emit::patchCalls(&mut e);
230		-	emit::patchAddrLoads(&mut e);
	275	+	emit::patchCalls(&mut generator.e);
	276	+	emit::patchAddrLoads(&mut generator.e, &dataSymMap);
231	277
232	278		// Emit data sections.
233		-	let roDataSize = data::emitSection(program, &dataSymMap, &e.labels, codeBase, roDataBuf, true);
234		-	let rwDataSize = data::emitSection(program, &dataSymMap, &e.labels, codeBase, rwDataBuf, false);
	279	+	let roDataSize = data::emitSection(globalData, &dataSymMap, &generator.e.labels, codeBase, roDataBuf, true);
	280	+	let rwDataSize = data::emitSection(globalData, &dataSymMap, &generator.e.labels, codeBase, rwDataBuf, false);
235	281
236	282		return Program {
237		-	code: emit::getCode(&e),
238		-	funcs: emit::getFuncs(&e),
	283	+	code: emit::getCode(&generator.e),
	284	+	funcs: emit::getFuncs(&generator.e),
239	285		roDataSize,
240	286		rwDataSize,
241		-	debugEntries: emit::getDebugEntries(&e),
	287	+	debugEntries: emit::getDebugEntries(&generator.e),
242	288		};
243	289		}

33	33		constant MAX_TESTS: u32 = 1024;
34	34
35	35		/// Temporary arena size (32 MB) - retains all parsed AST until resolution.
36	36		/// Used for: AST during parsing, then codegen scratch space.
37	37		constant TEMP_ARENA_SIZE: u32 = 33554432;
38		-	/// Main arena size (128 MB) - lives throughout compilation.
39		-	/// Used for: resolver data, types, symbols, lowered IL.
40		-	constant MAIN_ARENA_SIZE: u32 = 134217728;
	38	+	/// Main arena size (64 MB) - lives throughout compilation.
	39	+	/// Used for: resolver data, types, symbols, global IL data, and codegen output.
	40	+	constant MAIN_ARENA_SIZE: u32 = 67108864;
41	41
42	42		/// Temporary storage arena - reusable between phases.
43	43		static TEMP_ARENA: [u8; TEMP_ARENA_SIZE] = undefined;
44	44		/// Main storage arena - persists throughout compilation.
45	45		static MAIN_ARENA: [u8; MAIN_ARENA_SIZE] = undefined;

3	3		//! Emits RV64 machine code as `u32` list.
4	4
5	5		use std::lang::il;
6	6		use std::lang::alloc;
7	7		use std::lang::gen;
	8	+	use std::lang::gen::data;
8	9		use std::lang::gen::labels;
9	10		use std::lang::gen::types;
10	11		use std::collections::dict;
11	12		use std::mem;
12	13

14	15
15	16		/// Maximum number of instructions in code buffer.
16	17		constant MAX_INSTRS: u32 = 2097152;
17	18		/// Maximum code length before byte offset overflows signed 32-bits.
18	19		constant MAX_CODE_LEN: u32 = 0x7FFFFFFF / super::INSTR_SIZE as u32;
	20	+	/// Maximum positive value encodable by a signed 32-bit address calculation.
	21	+	constant MAX_I32_ADDR: u32 = 0x7FFFFFFF;
19	22		/// Maximum number of pending branches awaiting patching.
20	23		constant MAX_PENDING: u32 = 65536;
21	24		/// Maximum number of function entries.
22	25		constant MAX_FUNCS: u32 = 4096;
23	26		/// Maximum number of debug entries.

53	56		index: u32,
54	57		/// Target function name.
55	58		target: *[u8],
56	59		}
57	60
58		-	/// Function address load that needs offset patching.
59		-	/// Used when taking a function's address as a value.
	61	+	/// Address load that needs patching after layout is known.
60	62		export record PendingAddrLoad {
61	63		/// Index in code buffer where the load was emitted.
62	64		index: u32,
63		-	/// Target function name.
	65	+	/// Target function or data symbol name.
64	66		target: *[u8],
65	67		/// Destination register.
66	68		rd: gen::Reg,
	69	+	/// Whether this is an absolute data symbol address.
	70	+	isData: bool,
67	71		}
68	72
69	73		/// Adjusted base register and offset for addressing.
70	74		export record AdjustedOffset {
71	75		/// Base register.

287	291		assert e.pendingAddrLoadsLen < e.pendingAddrLoads.len, "recordAddrLoad: buffer full";
288	292		set e.pendingAddrLoads[e.pendingAddrLoadsLen] = PendingAddrLoad {
289	293		index: e.codeLen,
290	294		target,
291	295		rd: rd,
	296	+	isData: false,
292	297		};
293	298		set e.pendingAddrLoadsLen += 1;
294	299
295	300		emit(e, encode::nop()); // Placeholder for AUIPC.
296	301		emit(e, encode::nop()); // Placeholder for ADDI.
297	302		}
298	303
	304	+	/// Record a data address load needing later patching.
	305	+	/// Uses an absolute 32-bit load sequence matching the current data memory map.
	306	+	export fn recordDataAddrLoad(e: mut Emitter, target: [u8], rd: gen::Reg) {
	307	+	assert e.pendingAddrLoadsLen < e.pendingAddrLoads.len, "recordDataAddrLoad: buffer full";
	308	+	set e.pendingAddrLoads[e.pendingAddrLoadsLen] = PendingAddrLoad {
	309	+	index: e.codeLen,
	310	+	target,
	311	+	rd: rd,
	312	+	isData: true,
	313	+	};
	314	+	set e.pendingAddrLoadsLen += 1;
	315	+
	316	+	emit(e, encode::nop()); // Placeholder for LUI.
	317	+	emit(e, encode::nop()); // Placeholder for ADDIW.
	318	+	}
	319	+
299	320		/// Patch local branches and clear the pending list.
300	321		///
301	322		/// Called after each function.
302	323		///
303	324		/// Uses two-instruction sequences: short branches use `branch` and `nop`,

370	391		// `JALR ra, scratch, lo(offset)`.
371	392		patch(e, p.index + 1, encode::jalr(super::RA, super::SCRATCH1, s.lo));
372	393		}
373	394		}
374	395
375		-	/// Patch all pending function address loads.
376		-	/// Called after all functions have been generated.
377		-	/// Uses PC-relative addresses up to 2GB away.
378		-	export fn patchAddrLoads(e: *mut Emitter) {
	396	+	/// Patch all pending function and data address loads.
	397	+	/// Called after all functions have been generated and data layout is known.
	398	+	export fn patchAddrLoads(e: mut Emitter, dataSymMap: data::DataSymMap) {
379	399		for i in 0..e.pendingAddrLoadsLen {
380	400		let p = e.pendingAddrLoads[i];
	401	+	if p.isData {
	402	+	let addr = data::lookupAddr(dataSymMap, p.target) else {
	403	+	panic "patchAddrLoads: data symbol not found";
	404	+	};
	405	+	assert addr <= MAX_I32_ADDR, "patchAddrLoads: data address too large";
	406	+	let s = splitImm(addr as i32);
	407	+
	408	+	patch(e, p.index, encode::lui(p.rd, s.hi));
	409	+	patch(e, p.index + 1, encode::addiw(p.rd, p.rd, s.lo));
	410	+
	411	+	continue;
	412	+	}
	413	+
381	414		let offset = branchOffsetToFunc(e, p.index, p.target);
382	415		let s = splitImm(offset);
383		-
384	416		// `AUIPC rd, hi(offset)`.
385	417		patch(e, p.index, encode::auipc(p.rd, s.hi));
386	418		// `ADDI rd, rd, lo(offset)`.
387	419		patch(e, p.index + 1, encode::addi(p.rd, p.rd, s.lo));
388	420		}

70	69		/// Shift operation.
71	70		union ShiftOp { Sll, Srl, Sra }
72	71		/// Compare operation.
73	72		union CmpOp { Slt, Ult }
74	73
75		-	/// Selector errors.
76		-	export union SelectorError {
77		-	Internal,
78		-	}
79		-
80	74		/// A pending spill store to be flushed after instruction selection.
81	75		record PendingSpill {
82	76		/// The SSA register that was spilled.
83	77		ssa: il::Reg,
84	78		/// The physical register holding the value to store.

93	87		export record Selector {
94	88		/// Emitter for outputting instructions.
95	89		e: *mut emit::Emitter,
96	90		/// Register allocation result.
97	91		ralloc: *regalloc::AllocResult,
98		-	/// Hash-indexed data symbol map.
99		-	dataSymMap: *data::DataSymMap,
100	92		/// Total stack frame size.
101	93		frameSize: i32,
102	94		/// Running offset into the reserve region of the frame.
103	95		/// Tracks current position within the pre-allocated reserve slots.
104	96		reserveOffset: i32,

161	153		return scratch;
162	154		}
163	155		return getReg(s, ssa);
164	156		}
165	157
166		-	/// Look up symbol address in data map.
167		-	fn lookupDataSym(s: Selector, name: [u8]) -> u32 throws (SelectorError) {
168		-	let addr = data::lookupAddr(s.dataSymMap, name) else {
169		-	throw SelectorError::Internal;
170		-	};
171		-	return addr;
172		-	}
173		-
174	158		/// Resolve an IL value to the physical register holding it.
175	159		/// For non-spilled register values, returns the physical register directly.
176	160		/// For immediates, symbols, and spilled registers, materializes into `scratch`.
177	161		fn resolveVal(s: *mut Selector, scratch: gen::Reg, val: il::Val) -> gen::Reg {
178	162		match val {

185	169		}
186	170		emit::loadImm(s.e, scratch, imm);
187	171		return scratch;
188	172		},
189	173		case il::Val::DataSym(name) => {
190		-	let addr = try lookupDataSym(s, name) catch {
191		-	panic "resolveVal: data symbol not found";
192		-	};
193		-	emit::loadImm(s.e, scratch, addr as i64);
	174	+	emit::recordDataAddrLoad(s.e, name, scratch);
194	175		return scratch;
195	176		},
196	177		case il::Val::FnAddr(name) => {
197	178		emit::recordAddrLoad(s.e, name, scratch);
198	179		return scratch;

304	285		}
305	286
306	287		/// Select instructions for a function.
307	288		export fn selectFn(
308	289		e: *mut emit::Emitter,
309		-	dataSymMap: *data::DataSymMap,
310	290		ralloc: *regalloc::AllocResult,
311	291		func: *il::Fn
312	292		) {
313	293		// Reset block offsets for this function.
314	294		labels::resetBlocks(&mut e.labels);

323	303		isLeaf,
324	304		reserveInfo.isDynamic
325	305		);
326	306		// Synthetic block indices start after real blocks and the epilogue block.
327	307		let mut s = Selector {
328		-	e, ralloc, dataSymMap, frameSize: frame.totalSize,
	308	+	e, ralloc, frameSize: frame.totalSize,
329	309		reserveOffset: 0, pendingSpill: nil,
330	310		nextSynthBlock: func.blocks.len + 1,
331	311		isDynamic: frame.isDynamic,
332	312		};
333	313		// Record function name for printing.

34	34		/// Lay out data symbols for a single section.
35	35		/// Initialized data is placed first, then uninitialized, so that only
36	36		/// initialized data needs to be written to the output file.
37	37		/// Returns the updated offset past all placed symbols.
38	38		export fn layoutSection(
39		-	program: *il::Program,
	39	+	items: *[il::Data],
40	40		syms: *mut [DataSym],
41	41		count: *mut u32,
42	42		base: u32,
43	43		readOnly: bool
44	44		) -> u32 {
45	45		let mut offset: u32 = 0;
46	46
47	47		// Initialized data first.
48		-	for i in 0..program.data.len {
49		-	let data = &program.data[i];
	48	+	for i in 0..items.len {
	49	+	let data = &items[i];
50	50		if data.readOnly == readOnly and not data.isUndefined {
51	51		set offset = mem::alignUp(offset, data.alignment);
52	52		set syms[*count] = DataSym { name: data.name, addr: base + offset };
53	53		set *count += 1;
54	54		set offset += data.size;
55	55		}
56	56		}
57	57		// Uninitialized data after.
58		-	for i in 0..program.data.len {
59		-	let data = &program.data[i];
	58	+	for i in 0..items.len {
	59	+	let data = &items[i];
60	60		if data.readOnly == readOnly and data.isUndefined {
61	61		set offset = mem::alignUp(offset, data.alignment);
62	62		set syms[*count] = DataSym { name: data.name, addr: base + offset };
63	63		set *count += 1;
64	64		set offset += data.size;

69	69
70	70		/// Emit data bytes for a single section (read-only or read-write) into `buf`.
71	71		/// Iterates initialized data in the IL program, serializing each data item.
72	72		/// Returns the total number of bytes written.
73	73		export fn emitSection(
74		-	program: *il::Program,
	74	+	items: *[il::Data],
75	75		dataSymMap: *DataSymMap,
76	76		fnLabels: *labels::Labels,
77	77		codeBase: u32,
78	78		buf: *mut [u8],
79	79		readOnly: bool
80	80		) -> u32 {
81	81		let mut offset: u32 = 0;
82	82
83		-	for i in 0..program.data.len {
84		-	let data = &program.data[i];
	83	+	for i in 0..items.len {
	84	+	let data = &items[i];
85	85		if data.readOnly == readOnly and not data.isUndefined {
86	86		set offset = mem::alignUp(offset, data.alignment);
87	87		assert offset + data.size <= buf.len, "emitSection: buffer overflow";
88	88		for j in 0..data.values.len {
89	89		let v = &data.values[j];

389	389		export record Program {
390	390		/// Global data.
391	391		data: *[Data],
392	392		/// Functions.
393	393		fns: [Fn],
394		-	/// Index of entry point function, if any.
395		-	defaultFnIdx: ?u32,
396	394		}
397	395
398	396		///////////////////////
399	397		// Utility Functions //
400	398		///////////////////////

253	253		debug: bool,
254	254		/// Whether to lower `@test` functions.
255	255		buildTest: bool,
256	256		}
257	257
	258	+	/// Role of a lowered function in the final program.
	259	+	export union FnRole {
	260	+	/// Regular function.
	261	+	Normal,
	262	+	/// Program entry function marked with `@default`.
	263	+	Default,
	264	+	}
	265	+
	266	+	/// Function sink used by lowerers that consume functions as they are produced.
	267	+	export record FnSink {
	268	+	/// Opaque context passed to the sink callback.
	269	+	ctx: *mut opaque,
	270	+	/// Callback invoked for each lowered function.
	271	+	emitFn: fn(mut opaque, il::Fn, FnRole),
	272	+	}
	273	+
	274	+	/// Destination for functions produced by the lowerer.
	275	+	export union FnOutput {
	276	+	/// Store lowered functions in the provided slice.
	277	+	Accumulate(mut [il::Fn]),
	278	+	/// Send lowered functions to an external consumer immediately.
	279	+	Stream(FnSink),
	280	+	}
	281	+
258	282		/// Module-level lowering context. Shared across all function lowerings.
259	283		/// Holds global state like the data section (strings, constants) and provides
260	284		/// access to the resolver for type queries.
261	285		export record Lowerer {
262		-	/// Arena for IL allocations. All IL nodes are allocated here.
	286	+	/// Arena for persistent lowering state, including data and symbol names.
263	287		arena: *mut alloc::Arena,
	288	+	/// Arena for allocations owned by the function currently being lowered.
	289	+	fnArena: *mut alloc::Arena,
264	290		/// Allocator backed by the arena.
265	291		allocator: alloc::Allocator,
266	292		/// Resolver for type information. Used to query types, symbols, and
267	293		/// compile-time constant values during lowering.
268	294		resolver: *resolver::Resolver,

273	299		/// Current module being lowered.
274	300		currentMod: ?u16,
275	301		/// Global data items (string literals, constants, static arrays).
276	302		/// These become the data sections in the final binary.
277	303		data: *mut [il::Data],
278		-	/// Lowered functions.
279		-	fns: mut [il::Fn],
	304	+	/// Destination for lowered functions.
	305	+	output: FnOutput,
280	306		/// Map of function symbols to qualified names.
281	307		fnSyms: *mut [FnSymEntry],
282	308		/// Global error type tag table. Maps nominal types to unique tags.
283	309		errTags: *mut [ErrTagEntry],
284	310		/// Next error tag to assign (starts at 1; 0 = success).

327	353		self.errTags.append(ErrTagEntry { ty: errType, tag }, self.allocator);
328	354
329	355		return tag;
330	356		}
331	357
	358	+	/// Emit one function to the active output.
	359	+	fn emitFunction(self: mut Lowerer, func: il::Fn, role: FnRole) {
	360	+	match self.output {
	361	+	case FnOutput::Accumulate(accumulated) => {
	362	+	let mut fns = accumulated;
	363	+	fns.append(func, self.allocator);
	364	+	set self.output = FnOutput::Accumulate(fns);
	365	+	}
	366	+	case FnOutput::Stream(sink) => {
	367	+	sink.emitFn(sink.ctx, func, role);
	368	+	}
	369	+	}
	370	+	}
	371	+
	372	+	/// Get the function role for a top-level function declaration.
	373	+	fn lowerFnRole(isRoot: bool, attrs: ?ast::Attributes) -> FnRole {
	374	+	if isRoot and checkAttr(attrs, ast::Attribute::Default) {
	375	+	return FnRole::Default;
	376	+	}
	377	+	return FnRole::Normal;
	378	+	}
	379	+
332	380		/// Builder for accumulating data values during constant lowering.
333	381		record DataValueBuilder {
334	382		allocator: alloc::Allocator,
335	383		values: *mut [il::DataValue],
336	384		/// Whether all values pushed are undefined.

680	728		pkgName: *[u8],
681	729		arena: *mut alloc::Arena
682	730		) -> il::Program throws (LowerError) {
683	731		let mut low = Lowerer {
684	732		arena: arena,
	733	+	fnArena: arena,
685	734		allocator: alloc::arenaAllocator(arena),
686	735		resolver: res,
687	736		moduleGraph: nil,
688	737		pkgName,
689	738		currentMod: nil,
690	739		data: &mut [],
691		-	fns: &mut [],
	740	+	output: FnOutput::Accumulate(&mut []),
692	741		fnSyms: &mut [],
693	742		errTags: &mut [],
694	743		errTagCounter: 1,
695	744		options: LowerOptions { debug: false, buildTest: false },
696	745		};
697		-	let defaultFnIdx = try lowerDecls(&mut low, root, true);
	746	+	try lowerDecls(&mut low, root, true);
698	747
699		-	return il::Program {
700		-	data: &low.data[..],
701		-	fns: &low.fns[..],
702		-	defaultFnIdx,
703		-	};
	748	+	return finalize(&low);
704	749		}
705	750
706	751		/////////////////////////////////
707	752		// Multi-Module Lowering API //
708	753		/////////////////////////////////

711	756		export fn lowerer(
712	757		res: *resolver::Resolver,
713	758		graph: *module::ModuleGraph,
714	759		pkgName: *[u8],
715	760		arena: *mut alloc::Arena,
	761	+	fnArena: *mut alloc::Arena,
716	762		options: LowerOptions
717	763		) -> Lowerer {
718	764		return Lowerer {
719	765		arena,
	766	+	fnArena,
720	767		allocator: alloc::arenaAllocator(arena),
721	768		resolver: res,
722	769		moduleGraph: graph,
723	770		pkgName,
724	771		currentMod: nil,
725	772		data: &mut [],
726		-	fns: &mut [],
	773	+	output: FnOutput::Accumulate(&mut []),
727	774		fnSyms: &mut [],
728	775		errTags: &mut [],
729	776		errTagCounter: 1,
730	777		options,
731	778		};
732	779		}
733	780
734		-	/// Lower a module's AST into the lowerer accumulator.
	781	+	/// Lower a module's AST into the lowerer.
735	782		/// Call this for each module in the package, then use `finalize` to get the program.
736	783		export fn lowerModule(
737	784		low: *mut Lowerer,
738	785		moduleId: u16,
739	786		root: *ast::Node,
740	787		isRoot: bool
741		-	) -> ?u32 throws (LowerError) {
	788	+	) throws (LowerError) {
742	789		set low.currentMod = moduleId;
743		-	return try lowerDecls(low, root, isRoot);
	790	+	try lowerDecls(low, root, isRoot);
744	791		}
745	792
746	793		/// Lower all top-level declarations in a block.
747		-	fn lowerDecls(low: mut Lowerer, root: ast::Node, isRoot: bool) -> ?u32 throws (LowerError) {
	794	+	fn lowerDecls(low: mut Lowerer, root: ast::Node, isRoot: bool) throws (LowerError) {
748	795		let case ast::NodeValue::Block(block) = root.value else {
749	796		throw LowerError::ExpectedBlock(root);
750	797		};
751	798		let stmtsList = block.statements;
752		-	let mut defaultFnIdx: ?u32 = nil;
753	799
754	800		for node in stmtsList {
755	801		match node.value {
756	802		case ast::NodeValue::FnDecl(decl) => {
757	803		if let f = try lowerFnDecl(low, node, decl) {
758		-	if isRoot and checkAttr(decl.attrs, ast::Attribute::Default) {
759		-	set defaultFnIdx = low.fns.len;
760		-	}
761		-	low.fns.append(f, low.allocator);
	804	+	let role = lowerFnRole(isRoot, decl.attrs);
	805	+	emitFunction(low, f, role);
762	806		}
763	807		}
764	808		case ast::NodeValue::ConstDecl(decl) => {
765	809		try lowerDataDecl(low, node, decl.value, true);
766	810		}

770	814		case ast::NodeValue::InstanceDecl { traitName, targetType, methods } => {
771	815		try lowerInstanceDecl(low, node, traitName, targetType, methods);
772	816		}
773	817		case ast::NodeValue::MethodDecl { name, receiverName, receiverType, sig, body, .. } => {
774	818		if let f = try lowerMethodDecl(low, node, name, receiverName, sig, body) {
775		-	low.fns.append(f, low.allocator);
	819	+	emitFunction(low, f, FnRole::Normal);
776	820		}
777	821		}
778	822		else => {},
779	823		}
780	824		}
781		-	return defaultFnIdx;
782	825		}
783	826
784	827		/// Finalize lowering and return the unified IL program.
785		-	export fn finalize(low: *Lowerer, defaultFnIdx: ?u32) -> il::Program {
	828	+	export fn finalize(low: *Lowerer) -> il::Program {
	829	+	let mut fns: mut [il::Fn] = undefined;
	830	+	match low.output {
	831	+	case FnOutput::Accumulate(accumulated) => {
	832	+	set fns = accumulated;
	833	+	}
	834	+	case FnOutput::Stream(_) => {
	835	+	panic "finalize: cannot finalize streaming lowerer";
	836	+	}
	837	+	}
786	838		return il::Program {
787	839		data: &low.data[..],
788		-	fns: &low.fns[..],
789		-	defaultFnIdx,
	840	+	fns: &fns[..],
790	841		};
791	842		}
792	843
793	844		/////////////////////////////////
794	845		// Qualified Name Construction //

857	908		self: *mut Lowerer,
858	909		node: *ast::Node,
859	910		fnType: *resolver::FnType,
860	911		qualName: *[u8]
861	912		) -> FnLowerer {
862		-	let loopStack = try! alloc::allocSlice(self.arena, @sizeOf(LoopCtx), @alignOf(LoopCtx), MAX_LOOP_DEPTH) as *mut [LoopCtx];
	913	+	let loopStack = try! alloc::allocSlice(self.fnArena, @sizeOf(LoopCtx), @alignOf(LoopCtx), MAX_LOOP_DEPTH) as *mut [LoopCtx];
863	914
864	915		let mut fnLow = FnLowerer {
865	916		low: self,
866		-	allocator: alloc::arenaAllocator(self.arena),
	917	+	allocator: alloc::arenaAllocator(self.fnArena),
867	918		fnType: fnType,
868	919		fnName: qualName,
869	920		vars: &mut [],
870	921		params: &mut [],
871	922		blockData: &mut [],

926	977		// as the first argument; the callee writes the return value into it.
927	978		if requiresReturnParam(fnType) and not isExtern {
928	979		set fnLow.returnReg = nextReg(&mut fnLow);
929	980		}
930	981		let lowParams = try lowerParams(&mut fnLow, *fnType, decl.sig.params, nil);
931		-	let func = try! alloc::alloc(self.arena, @sizeOf(il::Fn), @alignOf(il::Fn)) as *mut il::Fn;
	982	+	let func = try! alloc::alloc(self.fnArena, @sizeOf(il::Fn), @alignOf(il::Fn)) as *mut il::Fn;
932	983
933	984		set *func = il::Fn {
934	985		name: qualName,
935	986		params: lowParams,
936	987		returnType: undefined,

1030	1081		throw LowerError::ExpectedIdentifier;
1031	1082		};
1032	1083		let qualName = instanceMethodName(self, nil, typeName, mName);
1033	1084		let func = try lowerMethod(self, methodNode, qualName, receiverName, sig, body)
1034	1085		else continue;
1035		-	self.fns.append(func, self.allocator);
	1086	+	emitFunction(self, func, FnRole::Normal);
1036	1087
1037	1088		let method = resolver::findTraitMethod(traitInfo, mName)
1038	1089		else panic "lowerInstanceDecl: method not found in trait";
1039	1090
1040	1091		set methodNames[method.index] = qualName;

1092	1143		let mut fnLow = fnLowerer(self, node, fnType, qualName);
1093	1144		if requiresReturnParam(fnType) {
1094	1145		set fnLow.returnReg = nextReg(&mut fnLow);
1095	1146		}
1096	1147		let lowParams = try lowerParams(&mut fnLow, *fnType, sig.params, receiverName);
1097		-	let func = try! alloc::alloc(self.arena, @sizeOf(il::Fn), @alignOf(il::Fn)) as *mut il::Fn;
	1148	+	let func = try! alloc::alloc(self.fnArena, @sizeOf(il::Fn), @alignOf(il::Fn)) as *mut il::Fn;
1098	1149
1099	1150		set *func = il::Fn {
1100	1151		name: qualName,
1101	1152		params: lowParams,
1102	1153		returnType: ilType(self, *fnType.returnType),

1154	1205		/// This ensures unique labels like `@then0`, `@then1`, etc.
1155	1206		fn labelWithSuffix(self: mut FnLowerer, base: [u8], suffix: u32) -> *[u8] throws (LowerError) {
1156	1207		let mut digits: [u8; fmt::U32_STR_LEN] = undefined;
1157	1208		let suffixText = fmt::formatU32(suffix, &mut digits[..]);
1158	1209		let totalLen = base.len + suffixText.len;
1159		-	let buf = try! alloc::allocSlice(self.low.arena, 1, 1, totalLen) as *mut [u8];
	1210	+	let buf = try! alloc::allocSlice(self.low.fnArena, 1, 1, totalLen) as *mut [u8];
1160	1211
1161	1212		try! mem::copy(&mut buf[..base.len], base);
1162	1213		try! mem::copy(&mut buf[base.len..totalLen], suffixText);
1163	1214
1164	1215		return &buf[..totalLen];

1923	1974		/// be switched to via [`switchToBlock`] before instructions can be emitted.
1924	1975		fn createBlock(self: mut FnLowerer, labelBase: [u8]) -> BlockId throws (LowerError) {
1925	1976		let label = try nextLabel(self, labelBase);
1926	1977		let id = BlockId(self.blockData.len);
1927	1978		let varCount = self.fnType.localCount;
1928		-	let vars = try! alloc::allocSlice(self.low.arena, @sizeOf(?il::Val), @alignOf(?il::Val), varCount) as *mut [?il::Val];
	1979	+	let vars = try! alloc::allocSlice(self.low.fnArena, @sizeOf(?il::Val), @alignOf(?il::Val), varCount) as *mut [?il::Val];
1929	1980
1930	1981		for i in 0..varCount {
1931	1982		set vars[i] = nil;
1932	1983		}
1933	1984		self.blockData.append(BlockData {

2502	2553		}
2503	2554
2504	2555		/// Finalize all blocks and return the block array.
2505	2556		fn finalizeBlocks(self: mut FnLowerer) -> [il::Block] throws (LowerError) {
2506	2557		let blocks = try! alloc::allocSlice(
2507		-	self.low.arena, @sizeOf(il::Block), @alignOf(il::Block), self.blockData.len
	2558	+	self.low.fnArena, @sizeOf(il::Block), @alignOf(il::Block), self.blockData.len
2508	2559		) as *mut [il::Block];
2509	2560
2510	2561		for i in 0..self.blockData.len {
2511	2562		let data = &self.blockData[i];
2512	2563

2564	2615		return block;
2565	2616		}
2566	2617
2567	2618		/// Allocate a slice of values in the lowering arena.
2568	2619		fn allocVals(self: mut FnLowerer, len: u32) -> mut [il::Val] throws (LowerError) {
2569		-	return try! alloc::allocSlice(self.low.arena, @sizeOf(il::Val), @alignOf(il::Val), len) as *mut [il::Val];
	2620	+	return try! alloc::allocSlice(self.low.fnArena, @sizeOf(il::Val), @alignOf(il::Val), len) as *mut [il::Val];
2570	2621		}
2571	2622
2572	2623		/// Allocate a single-value slice in the lowering arena.
2573	2624		fn allocVal(self: mut FnLowerer, val: il::Val) -> mut [il::Val] throws (LowerError) {
2574	2625		let args = try allocVals(self, 1);

2957	3008		fn growArgs(self: mut FnLowerer, args: mut [il::Val], capacity: u32) -> *mut [il::Val] {
2958	3009		if args.len >= capacity {
2959	3010		return args;
2960	3011		}
2961	3012		let newArgs = try! alloc::allocSlice(
2962		-	self.low.arena, @sizeOf(il::Val), @alignOf(il::Val), capacity
	3013	+	self.low.fnArena, @sizeOf(il::Val), @alignOf(il::Val), capacity
2963	3014		) as *mut [il::Val];
2964	3015
2965	3016		for arg, i in args {
2966	3017		set newArgs[i] = arg;
2967	3018		}