radiance · commit

refactor(lower): own control-flow edge arguments

2d2cb3d60788723d97b746954950095d053885da83013ff049260dad63ad2a31

Keep jump, branch, and switch arguments in the function lowerer's owned block state while SSA construction mutates them. Publish raw IL aliases only after each block is complete, so edge rewrites and growth stay in safe Radiance.

Alexis Sellier committed 4 days ago 1 parent 84158aa3

lib/std/lang/lower.rad +356 -151


/// A handle to a basic block within the current function.
/// Block handles are stable, they don't change as more blocks are added.
export record BlockId: Copy(u32);

/// Construction-time switch case with safely owned edge arguments.
record LowerSwitchCase {
    /// Constant value selecting this case.
    value: i64,
    /// Destination block.
    target: BlockId,
    /// Mutable argument capacity retained until final IL publication.
    args: *mut [il::Val],
}

/// Construction-time terminator edges with safely owned argument storage.
union LowerTerminator {
    /// No jump, branch, or switch has been recorded.
    None,
    /// One unconditional edge.
    Jmp {
        /// Destination block.
        target: BlockId,
        /// Mutable argument capacity retained until final IL publication.
        args: *mut [il::Val],
    },
    /// Two conditional edges.
    Br {
        /// Destination when the comparison succeeds.
        thenTarget: BlockId,
        /// Mutable argument capacity for the successful edge.
        thenArgs: *mut [il::Val],
        /// Destination when the comparison fails.
        elseTarget: BlockId,
        /// Mutable argument capacity for the failed edge.
        elseArgs: *mut [il::Val],
    },
    /// Default and case edges for a switch.
    Switch {
        /// Destination when no case matches.
        defaultTarget: BlockId,
        /// Mutable argument capacity for the default edge.
        defaultArgs: *mut [il::Val],
        /// Cases and their mutable argument capacities.
        cases: *mut [LowerSwitchCase],
    },
}

/// Internal block state during construction.
///
/// The key invariants:
///
/// - A block is "open" if it has no terminator; instructions can be added.

    /// whose predecessor arguments must be resolved.
    paramVars: *mut [u32],
    /// Instructions accumulated so far. The last instruction should eventually
    /// be a terminator.
    instrs: *mut [il::Instr],
    /// Safely owned edge data for the final jump, branch, or switch instruction.
    terminator: LowerTerminator,
    /// Debug source locations, one per instruction. Only populated when
    /// debug info is enabled.
    locs: *mut [il::SrcLoc],
    /// Predecessor block ids. Used for SSA construction to propagate values
    /// from predecessors when a variable is used before being defined locally.

    for i in 0..self.blockData.len {
        let blk = &mut self.blockData[i];
        if blk.vars[*v] == from {
            set blk.vars[*v] = to;
        }
        if blk.instrs.len > 0 {
            let ix = blk.instrs.len - 1;
            match &mut blk.instrs[ix] {
                case il::Instr::Jmp { args, .. } => {
                    unsafe {
                        rewriteValInSlice(*args, from, to);
                    }
                },
                case il::Instr::Br { thenArgs, elseArgs, .. } => {
                    unsafe {
                        rewriteValInSlice(*thenArgs, from, to);
                        rewriteValInSlice(*elseArgs, from, to);
                    }
                }
                case il::Instr::Switch { defaultArgs, cases, .. } => {
                    unsafe {
                        rewriteValInSlice(*defaultArgs, from, to);
                        rewriteSwitchArgs(*cases, from, to);
                    }
                }
                else => {}
            }
        rewriteTerminatorArgs(&mut blk.terminator, from, to);
    }
}

/// Rewrite provisional values in safely owned terminator edge arguments.
fn rewriteTerminatorArgs(
    terminator: &mut LowerTerminator,
    from: il::Val,
    to: il::Val,
) {
    match terminator {
        case LowerTerminator::None => {}
        case LowerTerminator::Jmp { args, .. } => {
            rewriteValInSlice(&mut args[..], from, to);
        }
        case LowerTerminator::Br { thenArgs, elseArgs, .. } => {
            rewriteValInSlice(&mut thenArgs[..], from, to);
            rewriteValInSlice(&mut elseArgs[..], from, to);
        }
        case LowerTerminator::Switch { defaultArgs, cases, .. } => {
            rewriteValInSlice(&mut defaultArgs[..], from, to);
            rewriteSwitchArgs(&mut cases[..], from, to);
        }
    }
}

/// Replace a provisional value in each switch case's argument sequence.
fn rewriteSwitchArgs(cases: &mut [il::SwitchCase], from: il::Val, to: il::Val) {
/// Replace a provisional value in each construction-time switch argument sequence.
fn rewriteSwitchArgs(cases: &mut [LowerSwitchCase], from: il::Val, to: il::Val) {
    for i in 0..cases.len {
        let branch = &mut cases[i];
        unsafe {
            rewriteValInSlice(branch.args, from, to);
        }
        rewriteValInSlice(cases[i].args, from, to);
    }
}

/// Replace all occurrences of `from` with `to` in an args slice.
fn rewriteValInSlice(args: &mut [il::Val], from: il::Val, to: il::Val) {

    blocks.append(BlockData {
        label,
        params: &mut [],
        paramVars: &mut [],
        instrs: &mut [],
        terminator: LowerTerminator::None,
        locs: &mut [],
        preds: &mut [],
        vars,
        sealState: Sealed::No,
        loopDepth,

        block.locs.append(srcLoc, allocator);
    }
    block.instrs.append(*instr, allocator);
}

/// Emit an unconditional jump to `target`.
fn emitJmp 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, target: BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
/// Record a terminator instruction and retain its mutable edges in safe storage.
fn recordTerminator 'arena 'phase 'function (
    self: &mut FnLowerer 'arena 'phase 'function,
    instr: il::Instr,
    terminator: LowerTerminator,
) where 'arena: 'phase, 'phase: 'function {
    let block = currentBlock(self);
    emit(self, instr);
    set self.blockData[*block].terminator = terminator;
}

/// Emit a jump placeholder while retaining its mutable edge arguments.
fn emitJmpTerminator 'arena 'phase 'function (
    self: &mut FnLowerer 'arena 'phase 'function,
    target: BlockId,
    args: *mut [il::Val],
) where 'arena: 'phase, 'phase: 'function {
    unsafe {
        recordTerminator(
            self,
            il::Instr::Jmp { target: *target, args: &mut [] },
            LowerTerminator::Jmp { target, args },
        );
    }
}

/// Emit a branch placeholder while retaining its mutable edge arguments.
fn emitBrTerminator 'arena 'phase 'function (
    self: &mut FnLowerer 'arena 'phase 'function,
    op: il::CmpOp,
    typ: il::Type,
    a: il::Val,
    b: il::Val,
    thenTarget: BlockId,
    elseTarget: BlockId,
    terminator: LowerTerminator,
) where 'arena: 'phase, 'phase: 'function {
    unsafe {
        emit(self, il::Instr::Jmp { target: *target, args: &mut [] });
        recordTerminator(
            self,
            il::Instr::Br {
                op, typ, a, b,
                thenTarget: *thenTarget, thenArgs: &mut [],
                elseTarget: *elseTarget, elseArgs: &mut [],
            },
            terminator,
        );
    }
}

/// Emit a switch placeholder while retaining its mutable edge arguments and cases.
fn emitSwitchTerminator 'arena 'phase 'function (
    self: &mut FnLowerer 'arena 'phase 'function,
    val: il::Val,
    defaultTarget: BlockId,
    terminator: LowerTerminator,
) where 'arena: 'phase, 'phase: 'function {
    unsafe {
        recordTerminator(
            self,
            il::Instr::Switch {
                val,
                defaultTarget: *defaultTarget,
                defaultArgs: &mut [],
                cases: &mut [],
            },
            terminator,
        );
    }
}

/// Emit an unconditional jump to `target`.
fn emitJmp 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, target: BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
    emitJmpTerminator(self, target, &mut []);
    addPredecessor(self, target, currentBlock(self));
}

/// Emit an unconditional jump to `target` with a single argument.
fn emitJmpWithArg 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, target: BlockId, arg: il::Val) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
    unsafe {
        let args = try allocVal(self, arg);
        emit(self, il::Instr::Jmp { target: *target, args });
    }
    let args = try allocVal(self, arg);
    emitJmpTerminator(self, target, args);
    addPredecessor(self, target, currentBlock(self));
}

/// Emit an unconditional jump to `target` and switch to it.
fn switchAndJumpTo 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, target: BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {

}

/// Emit a conditional branch based on `cond`.
fn emitBr 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, cond: il::Reg, thenBlock: BlockId, elseBlock: BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
    assert thenBlock <> elseBlock;
    unsafe {
        emit(self, il::Instr::Br {
            op: il::CmpOp::Ne,
            typ: il::Type::W32,
            a: il::Val::Reg(cond),
            b: il::Val::Imm(0),
            thenTarget: *thenBlock,
            thenArgs: &mut [],
            elseTarget: *elseBlock,
            elseArgs: &mut [],
        });
    }
    emitBrTerminator(
        self,
        il::CmpOp::Ne,
        il::Type::W32,
        il::Val::Reg(cond),
        il::Val::Imm(0),
        thenBlock, elseBlock,
        LowerTerminator::Br {
            thenTarget: thenBlock, thenArgs: &mut [],
            elseTarget: elseBlock, elseArgs: &mut [],
        },
    );
    addPredecessor(self, thenBlock, currentBlock(self));
    addPredecessor(self, elseBlock, currentBlock(self));
}

/// Emit a compare-and-branch instruction with the given comparison op.

    b: il::Val,
    thenBlock: BlockId,
    elseBlock: BlockId
) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
    assert thenBlock <> elseBlock;
    unsafe {
        emit(self, il::Instr::Br {
            op, typ, a, b,
            thenTarget: *thenBlock, thenArgs: &mut [],
            elseTarget: *elseBlock, elseArgs: &mut [],
        });
    }
    emitBrTerminator(
        self,
        op, typ, a, b,
        thenBlock, elseBlock,
        LowerTerminator::Br {
            thenTarget: thenBlock, thenArgs: &mut [],
            elseTarget: elseBlock, elseArgs: &mut [],
        },
    );
    addPredecessor(self, thenBlock, currentBlock(self));
    addPredecessor(self, elseBlock, currentBlock(self));
}

/// Emit a guard that traps with `ebreak` when a comparison is false.

        }
    }
    preds.append(*pred, allocator);
}

/// Return whether construction-time edge data must be published.
fn hasLowerTerminator(terminator: &LowerTerminator) -> bool {
    match terminator {
        case LowerTerminator::None => return false,
        case LowerTerminator::Jmp { .. },
             LowerTerminator::Br { .. },
             LowerTerminator::Switch { .. } =>
            return true,
    }
}

/// Publish one block's safely owned edge arguments into its final IL terminator.
unsafe fn publishTerminator(
    instr: il::Instr,
    terminator: &mut LowerTerminator,
    allocator: alloc::Allocator,
) -> il::Instr {
    match terminator {
        case LowerTerminator::None => return instr,
        case LowerTerminator::Jmp { args, .. } => {
            let case il::Instr::Jmp { target, .. } = instr
                else panic "publishTerminator: expected jump terminator";
            return il::Instr::Jmp {
                target,
                args: (&mut args[..]) as *unsafe mut [il::Val],
            };
        }
        case LowerTerminator::Br { thenArgs, elseArgs, .. } => {
            let case il::Instr::Br {
                op, typ, a, b, thenTarget, elseTarget, ..
            } = instr
            else panic "publishTerminator: expected branch terminator";
            return il::Instr::Br {
                op, typ, a, b,
                thenTarget,
                thenArgs: (&mut thenArgs[..]) as *unsafe mut [il::Val],
                elseTarget,
                elseArgs: (&mut elseArgs[..]) as *unsafe mut [il::Val],
            };
        }
        case LowerTerminator::Switch { defaultArgs, cases, .. } => {
            let case il::Instr::Switch { val, defaultTarget, .. } = instr
                else panic "publishTerminator: expected switch terminator";
            let mut publishedCases: *mut [il::SwitchCase] = &mut [];
            for i in 0..cases.len {
                let branch = &mut cases[i];
                publishedCases.append(il::SwitchCase {
                    value: branch.value,
                    target: *branch.target,
                    args: (&mut branch.args[..]) as *unsafe mut [il::Val],
                }, allocator);
            }
            return il::Instr::Switch {
                val,
                defaultTarget,
                defaultArgs: (&mut defaultArgs[..]) as *unsafe mut [il::Val],
                cases: (&mut publishedCases[..]) as *unsafe mut [il::SwitchCase],
            };
        }
    }
}

/// Finalize all blocks and return the block array.
unsafe fn finalizeBlocks 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function) -> *unsafe [il::Block] throws (LowerError) where 'arena: 'phase, 'phase: 'function {
    let mut blocks: *mut [il::Block] = &mut [];
    let allocator = alloc::arenaAllocator(self.arena);
    let allocator = self.allocator;

    for i in 0..self.blockData.len {
        let data = &mut self.blockData[i];
        if hasLowerTerminator(&self.blockData[i].terminator) {
            let instrIdx = self.blockData[i].instrs.len - 1;
            let instr = self.blockData[i].instrs[instrIdx];
            let published = publishTerminator(
                instr,
                &mut self.blockData[i].terminator,
                allocator,
            );
            set self.blockData[i].instrs[instrIdx] = published;
        }

        let data = &mut self.blockData[i];
        blocks.append(il::Block {
            label: data.label,
            params: &data.params[..],
            instrs: (&mut data.instrs[..]) as *unsafe mut [il::Instr],
            locs: &data.locs[..],

    }
    return result;
}

/// Allocate a single-value slice in the lowering arena.
unsafe fn allocVal 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, val: il::Val) -> *unsafe mut [il::Val] throws (LowerError) where 'arena: 'phase, 'phase: 'function {
fn allocVal 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, val: il::Val) -> *mut [il::Val] throws (LowerError) where 'arena: 'phase, 'phase: 'function {
    let values = [val];
    return try copyVals(self, &values[..]);
    return valueStorage(&values[..], values.len, self.allocator);
}

////////////////////////
// SSA Var Management //
////////////////////////

        let pred = BlockId(self.blockData[*block].preds[index]);
        // This may recursively trigger more block arg resolution if the
        // predecessor also needs to look up the variable from its predecessors.
        let val = try useVarInBlock(self, pred, v);
        assert val <> il::Val::Undef, "createBlockParam: predecessor provides undef value for block parameter";
        patchTerminatorArg(self, pred, *block, paramIdx, val);
        patchTerminatorArg(self, pred, block, paramIdx, val);
    }
}

/// Check if a block parameter is trivial, i.e. all predecessors provide
/// the same value. Returns the trivial value if so.

/// Patch a single terminator argument for a specific edge. This is used during
/// SSA construction to pass variable values along control flow edges.
fn patchTerminatorArg 'arena 'phase 'function (
    self: &mut FnLowerer 'arena 'phase 'function,
    from: BlockId,         // The predecessor block containing the terminator to patch.
    target: u32,           // The index of the target block we're passing the value to.
    target: BlockId,       // The target block receiving the value.
    paramIdx: u32,         // The index of the block parameter to set.
    val: il::Val           // The value to pass as the argument.
) where 'arena: 'phase, 'phase: 'function {
    let allocator = self.allocator;
    // Get mutable block data by block id.
    let data = &mut self.blockData[*from];
    let ix = data.instrs.len - 1; // The terminator is always the last instruction.
    patchEdgeArgs(&mut data.instrs[ix], target, paramIdx, val, allocator);
    patchEdgeArgs(&mut data.terminator, target, paramIdx, val, allocator);
}

/// Patch all edges in a terminator that pass a value to the target block.
fn patchEdgeArgs(instr: &mut il::Instr, target: u32, paramIdx: u32, val: il::Val, allocator: alloc::Allocator) {
    // TODO: We shouldn't need to use a mutable subscript here, given that the
    // fields are already mutable.
    match instr {
        case il::Instr::Jmp { args, .. } => {
            unsafe {
                set *args = growArgs(*args, paramIdx + 1, allocator);
                set args[paramIdx] = val;
            }
        }
        case il::Instr::Br { thenTarget, thenArgs, elseTarget, elseArgs, .. } => {
            // Nb. both branches could target the same block (e.g. `if cond { x } else { x }`).
/// Patch all construction-time edges that pass a value to the target block.
fn patchEdgeArgs(
    terminator: &mut LowerTerminator,
    target: BlockId,
    paramIdx: u32,
    val: il::Val,
    allocator: alloc::Allocator,
) {
    let capacity = paramIdx + 1;
    match terminator {
        case LowerTerminator::None =>
            panic "patchEdgeArgs: predecessor has no edge terminator",
        case LowerTerminator::Jmp { args, .. } => {
            growArgs(args, capacity, allocator);
            set args[paramIdx] = val;
        }
        case LowerTerminator::Br { thenTarget, thenArgs, elseTarget, elseArgs } => {
            // Both branches may target the same block.
            if *thenTarget == target {
                unsafe {
                    set *thenArgs = growArgs(*thenArgs, paramIdx + 1, allocator);
                    set thenArgs[paramIdx] = val;
                }
                growArgs(thenArgs, capacity, allocator);
                set thenArgs[paramIdx] = val;
            }
            if *elseTarget == target {
                unsafe {
                    set *elseArgs = growArgs(*elseArgs, paramIdx + 1, allocator);
                    set elseArgs[paramIdx] = val;
                }
                growArgs(elseArgs, capacity, allocator);
                set elseArgs[paramIdx] = val;
            }
        }
        case il::Instr::Switch { defaultTarget, defaultArgs, cases, .. } => {
        case LowerTerminator::Switch { defaultTarget, defaultArgs, cases } => {
            if *defaultTarget == target {
                unsafe {
                    set *defaultArgs = growArgs(*defaultArgs, paramIdx + 1, allocator);
                    set defaultArgs[paramIdx] = val;
                }
                growArgs(defaultArgs, capacity, allocator);
                set defaultArgs[paramIdx] = val;
            }
            unsafe {
                patchSwitchArgs(*cases, target, paramIdx, val, allocator);
            }
        }
        else => {
            // Other terminators (e.g. `Ret`, `Unreachable`) don't have successor blocks.
            patchSwitchArgs(&mut cases[..], target, paramIdx, val, allocator);
        }
    }
}

/// Patch every switch case that passes a value to the target block.
fn patchSwitchArgs(cases: &mut [il::SwitchCase], target: u32, paramIdx: u32, val: il::Val, allocator: alloc::Allocator) {
/// Patch every construction-time switch case targeting the given block.
fn patchSwitchArgs(
    cases: &mut [LowerSwitchCase],
    target: BlockId,
    paramIdx: u32,
    val: il::Val,
    allocator: alloc::Allocator,
) {
    let capacity = paramIdx + 1;
    for i in 0..cases.len {
        let branch = &mut cases[i];
        if branch.target == target {
            unsafe {
                set branch.args = growArgs(branch.args, capacity, allocator);
                set branch.args[paramIdx] = val;
            }
            growArgs(&mut branch.args, capacity, allocator);
            set branch.args[paramIdx] = val;
        }
    }
}

/// Grow an args array to hold at least the given capacity.
unsafe fn growArgs(args: *unsafe mut [il::Val], capacity: u32, allocator: alloc::Allocator) -> *unsafe mut [il::Val] {
/// Grow an argument array to hold the requested capacity.
fn growArgs(args: &mut *mut [il::Val], capacity: u32, allocator: alloc::Allocator) {
    if args.len >= capacity {
        return args;
        return;
    }
    let newArgs = valueStorage(args, capacity, allocator);
    return (&mut newArgs[..]) as *unsafe mut [il::Val];
    set *args = valueStorage(&args[..], capacity, allocator);
}

/// Select a receiver or explicit parameter name from its AST declaration.
fn paramName(params: &[*ast::Node], receiver: ?*ast::Node, index: u32) -> *[u8] throws (LowerError) {
    let mut position = index;

        }
    }
    return try finalizeBlocks(self);
}

/// Append an initialized switch case to owned storage.
/// Append a construction-time switch case with safely owned arguments.
fn appendSwitchCase(
    cases: &mut *mut [il::SwitchCase],
    cases: &mut *mut [LowerSwitchCase],
    value: i64,
    target: BlockId,
    args: *unsafe mut [il::Val],
    args: *mut [il::Val],
    allocator: alloc::Allocator,
) {
    cases.append(il::SwitchCase { value, target: *target, args }, allocator);
    cases.append(LowerSwitchCase { value, target, args }, allocator);
}

/// Lower a scalar match as a switch instruction.
unsafe fn lowerMatchSwitch 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, prongs: *[*ast::Node], subject: &MatchSubject, mergeBlock: &mut ?BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
    let mut blocks: *mut [BlockId] = &mut [];
    let mut cases: *mut [il::SwitchCase] = &mut [];
    let mut cases: *mut [LowerSwitchCase] = &mut [];
    let mut defaultIdx: u32 = 0;
    let entry = currentBlock(self);

    for p, i in prongs {
        let case ast::NodeValue::MatchProng(prong) = p.value

                }
            }
        }
        addPredecessor(self, blocks[i], entry);
    }
    emit(self, il::Instr::Switch {
        val: subject.val,
        defaultTarget: *blocks[defaultIdx],
        defaultArgs: &mut [],
        cases: (&mut cases[..]) as *unsafe mut [il::SwitchCase]
    });
    emitSwitchTerminator(
        self,
        subject.val,
        blocks[defaultIdx],
        LowerTerminator::Switch {
            defaultTarget: blocks[defaultIdx],
            defaultArgs: &mut [],
            cases,
        },
    );

    for p, i in prongs {
        let case ast::NodeValue::MatchProng(prong) = p.value
            else throw LowerError::UnexpectedNodeValue(p);



    let falseArgs = try allocVal(self, il::Val::Imm(0));
    let trueArgs = try allocVal(self, il::Val::Imm(1));

    // Check if tags differ.
    emit(self, il::Instr::Br {
        op: il::CmpOp::Eq, typ: il::Type::W8, a: tagA, b: tagB,
        thenTarget: *nilCheck, thenArgs: &mut [],
        elseTarget: *mergeBlock, elseArgs: falseArgs,
    });
    emitBrTerminator(
        self,
        il::CmpOp::Eq, il::Type::W8, tagA, tagB,
        nilCheck, mergeBlock,
        LowerTerminator::Br {
            thenTarget: nilCheck, thenArgs: &mut [],
            elseTarget: mergeBlock, elseArgs: falseArgs,
        },
    );
    addPredecessor(self, nilCheck, currentBlock(self));
    addPredecessor(self, mergeBlock, currentBlock(self));

    // Check if both are `nil`.
    try switchToAndSeal(self, nilCheck);
    emit(self, il::Instr::Br {
        op: il::CmpOp::Ne, typ: il::Type::W8, a: tagA, b: il::Val::Imm(0),
        thenTarget: *payloadCmp, thenArgs: &mut [],
        elseTarget: *mergeBlock, elseArgs: trueArgs,
    });
    emitBrTerminator(
        self,
        il::CmpOp::Ne, il::Type::W8, tagA, il::Val::Imm(0),
        payloadCmp, mergeBlock,
        LowerTerminator::Br {
            thenTarget: payloadCmp, thenArgs: &mut [],
            elseTarget: mergeBlock, elseArgs: trueArgs,
        },
    );
    addPredecessor(self, payloadCmp, currentBlock(self));
    addPredecessor(self, mergeBlock, currentBlock(self));

    // Both are non-`nil`, compare payloads.
    try switchToAndSeal(self, payloadCmp);

    // Compare tags: if they differ, jump to merge with `false`; otherwise check payloads.
    let falseArgs = try allocVal(self, il::Val::Imm(0));

    assert tagBlock <> mergeBlock;

    // TODO: Use the helper once the compiler supports more than eight function params.
    emit(self, il::Instr::Br {
        op: il::CmpOp::Eq, typ: il::Type::W8, a: tagA, b: tagB,
        thenTarget: *tagBlock, thenArgs: &mut [],
        elseTarget: *mergeBlock, elseArgs: falseArgs,
    });
    emitBrTerminator(
        self,
        il::CmpOp::Eq, il::Type::W8, tagA, tagB,
        tagBlock, mergeBlock,
        LowerTerminator::Br {
            thenTarget: tagBlock, thenArgs: &mut [],
            elseTarget: mergeBlock, elseArgs: falseArgs,
        },
    );
    addPredecessor(self, tagBlock, currentBlock(self));
    addPredecessor(self, mergeBlock, currentBlock(self));

    // Create comparison blocks for each non-void variant and build switch cases.
    // Void variants jump directly to merge with `true`.
    let trueArgs = try allocVal(self, il::Val::Imm(1));
    let mut cases: *mut [il::SwitchCase] = &mut [];
    let mut cases: *mut [LowerSwitchCase] = &mut [];
    let mut caseBlocks: *mut [BlockId] = &mut [];
    for variant, i in unionInfo.variants {
        if variant.valueType == resolver::Type::Void {
            let trueArgs = try allocVal(self, il::Val::Imm(1));
            caseBlocks.append(mergeBlock, alloc::arenaAllocator(self.arena));
            appendSwitchCase(&mut cases, i as i64, mergeBlock, trueArgs, alloc::arenaAllocator(self.arena));
        } else {
            let payloadBlock = try createBlock(self, "eq#payload");
            caseBlocks.append(payloadBlock, alloc::arenaAllocator(self.arena));
            appendSwitchCase(&mut cases, i as i64, payloadBlock, &mut [], alloc::arenaAllocator(self.arena));
        }
    }

    // Emit switch in @tag block. Default arm is unreachable since we cover all variants.
    let unreachableBlock = try createBlock(self, "eq#unreachable");
    try switchToAndSeal(self, tagBlock);
    emit(self, il::Instr::Switch {
        val: tagA,
        defaultTarget: *unreachableBlock,
        defaultArgs: &mut [],
        cases: (&mut cases[..]) as *unsafe mut [il::SwitchCase]
    });
    emitSwitchTerminator(
        self,
        tagA,
        unreachableBlock,
        LowerTerminator::Switch {
            defaultTarget: unreachableBlock,
            defaultArgs: &mut [],
            cases,
        },
    );

    // Add predecessor edges for switch targets.
    addPredecessor(self, unreachableBlock, tagBlock);
    for c in &cases[..] {
        addPredecessor(self, BlockId(c.target), tagBlock);
    for caseBlock in &caseBlocks[..] {
        addPredecessor(self, caseBlock, tagBlock);
    }
    let valOffset = unionInfo.valOffset as i32;

    // Emit payload comparison blocks for non-void variants.
    for variant, i in unionInfo.variants {
        let caseBlock = BlockId(cases[i].target);
        let caseBlock = caseBlocks[i];
        if caseBlock <> mergeBlock {
            try switchToAndSeal(self, caseBlock);
            let payloadEq = try emitEqAtOffset(
                self, a, b, offset + valOffset, variant.valueType
            );

) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
    let entry = currentBlock(self);

    // First pass: create blocks, resolve error types, and build switch cases.
    let mut targets: [?CatchTarget; MAX_CATCH_CLAUSES] = [nil; MAX_CATCH_CLAUSES];
    let mut cases: *mut [il::SwitchCase] = &mut [];
    let mut cases: *mut [LowerSwitchCase] = &mut [];
    let mut defaultIdx: ?u32 = nil;

    for clauseNode, i in catches {
        let case ast::NodeValue::CatchClause(clause) = clauseNode.value
            else panic "lowerMultiCatch: expected CatchClause";

        let block = try createBlock(self, "unreachable");
        addPredecessor(self, block, entry);
        set selectedDefault = block;
    }
    let defaultTarget = selectedDefault else panic "lowerMultiCatch: missing default destination";
    emit(self, il::Instr::Switch {
        val: il::Val::Reg(tagReg),
        defaultTarget: *defaultTarget,
        defaultArgs: &mut [],
        cases: (&mut cases[..]) as *unsafe mut [il::SwitchCase]
    });
    emitSwitchTerminator(
        self,
        il::Val::Reg(tagReg),
        defaultTarget,
        LowerTerminator::Switch {
            defaultTarget,
            defaultArgs: &mut [],
            cases,
        },
    );

    // Second pass: emit each catch clause body.
    for clauseNode, i in catches {
        let case ast::NodeValue::CatchClause(clause) = clauseNode.value
            else panic "lowerMultiCatch: expected CatchClause";

528	528
529	529		/// A handle to a basic block within the current function.
530	530		/// Block handles are stable, they don't change as more blocks are added.
531	531		export record BlockId: Copy(u32);
532	532
	533	+	/// Construction-time switch case with safely owned edge arguments.
	534	+	record LowerSwitchCase {
	535	+	/// Constant value selecting this case.
	536	+	value: i64,
	537	+	/// Destination block.
	538	+	target: BlockId,
	539	+	/// Mutable argument capacity retained until final IL publication.
	540	+	args: *mut [il::Val],
	541	+	}
	542	+
	543	+	/// Construction-time terminator edges with safely owned argument storage.
	544	+	union LowerTerminator {
	545	+	/// No jump, branch, or switch has been recorded.
	546	+	None,
	547	+	/// One unconditional edge.
	548	+	Jmp {
	549	+	/// Destination block.
	550	+	target: BlockId,
	551	+	/// Mutable argument capacity retained until final IL publication.
	552	+	args: *mut [il::Val],
	553	+	},
	554	+	/// Two conditional edges.
	555	+	Br {
	556	+	/// Destination when the comparison succeeds.
	557	+	thenTarget: BlockId,
	558	+	/// Mutable argument capacity for the successful edge.
	559	+	thenArgs: *mut [il::Val],
	560	+	/// Destination when the comparison fails.
	561	+	elseTarget: BlockId,
	562	+	/// Mutable argument capacity for the failed edge.
	563	+	elseArgs: *mut [il::Val],
	564	+	},
	565	+	/// Default and case edges for a switch.
	566	+	Switch {
	567	+	/// Destination when no case matches.
	568	+	defaultTarget: BlockId,
	569	+	/// Mutable argument capacity for the default edge.
	570	+	defaultArgs: *mut [il::Val],
	571	+	/// Cases and their mutable argument capacities.
	572	+	cases: *mut [LowerSwitchCase],
	573	+	},
	574	+	}
	575	+
533	576		/// Internal block state during construction.
534	577		///
535	578		/// The key invariants:
536	579		///
537	580		/// - A block is "open" if it has no terminator; instructions can be added.

549	592		/// whose predecessor arguments must be resolved.
550	593		paramVars: *mut [u32],
551	594		/// Instructions accumulated so far. The last instruction should eventually
552	595		/// be a terminator.
553	596		instrs: *mut [il::Instr],
	597	+	/// Safely owned edge data for the final jump, branch, or switch instruction.
	598	+	terminator: LowerTerminator,
554	599		/// Debug source locations, one per instruction. Only populated when
555	600		/// debug info is enabled.
556	601		locs: *mut [il::SrcLoc],
557	602		/// Predecessor block ids. Used for SSA construction to propagate values
558	603		/// from predecessors when a variable is used before being defined locally.

2101	2146		for i in 0..self.blockData.len {
2102	2147		let blk = &mut self.blockData[i];
2103	2148		if blk.vars[*v] == from {
2104	2149		set blk.vars[*v] = to;
2105	2150		}
2106		-	if blk.instrs.len > 0 {
2107		-	let ix = blk.instrs.len - 1;
2108		-	match &mut blk.instrs[ix] {
2109		-	case il::Instr::Jmp { args, .. } => {
2110		-	unsafe {
2111		-	rewriteValInSlice(*args, from, to);
2112		-	}
2113		-	},
2114		-	case il::Instr::Br { thenArgs, elseArgs, .. } => {
2115		-	unsafe {
2116		-	rewriteValInSlice(*thenArgs, from, to);
2117		-	rewriteValInSlice(*elseArgs, from, to);
2118		-	}
2119		-	}
2120		-	case il::Instr::Switch { defaultArgs, cases, .. } => {
2121		-	unsafe {
2122		-	rewriteValInSlice(*defaultArgs, from, to);
2123		-	rewriteSwitchArgs(*cases, from, to);
2124		-	}
2125		-	}
2126		-	else => {}
2127		-	}
	2151	+	rewriteTerminatorArgs(&mut blk.terminator, from, to);
	2152	+	}
	2153	+	}
	2154	+
	2155	+	/// Rewrite provisional values in safely owned terminator edge arguments.
	2156	+	fn rewriteTerminatorArgs(
	2157	+	terminator: &mut LowerTerminator,
	2158	+	from: il::Val,
	2159	+	to: il::Val,
	2160	+	) {
	2161	+	match terminator {
	2162	+	case LowerTerminator::None => {}
	2163	+	case LowerTerminator::Jmp { args, .. } => {
	2164	+	rewriteValInSlice(&mut args[..], from, to);
	2165	+	}
	2166	+	case LowerTerminator::Br { thenArgs, elseArgs, .. } => {
	2167	+	rewriteValInSlice(&mut thenArgs[..], from, to);
	2168	+	rewriteValInSlice(&mut elseArgs[..], from, to);
	2169	+	}
	2170	+	case LowerTerminator::Switch { defaultArgs, cases, .. } => {
	2171	+	rewriteValInSlice(&mut defaultArgs[..], from, to);
	2172	+	rewriteSwitchArgs(&mut cases[..], from, to);
2128	2173		}
2129	2174		}
2130	2175		}
2131	2176
2132		-	/// Replace a provisional value in each switch case's argument sequence.
2133		-	fn rewriteSwitchArgs(cases: &mut [il::SwitchCase], from: il::Val, to: il::Val) {
	2177	+	/// Replace a provisional value in each construction-time switch argument sequence.
	2178	+	fn rewriteSwitchArgs(cases: &mut [LowerSwitchCase], from: il::Val, to: il::Val) {
2134	2179		for i in 0..cases.len {
2135		-	let branch = &mut cases[i];
2136		-	unsafe {
2137		-	rewriteValInSlice(branch.args, from, to);
2138		-	}
	2180	+	rewriteValInSlice(cases[i].args, from, to);
2139	2181		}
2140	2182		}
2141	2183
2142	2184		/// Replace all occurrences of `from` with `to` in an args slice.
2143	2185		fn rewriteValInSlice(args: &mut [il::Val], from: il::Val, to: il::Val) {

2193	2235		blocks.append(BlockData {
2194	2236		label,
2195	2237		params: &mut [],
2196	2238		paramVars: &mut [],
2197	2239		instrs: &mut [],
	2240	+	terminator: LowerTerminator::None,
2198	2241		locs: &mut [],
2199	2242		preds: &mut [],
2200	2243		vars,
2201	2244		sealState: Sealed::No,
2202	2245		loopDepth,

2298	2341		block.locs.append(srcLoc, allocator);
2299	2342		}
2300	2343		block.instrs.append(*instr, allocator);
2301	2344		}
2302	2345
2303		-	/// Emit an unconditional jump to `target`.
2304		-	fn emitJmp 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, target: BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
	2346	+	/// Record a terminator instruction and retain its mutable edges in safe storage.
	2347	+	fn recordTerminator 'arena 'phase 'function (
	2348	+	self: &mut FnLowerer 'arena 'phase 'function,
	2349	+	instr: il::Instr,
	2350	+	terminator: LowerTerminator,
	2351	+	) where 'arena: 'phase, 'phase: 'function {
	2352	+	let block = currentBlock(self);
	2353	+	emit(self, instr);
	2354	+	set self.blockData[*block].terminator = terminator;
	2355	+	}
	2356	+
	2357	+	/// Emit a jump placeholder while retaining its mutable edge arguments.
	2358	+	fn emitJmpTerminator 'arena 'phase 'function (
	2359	+	self: &mut FnLowerer 'arena 'phase 'function,
	2360	+	target: BlockId,
	2361	+	args: *mut [il::Val],
	2362	+	) where 'arena: 'phase, 'phase: 'function {
	2363	+	unsafe {
	2364	+	recordTerminator(
	2365	+	self,
	2366	+	il::Instr::Jmp { target: *target, args: &mut [] },
	2367	+	LowerTerminator::Jmp { target, args },
	2368	+	);
	2369	+	}
	2370	+	}
	2371	+
	2372	+	/// Emit a branch placeholder while retaining its mutable edge arguments.
	2373	+	fn emitBrTerminator 'arena 'phase 'function (
	2374	+	self: &mut FnLowerer 'arena 'phase 'function,
	2375	+	op: il::CmpOp,
	2376	+	typ: il::Type,
	2377	+	a: il::Val,
	2378	+	b: il::Val,
	2379	+	thenTarget: BlockId,
	2380	+	elseTarget: BlockId,
	2381	+	terminator: LowerTerminator,
	2382	+	) where 'arena: 'phase, 'phase: 'function {
2305	2383		unsafe {
2306		-	emit(self, il::Instr::Jmp { target: *target, args: &mut [] });
	2384	+	recordTerminator(
	2385	+	self,
	2386	+	il::Instr::Br {
	2387	+	op, typ, a, b,
	2388	+	thenTarget: *thenTarget, thenArgs: &mut [],
	2389	+	elseTarget: *elseTarget, elseArgs: &mut [],
	2390	+	},
	2391	+	terminator,
	2392	+	);
2307	2393		}
	2394	+	}
	2395	+
	2396	+	/// Emit a switch placeholder while retaining its mutable edge arguments and cases.
	2397	+	fn emitSwitchTerminator 'arena 'phase 'function (
	2398	+	self: &mut FnLowerer 'arena 'phase 'function,
	2399	+	val: il::Val,
	2400	+	defaultTarget: BlockId,
	2401	+	terminator: LowerTerminator,
	2402	+	) where 'arena: 'phase, 'phase: 'function {
	2403	+	unsafe {
	2404	+	recordTerminator(
	2405	+	self,
	2406	+	il::Instr::Switch {
	2407	+	val,
	2408	+	defaultTarget: *defaultTarget,
	2409	+	defaultArgs: &mut [],
	2410	+	cases: &mut [],
	2411	+	},
	2412	+	terminator,
	2413	+	);
	2414	+	}
	2415	+	}
	2416	+
	2417	+	/// Emit an unconditional jump to `target`.
	2418	+	fn emitJmp 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, target: BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
	2419	+	emitJmpTerminator(self, target, &mut []);
2308	2420		addPredecessor(self, target, currentBlock(self));
2309	2421		}
2310	2422
2311	2423		/// Emit an unconditional jump to `target` with a single argument.
2312	2424		fn emitJmpWithArg 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, target: BlockId, arg: il::Val) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
2313		-	unsafe {
2314		-	let args = try allocVal(self, arg);
2315		-	emit(self, il::Instr::Jmp { target: *target, args });
2316		-	}
	2425	+	let args = try allocVal(self, arg);
	2426	+	emitJmpTerminator(self, target, args);
2317	2427		addPredecessor(self, target, currentBlock(self));
2318	2428		}
2319	2429
2320	2430		/// Emit an unconditional jump to `target` and switch to it.
2321	2431		fn switchAndJumpTo 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, target: BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {

2324	2434		}
2325	2435
2326	2436		/// Emit a conditional branch based on `cond`.
2327	2437		fn emitBr 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, cond: il::Reg, thenBlock: BlockId, elseBlock: BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
2328	2438		assert thenBlock <> elseBlock;
2329		-	unsafe {
2330		-	emit(self, il::Instr::Br {
2331		-	op: il::CmpOp::Ne,
2332		-	typ: il::Type::W32,
2333		-	a: il::Val::Reg(cond),
2334		-	b: il::Val::Imm(0),
2335		-	thenTarget: *thenBlock,
2336		-	thenArgs: &mut [],
2337		-	elseTarget: *elseBlock,
2338		-	elseArgs: &mut [],
2339		-	});
2340		-	}
	2439	+	emitBrTerminator(
	2440	+	self,
	2441	+	il::CmpOp::Ne,
	2442	+	il::Type::W32,
	2443	+	il::Val::Reg(cond),
	2444	+	il::Val::Imm(0),
	2445	+	thenBlock, elseBlock,
	2446	+	LowerTerminator::Br {
	2447	+	thenTarget: thenBlock, thenArgs: &mut [],
	2448	+	elseTarget: elseBlock, elseArgs: &mut [],
	2449	+	},
	2450	+	);
2341	2451		addPredecessor(self, thenBlock, currentBlock(self));
2342	2452		addPredecessor(self, elseBlock, currentBlock(self));
2343	2453		}
2344	2454
2345	2455		/// Emit a compare-and-branch instruction with the given comparison op.

2351	2461		b: il::Val,
2352	2462		thenBlock: BlockId,
2353	2463		elseBlock: BlockId
2354	2464		) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
2355	2465		assert thenBlock <> elseBlock;
2356		-	unsafe {
2357		-	emit(self, il::Instr::Br {
2358		-	op, typ, a, b,
2359		-	thenTarget: *thenBlock, thenArgs: &mut [],
2360		-	elseTarget: *elseBlock, elseArgs: &mut [],
2361		-	});
2362		-	}
	2466	+	emitBrTerminator(
	2467	+	self,
	2468	+	op, typ, a, b,
	2469	+	thenBlock, elseBlock,
	2470	+	LowerTerminator::Br {
	2471	+	thenTarget: thenBlock, thenArgs: &mut [],
	2472	+	elseTarget: elseBlock, elseArgs: &mut [],
	2473	+	},
	2474	+	);
2363	2475		addPredecessor(self, thenBlock, currentBlock(self));
2364	2476		addPredecessor(self, elseBlock, currentBlock(self));
2365	2477		}
2366	2478
2367	2479		/// Emit a guard that traps with `ebreak` when a comparison is false.

2804	2916		}
2805	2917		}
2806	2918		preds.append(*pred, allocator);
2807	2919		}
2808	2920
	2921	+	/// Return whether construction-time edge data must be published.
	2922	+	fn hasLowerTerminator(terminator: &LowerTerminator) -> bool {
	2923	+	match terminator {
	2924	+	case LowerTerminator::None => return false,
	2925	+	case LowerTerminator::Jmp { .. },
	2926	+	LowerTerminator::Br { .. },
	2927	+	LowerTerminator::Switch { .. } =>
	2928	+	return true,
	2929	+	}
	2930	+	}
	2931	+
	2932	+	/// Publish one block's safely owned edge arguments into its final IL terminator.
	2933	+	unsafe fn publishTerminator(
	2934	+	instr: il::Instr,
	2935	+	terminator: &mut LowerTerminator,
	2936	+	allocator: alloc::Allocator,
	2937	+	) -> il::Instr {
	2938	+	match terminator {
	2939	+	case LowerTerminator::None => return instr,
	2940	+	case LowerTerminator::Jmp { args, .. } => {
	2941	+	let case il::Instr::Jmp { target, .. } = instr
	2942	+	else panic "publishTerminator: expected jump terminator";
	2943	+	return il::Instr::Jmp {
	2944	+	target,
	2945	+	args: (&mut args[..]) as *unsafe mut [il::Val],
	2946	+	};
	2947	+	}
	2948	+	case LowerTerminator::Br { thenArgs, elseArgs, .. } => {
	2949	+	let case il::Instr::Br {
	2950	+	op, typ, a, b, thenTarget, elseTarget, ..
	2951	+	} = instr
	2952	+	else panic "publishTerminator: expected branch terminator";
	2953	+	return il::Instr::Br {
	2954	+	op, typ, a, b,
	2955	+	thenTarget,
	2956	+	thenArgs: (&mut thenArgs[..]) as *unsafe mut [il::Val],
	2957	+	elseTarget,
	2958	+	elseArgs: (&mut elseArgs[..]) as *unsafe mut [il::Val],
	2959	+	};
	2960	+	}
	2961	+	case LowerTerminator::Switch { defaultArgs, cases, .. } => {
	2962	+	let case il::Instr::Switch { val, defaultTarget, .. } = instr
	2963	+	else panic "publishTerminator: expected switch terminator";
	2964	+	let mut publishedCases: *mut [il::SwitchCase] = &mut [];
	2965	+	for i in 0..cases.len {
	2966	+	let branch = &mut cases[i];
	2967	+	publishedCases.append(il::SwitchCase {
	2968	+	value: branch.value,
	2969	+	target: *branch.target,
	2970	+	args: (&mut branch.args[..]) as *unsafe mut [il::Val],
	2971	+	}, allocator);
	2972	+	}
	2973	+	return il::Instr::Switch {
	2974	+	val,
	2975	+	defaultTarget,
	2976	+	defaultArgs: (&mut defaultArgs[..]) as *unsafe mut [il::Val],
	2977	+	cases: (&mut publishedCases[..]) as *unsafe mut [il::SwitchCase],
	2978	+	};
	2979	+	}
	2980	+	}
	2981	+	}
	2982	+
2809	2983		/// Finalize all blocks and return the block array.
2810	2984		unsafe fn finalizeBlocks 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function) -> *unsafe [il::Block] throws (LowerError) where 'arena: 'phase, 'phase: 'function {
2811	2985		let mut blocks: *mut [il::Block] = &mut [];
2812		-	let allocator = alloc::arenaAllocator(self.arena);
	2986	+	let allocator = self.allocator;
2813	2987
2814	2988		for i in 0..self.blockData.len {
2815		-	let data = &mut self.blockData[i];
	2989	+	if hasLowerTerminator(&self.blockData[i].terminator) {
	2990	+	let instrIdx = self.blockData[i].instrs.len - 1;
	2991	+	let instr = self.blockData[i].instrs[instrIdx];
	2992	+	let published = publishTerminator(
	2993	+	instr,
	2994	+	&mut self.blockData[i].terminator,
	2995	+	allocator,
	2996	+	);
	2997	+	set self.blockData[i].instrs[instrIdx] = published;
	2998	+	}
2816	2999
	3000	+	let data = &mut self.blockData[i];
2817	3001		blocks.append(il::Block {
2818	3002		label: data.label,
2819	3003		params: &data.params[..],
2820	3004		instrs: (&mut data.instrs[..]) as *unsafe mut [il::Instr],
2821	3005		locs: &data.locs[..],

2886	3070		}
2887	3071		return result;
2888	3072		}
2889	3073
2890	3074		/// Allocate a single-value slice in the lowering arena.
2891		-	unsafe fn allocVal 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, val: il::Val) -> *unsafe mut [il::Val] throws (LowerError) where 'arena: 'phase, 'phase: 'function {
	3075	+	fn allocVal 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, val: il::Val) -> *mut [il::Val] throws (LowerError) where 'arena: 'phase, 'phase: 'function {
2892	3076		let values = [val];
2893		-	return try copyVals(self, &values[..]);
	3077	+	return valueStorage(&values[..], values.len, self.allocator);
2894	3078		}
2895	3079
2896	3080		////////////////////////
2897	3081		// SSA Var Management //
2898	3082		////////////////////////

3194	3378		let pred = BlockId(self.blockData[*block].preds[index]);
3195	3379		// This may recursively trigger more block arg resolution if the
3196	3380		// predecessor also needs to look up the variable from its predecessors.
3197	3381		let val = try useVarInBlock(self, pred, v);
3198	3382		assert val <> il::Val::Undef, "createBlockParam: predecessor provides undef value for block parameter";
3199		-	patchTerminatorArg(self, pred, *block, paramIdx, val);
	3383	+	patchTerminatorArg(self, pred, block, paramIdx, val);
3200	3384		}
3201	3385		}
3202	3386
3203	3387		/// Check if a block parameter is trivial, i.e. all predecessors provide
3204	3388		/// the same value. Returns the trivial value if so.

3237	3421		/// Patch a single terminator argument for a specific edge. This is used during
3238	3422		/// SSA construction to pass variable values along control flow edges.
3239	3423		fn patchTerminatorArg 'arena 'phase 'function (
3240	3424		self: &mut FnLowerer 'arena 'phase 'function,
3241	3425		from: BlockId, // The predecessor block containing the terminator to patch.
3242		-	target: u32, // The index of the target block we're passing the value to.
	3426	+	target: BlockId, // The target block receiving the value.
3243	3427		paramIdx: u32, // The index of the block parameter to set.
3244	3428		val: il::Val // The value to pass as the argument.
3245	3429		) where 'arena: 'phase, 'phase: 'function {
3246	3430		let allocator = self.allocator;
3247		-	// Get mutable block data by block id.
3248	3431		let data = &mut self.blockData[*from];
3249		-	let ix = data.instrs.len - 1; // The terminator is always the last instruction.
3250		-	patchEdgeArgs(&mut data.instrs[ix], target, paramIdx, val, allocator);
	3432	+	patchEdgeArgs(&mut data.terminator, target, paramIdx, val, allocator);
3251	3433		}
3252	3434
3253		-	/// Patch all edges in a terminator that pass a value to the target block.
3254		-	fn patchEdgeArgs(instr: &mut il::Instr, target: u32, paramIdx: u32, val: il::Val, allocator: alloc::Allocator) {
3255		-	// TODO: We shouldn't need to use a mutable subscript here, given that the
3256		-	// fields are already mutable.
3257		-	match instr {
3258		-	case il::Instr::Jmp { args, .. } => {
3259		-	unsafe {
3260		-	set args = growArgs(args, paramIdx + 1, allocator);
3261		-	set args[paramIdx] = val;
3262		-	}
3263		-	}
3264		-	case il::Instr::Br { thenTarget, thenArgs, elseTarget, elseArgs, .. } => {
3265		-	// Nb. both branches could target the same block (e.g. `if cond { x } else { x }`).
	3435	+	/// Patch all construction-time edges that pass a value to the target block.
	3436	+	fn patchEdgeArgs(
	3437	+	terminator: &mut LowerTerminator,
	3438	+	target: BlockId,
	3439	+	paramIdx: u32,
	3440	+	val: il::Val,
	3441	+	allocator: alloc::Allocator,
	3442	+	) {
	3443	+	let capacity = paramIdx + 1;
	3444	+	match terminator {
	3445	+	case LowerTerminator::None =>
	3446	+	panic "patchEdgeArgs: predecessor has no edge terminator",
	3447	+	case LowerTerminator::Jmp { args, .. } => {
	3448	+	growArgs(args, capacity, allocator);
	3449	+	set args[paramIdx] = val;
	3450	+	}
	3451	+	case LowerTerminator::Br { thenTarget, thenArgs, elseTarget, elseArgs } => {
	3452	+	// Both branches may target the same block.
3266	3453		if *thenTarget == target {
3267		-	unsafe {
3268		-	set thenArgs = growArgs(thenArgs, paramIdx + 1, allocator);
3269		-	set thenArgs[paramIdx] = val;
3270		-	}
	3454	+	growArgs(thenArgs, capacity, allocator);
	3455	+	set thenArgs[paramIdx] = val;
3271	3456		}
3272	3457		if *elseTarget == target {
3273		-	unsafe {
3274		-	set elseArgs = growArgs(elseArgs, paramIdx + 1, allocator);
3275		-	set elseArgs[paramIdx] = val;
3276		-	}
	3458	+	growArgs(elseArgs, capacity, allocator);
	3459	+	set elseArgs[paramIdx] = val;
3277	3460		}
3278	3461		}
3279		-	case il::Instr::Switch { defaultTarget, defaultArgs, cases, .. } => {
	3462	+	case LowerTerminator::Switch { defaultTarget, defaultArgs, cases } => {
3280	3463		if *defaultTarget == target {
3281		-	unsafe {
3282		-	set defaultArgs = growArgs(defaultArgs, paramIdx + 1, allocator);
3283		-	set defaultArgs[paramIdx] = val;
3284		-	}
	3464	+	growArgs(defaultArgs, capacity, allocator);
	3465	+	set defaultArgs[paramIdx] = val;
3285	3466		}
3286		-	unsafe {
3287		-	patchSwitchArgs(*cases, target, paramIdx, val, allocator);
3288		-	}
3289		-	}
3290		-	else => {
3291		-	// Other terminators (e.g. `Ret`, `Unreachable`) don't have successor blocks.
	3467	+	patchSwitchArgs(&mut cases[..], target, paramIdx, val, allocator);
3292	3468		}
3293	3469		}
3294	3470		}
3295	3471
3296		-	/// Patch every switch case that passes a value to the target block.
3297		-	fn patchSwitchArgs(cases: &mut [il::SwitchCase], target: u32, paramIdx: u32, val: il::Val, allocator: alloc::Allocator) {
	3472	+	/// Patch every construction-time switch case targeting the given block.
	3473	+	fn patchSwitchArgs(
	3474	+	cases: &mut [LowerSwitchCase],
	3475	+	target: BlockId,
	3476	+	paramIdx: u32,
	3477	+	val: il::Val,
	3478	+	allocator: alloc::Allocator,
	3479	+	) {
3298	3480		let capacity = paramIdx + 1;
3299	3481		for i in 0..cases.len {
3300	3482		let branch = &mut cases[i];
3301	3483		if branch.target == target {
3302		-	unsafe {
3303		-	set branch.args = growArgs(branch.args, capacity, allocator);
3304		-	set branch.args[paramIdx] = val;
3305		-	}
	3484	+	growArgs(&mut branch.args, capacity, allocator);
	3485	+	set branch.args[paramIdx] = val;
3306	3486		}
3307	3487		}
3308	3488		}
3309	3489
3310		-	/// Grow an args array to hold at least the given capacity.
3311		-	unsafe fn growArgs(args: unsafe mut [il::Val], capacity: u32, allocator: alloc::Allocator) -> unsafe mut [il::Val] {
	3490	+	/// Grow an argument array to hold the requested capacity.
	3491	+	fn growArgs(args: &mut *mut [il::Val], capacity: u32, allocator: alloc::Allocator) {
3312	3492		if args.len >= capacity {
3313		-	return args;
	3493	+	return;
3314	3494		}
3315		-	let newArgs = valueStorage(args, capacity, allocator);
3316		-	return (&mut newArgs[..]) as *unsafe mut [il::Val];
	3495	+	set *args = valueStorage(&args[..], capacity, allocator);
3317	3496		}
3318	3497
3319	3498		/// Select a receiver or explicit parameter name from its AST declaration.
3320	3499		fn paramName(params: &[ast::Node], receiver: ?ast::Node, index: u32) -> *[u8] throws (LowerError) {
3321	3500		let mut position = index;

3816	3995		}
3817	3996		}
3818	3997		return try finalizeBlocks(self);
3819	3998		}
3820	3999
3821		-	/// Append an initialized switch case to owned storage.
	4000	+	/// Append a construction-time switch case with safely owned arguments.
3822	4001		fn appendSwitchCase(
3823		-	cases: &mut *mut [il::SwitchCase],
	4002	+	cases: &mut *mut [LowerSwitchCase],
3824	4003		value: i64,
3825	4004		target: BlockId,
3826		-	args: *unsafe mut [il::Val],
	4005	+	args: *mut [il::Val],
3827	4006		allocator: alloc::Allocator,
3828	4007		) {
3829		-	cases.append(il::SwitchCase { value, target: *target, args }, allocator);
	4008	+	cases.append(LowerSwitchCase { value, target, args }, allocator);
3830	4009		}
3831	4010
3832	4011		/// Lower a scalar match as a switch instruction.
3833	4012		unsafe fn lowerMatchSwitch 'arena 'phase 'function (self: &mut FnLowerer 'arena 'phase 'function, prongs: [ast::Node], subject: &MatchSubject, mergeBlock: &mut ?BlockId) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
3834	4013		let mut blocks: *mut [BlockId] = &mut [];
3835		-	let mut cases: *mut [il::SwitchCase] = &mut [];
	4014	+	let mut cases: *mut [LowerSwitchCase] = &mut [];
3836	4015		let mut defaultIdx: u32 = 0;
3837	4016		let entry = currentBlock(self);
3838	4017
3839	4018		for p, i in prongs {
3840	4019		let case ast::NodeValue::MatchProng(prong) = p.value

3855	4034		}
3856	4035		}
3857	4036		}
3858	4037		addPredecessor(self, blocks[i], entry);
3859	4038		}
3860		-	emit(self, il::Instr::Switch {
3861		-	val: subject.val,
3862		-	defaultTarget: *blocks[defaultIdx],
3863		-	defaultArgs: &mut [],
3864		-	cases: (&mut cases[..]) as *unsafe mut [il::SwitchCase]
3865		-	});
	4039	+	emitSwitchTerminator(
	4040	+	self,
	4041	+	subject.val,
	4042	+	blocks[defaultIdx],
	4043	+	LowerTerminator::Switch {
	4044	+	defaultTarget: blocks[defaultIdx],
	4045	+	defaultArgs: &mut [],
	4046	+	cases,
	4047	+	},
	4048	+	);
3866	4049
3867	4050		for p, i in prongs {
3868	4051		let case ast::NodeValue::MatchProng(prong) = p.value
3869	4052		else throw LowerError::UnexpectedNodeValue(p);
3870	4053

4866	5049
4867	5050		let falseArgs = try allocVal(self, il::Val::Imm(0));
4868	5051		let trueArgs = try allocVal(self, il::Val::Imm(1));
4869	5052
4870	5053		// Check if tags differ.
4871		-	emit(self, il::Instr::Br {
4872		-	op: il::CmpOp::Eq, typ: il::Type::W8, a: tagA, b: tagB,
4873		-	thenTarget: *nilCheck, thenArgs: &mut [],
4874		-	elseTarget: *mergeBlock, elseArgs: falseArgs,
4875		-	});
	5054	+	emitBrTerminator(
	5055	+	self,
	5056	+	il::CmpOp::Eq, il::Type::W8, tagA, tagB,
	5057	+	nilCheck, mergeBlock,
	5058	+	LowerTerminator::Br {
	5059	+	thenTarget: nilCheck, thenArgs: &mut [],
	5060	+	elseTarget: mergeBlock, elseArgs: falseArgs,
	5061	+	},
	5062	+	);
4876	5063		addPredecessor(self, nilCheck, currentBlock(self));
4877	5064		addPredecessor(self, mergeBlock, currentBlock(self));
4878	5065
4879	5066		// Check if both are `nil`.
4880	5067		try switchToAndSeal(self, nilCheck);
4881		-	emit(self, il::Instr::Br {
4882		-	op: il::CmpOp::Ne, typ: il::Type::W8, a: tagA, b: il::Val::Imm(0),
4883		-	thenTarget: *payloadCmp, thenArgs: &mut [],
4884		-	elseTarget: *mergeBlock, elseArgs: trueArgs,
4885		-	});
	5068	+	emitBrTerminator(
	5069	+	self,
	5070	+	il::CmpOp::Ne, il::Type::W8, tagA, il::Val::Imm(0),
	5071	+	payloadCmp, mergeBlock,
	5072	+	LowerTerminator::Br {
	5073	+	thenTarget: payloadCmp, thenArgs: &mut [],
	5074	+	elseTarget: mergeBlock, elseArgs: trueArgs,
	5075	+	},
	5076	+	);
4886	5077		addPredecessor(self, payloadCmp, currentBlock(self));
4887	5078		addPredecessor(self, mergeBlock, currentBlock(self));
4888	5079
4889	5080		// Both are non-`nil`, compare payloads.
4890	5081		try switchToAndSeal(self, payloadCmp);

4946	5137		// Compare tags: if they differ, jump to merge with `false`; otherwise check payloads.
4947	5138		let falseArgs = try allocVal(self, il::Val::Imm(0));
4948	5139
4949	5140		assert tagBlock <> mergeBlock;
4950	5141
4951		-	// TODO: Use the helper once the compiler supports more than eight function params.
4952		-	emit(self, il::Instr::Br {
4953		-	op: il::CmpOp::Eq, typ: il::Type::W8, a: tagA, b: tagB,
4954		-	thenTarget: *tagBlock, thenArgs: &mut [],
4955		-	elseTarget: *mergeBlock, elseArgs: falseArgs,
4956		-	});
	5142	+	emitBrTerminator(
	5143	+	self,
	5144	+	il::CmpOp::Eq, il::Type::W8, tagA, tagB,
	5145	+	tagBlock, mergeBlock,
	5146	+	LowerTerminator::Br {
	5147	+	thenTarget: tagBlock, thenArgs: &mut [],
	5148	+	elseTarget: mergeBlock, elseArgs: falseArgs,
	5149	+	},
	5150	+	);
4957	5151		addPredecessor(self, tagBlock, currentBlock(self));
4958	5152		addPredecessor(self, mergeBlock, currentBlock(self));
4959	5153
4960	5154		// Create comparison blocks for each non-void variant and build switch cases.
4961	5155		// Void variants jump directly to merge with `true`.
4962		-	let trueArgs = try allocVal(self, il::Val::Imm(1));
4963		-	let mut cases: *mut [il::SwitchCase] = &mut [];
	5156	+	let mut cases: *mut [LowerSwitchCase] = &mut [];
	5157	+	let mut caseBlocks: *mut [BlockId] = &mut [];
4964	5158		for variant, i in unionInfo.variants {
4965	5159		if variant.valueType == resolver::Type::Void {
	5160	+	let trueArgs = try allocVal(self, il::Val::Imm(1));
	5161	+	caseBlocks.append(mergeBlock, alloc::arenaAllocator(self.arena));
4966	5162		appendSwitchCase(&mut cases, i as i64, mergeBlock, trueArgs, alloc::arenaAllocator(self.arena));
4967	5163		} else {
4968	5164		let payloadBlock = try createBlock(self, "eq#payload");
	5165	+	caseBlocks.append(payloadBlock, alloc::arenaAllocator(self.arena));
4969	5166		appendSwitchCase(&mut cases, i as i64, payloadBlock, &mut [], alloc::arenaAllocator(self.arena));
4970	5167		}
4971	5168		}
4972	5169
4973	5170		// Emit switch in @tag block. Default arm is unreachable since we cover all variants.
4974	5171		let unreachableBlock = try createBlock(self, "eq#unreachable");
4975	5172		try switchToAndSeal(self, tagBlock);
4976		-	emit(self, il::Instr::Switch {
4977		-	val: tagA,
4978		-	defaultTarget: *unreachableBlock,
4979		-	defaultArgs: &mut [],
4980		-	cases: (&mut cases[..]) as *unsafe mut [il::SwitchCase]
4981		-	});
	5173	+	emitSwitchTerminator(
	5174	+	self,
	5175	+	tagA,
	5176	+	unreachableBlock,
	5177	+	LowerTerminator::Switch {
	5178	+	defaultTarget: unreachableBlock,
	5179	+	defaultArgs: &mut [],
	5180	+	cases,
	5181	+	},
	5182	+	);
4982	5183
4983	5184		// Add predecessor edges for switch targets.
4984	5185		addPredecessor(self, unreachableBlock, tagBlock);
4985		-	for c in &cases[..] {
4986		-	addPredecessor(self, BlockId(c.target), tagBlock);
	5186	+	for caseBlock in &caseBlocks[..] {
	5187	+	addPredecessor(self, caseBlock, tagBlock);
4987	5188		}
4988	5189		let valOffset = unionInfo.valOffset as i32;
4989	5190
4990	5191		// Emit payload comparison blocks for non-void variants.
4991	5192		for variant, i in unionInfo.variants {
4992		-	let caseBlock = BlockId(cases[i].target);
	5193	+	let caseBlock = caseBlocks[i];
4993	5194		if caseBlock <> mergeBlock {
4994	5195		try switchToAndSeal(self, caseBlock);
4995	5196		let payloadEq = try emitEqAtOffset(
4996	5197		self, a, b, offset + valOffset, variant.valueType
4997	5198		);

6753	6954		) throws (LowerError) where 'arena: 'phase, 'phase: 'function {
6754	6955		let entry = currentBlock(self);
6755	6956
6756	6957		// First pass: create blocks, resolve error types, and build switch cases.
6757	6958		let mut targets: [?CatchTarget; MAX_CATCH_CLAUSES] = [nil; MAX_CATCH_CLAUSES];
6758		-	let mut cases: *mut [il::SwitchCase] = &mut [];
	6959	+	let mut cases: *mut [LowerSwitchCase] = &mut [];
6759	6960		let mut defaultIdx: ?u32 = nil;
6760	6961
6761	6962		for clauseNode, i in catches {
6762	6963		let case ast::NodeValue::CatchClause(clause) = clauseNode.value
6763	6964		else panic "lowerMultiCatch: expected CatchClause";

6785	6986		let block = try createBlock(self, "unreachable");
6786	6987		addPredecessor(self, block, entry);
6787	6988		set selectedDefault = block;
6788	6989		}
6789	6990		let defaultTarget = selectedDefault else panic "lowerMultiCatch: missing default destination";
6790		-	emit(self, il::Instr::Switch {
6791		-	val: il::Val::Reg(tagReg),
6792		-	defaultTarget: *defaultTarget,
6793		-	defaultArgs: &mut [],
6794		-	cases: (&mut cases[..]) as *unsafe mut [il::SwitchCase]
6795		-	});
	6991	+	emitSwitchTerminator(
	6992	+	self,
	6993	+	il::Val::Reg(tagReg),
	6994	+	defaultTarget,
	6995	+	LowerTerminator::Switch {
	6996	+	defaultTarget,
	6997	+	defaultArgs: &mut [],
	6998	+	cases,
	6999	+	},
	7000	+	);
6796	7001
6797	7002		// Second pass: emit each catch clause body.
6798	7003		for clauseNode, i in catches {
6799	7004		let case ast::NodeValue::CatchClause(clause) = clauseNode.value
6800	7005		else panic "lowerMultiCatch: expected CatchClause";