lib/std/arch/rv64/emit.rad 33.2 KiB raw
1
//! RV64 binary emission.
2
//!
3
//! Emits RV64 machine code as `u32` list.
4
5
use std::lang::il;
6
use std::lang::alloc;
7
use std::lang::gen;
8
use std::lang::gen::data;
9
use std::lang::gen::labels;
10
use std::lang::gen::types;
11
use std::collections::dict;
12
use std::mem;
13
14
use super::encode;
15
16
/// Maximum number of instructions in code buffer.
17
export constant MAX_INSTRS: u32 = 2097152;
18
/// Maximum code length before byte offset overflows signed 32-bits.
19
constant MAX_CODE_LEN: u32 = 0x7FFFFFFF / super::INSTR_SIZE as u32;
20
/// Maximum number of pending branches awaiting patching.
21
export constant MAX_PENDING: u32 = 65536;
22
/// Maximum number of function entries.
23
export constant MAX_FUNCS: u32 = 4096;
24
/// Maximum number of debug entries.
25
export constant MAX_DEBUG_ENTRIES: u32 = 524288;
26
27
//////////////////////
28
// Emission Context //
29
//////////////////////
30
31
/// Branch/jump that needs offset patching after all blocks are emitted.
32
export record PendingBranch: Copy {
33
    /// Index into code buffer where the branch instruction is.
34
    index: u32,
35
    /// Target block index.
36
    target: u32,
37
    /// Type of branch for re-encoding.
38
    kind: BranchKind,
39
}
40
41
/// Type of branch instruction.
42
export union BranchKind: Copy {
43
    /// Conditional branch (B-type encoding).
44
    Cond { op: il::CmpOp, rs1: gen::Reg, rs2: gen::Reg },
45
    /// Inverted conditional branch (B-type encoding with negated condition).
46
    InvertedCond { op: il::CmpOp, rs1: gen::Reg, rs2: gen::Reg },
47
    /// Unconditional jump (J-type encoding).
48
    Jump,
49
}
50
51
/// Function call that needs offset patching.
52
export record PendingCall: Copy {
53
    /// Index in code buffer where the call was emitted.
54
    index: u32,
55
    /// Target function name.
56
    target: *[u8],
57
}
58
59
/// Assembly jump that needs offset patching after all text is emitted.
60
export record PendingJump: Copy {
61
    /// Index in code buffer where the jump was emitted.
62
    index: u32,
63
    /// Target function name.
64
    target: *[u8],
65
    /// Destination register.
66
    rd: gen::Reg,
67
}
68
69
/// Address load that needs patching after layout is known.
70
export record PendingAddrLoad: Copy {
71
    /// Index in code buffer where the load was emitted.
72
    index: u32,
73
    /// Target function or data symbol name.
74
    target: *[u8],
75
    /// Destination register.
76
    rd: gen::Reg,
77
    /// Whether this is an absolute data symbol address.
78
    isData: bool,
79
}
80
81
/// Adjusted base register and offset for addressing.
82
export record AdjustedOffset: Copy {
83
    /// Base register.
84
    base: gen::Reg,
85
    /// Byte offset from register.
86
    offset: i32,
87
}
88
89
/// Callee-saved register with its stack offset.
90
export record SavedReg: Copy {
91
    /// Register to save/restore.
92
    reg: gen::Reg,
93
    /// Offset from SP.
94
    offset: i32,
95
}
96
97
/// Emission context. Tracks state during code generation.
98
export record Emitter {
99
    /// First failure. Further emission stops until the workspace is rebuilt.
100
    error: ?super::Error,
101
    /// Data address loads use the current domain package-state table.
102
    sharedData: bool,
103
    /// Emitted instructions storage.
104
    code: *mut [u32],
105
    /// Current number of emitted instructions.
106
    codeLen: u32,
107
    /// Local branches needing offset patching.
108
    pendingBranches: *mut [PendingBranch],
109
    /// Number of local branches that need patching.
110
    pendingBranchesLen: u32,
111
    /// Function calls needing offset patching.
112
    pendingCalls: *mut [PendingCall],
113
    /// Number of function calls that need patching.
114
    pendingCallsLen: u32,
115
    /// Assembly jumps needing offset patching.
116
    pendingJumps: *mut [PendingJump],
117
    /// Number of assembly jumps that need patching.
118
    pendingJumpsLen: u32,
119
    /// Function address loads needing offset patching.
120
    pendingAddrLoads: *mut [PendingAddrLoad],
121
    /// Number of function address loads that need patching.
122
    pendingAddrLoadsLen: u32,
123
    /// Block label tracking.
124
    labels: labels::Labels,
125
    /// Function start positions for printing.
126
    funcs: *mut [types::FuncAddr],
127
    /// Number of function start positions recorded.
128
    funcsLen: u32,
129
    /// Debug entries mapping PCs to source locations.
130
    debugEntries: *mut [types::DebugEntry],
131
    /// Number of debug entries recorded.
132
    debugEntriesLen: u32,
133
}
134
135
/// Caller-owned storage for program emission.
136
export record Storage {
137
    /// Emitted instruction buffer.
138
    code: *mut [u32],
139
    /// Local branch patch buffer.
140
    pendingBranches: *mut [PendingBranch],
141
    /// Function call patch buffer.
142
    pendingCalls: *mut [PendingCall],
143
    /// Assembly jump patch buffer.
144
    pendingJumps: *mut [PendingJump],
145
    /// Address load patch buffer.
146
    pendingAddrLoads: *mut [PendingAddrLoad],
147
    /// Per-function block offset buffer.
148
    blockOffsets: *mut [i32],
149
    /// Function label hash table storage.
150
    funcEntries: *mut [dict::Entry],
151
    /// Printed function address buffer.
152
    funcs: *mut [types::FuncAddr],
153
    /// Debug location buffer.
154
    debugEntries: *mut [types::DebugEntry],
155
}
156
157
/// Computed stack frame layout for a function.
158
export record Frame: Copy {
159
    /// Total frame size in bytes (aligned).
160
    totalSize: i32,
161
    /// Callee-saved registers and their offsets.
162
    // TODO: Use constant length when language supports it.
163
    savedRegs: [SavedReg; super::NUM_SAVED_REGISTERS],
164
    /// Number of saved registers.
165
    savedRegsLen: u32,
166
    /// Epilogue block index for return jumps.
167
    epilogueBlock: u32,
168
    /// Whether this is a leaf function. Leaf functions
169
    /// skip saving/restoring RA since it is never clobbered.
170
    isLeaf: bool,
171
    /// Whether the function has dynamic stack allocations.
172
    /// When false, SP never changes after the prologue.
173
    isDynamic: bool,
174
}
175
176
/// Compute frame layout from local size and used callee-saved registers.
177
export fn computeFrame(localSize: i32, usedCalleeSaved: u32, epilogueBlock: u32, isLeaf: bool, isDynamic: bool) -> Frame {
178
    let mut frame = Frame {
179
        totalSize: 0,
180
        savedRegs: [SavedReg { reg: super::ZERO, offset: 0 }; super::NUM_SAVED_REGISTERS],
181
        savedRegsLen: 0,
182
        epilogueBlock,
183
        isLeaf,
184
        isDynamic,
185
    };
186
    // Skip frame allocation for leaf functions with no locals and no
187
    // callee-saved registers. Leaf functions don't call other functions,
188
    // so RA is never clobbered and doesn't need saving.
189
    if isLeaf and localSize == 0 and usedCalleeSaved == 0 {
190
        return frame;
191
    }
192
    // Compute total frame size. Includes RA and FP registers.
193
    let savedRegs = mem::popCount(usedCalleeSaved) + 2;
194
    let totalSize = mem::alignUpI32(
195
        localSize + savedRegs * super::DWORD_SIZE,
196
        super::STACK_ALIGNMENT
197
    );
198
    set frame.totalSize = totalSize;
199
200
    // Build list of callee-saved registers with offsets.
201
    let mut offset = totalSize - (super::DWORD_SIZE * 3);
202
    for reg, i in super::CALLEE_SAVED {
203
        // Check if this register is in use.
204
        if (usedCalleeSaved & (1 << i)) <> 0 {
205
            set frame.savedRegs[frame.savedRegsLen] = SavedReg {
206
                reg,
207
                offset,
208
            };
209
            set frame.savedRegsLen += 1;
210
            set offset -= super::DWORD_SIZE;
211
        }
212
    }
213
    return frame;
214
}
215
216
/// Create a new emitter.
217
export unsafe fn emitter(arena: &mut alloc::Arena, debug: bool) -> Emitter throws (alloc::AllocError) {
218
    let storage = try allocateStorage(arena, debug);
219
    return emitterWithStorage(storage, debug);
220
}
221
222
/// Allocate emitter storage from an arena.
223
export unsafe fn allocateStorage(arena: &mut alloc::Arena, debug: bool) -> Storage throws (alloc::AllocError) {
224
    let code = try alloc::allocSlice(arena, @sizeOf(u32), @alignOf(u32), MAX_INSTRS);
225
    let pendingBranches = try alloc::allocSlice(arena, @sizeOf(PendingBranch), @alignOf(PendingBranch), MAX_PENDING);
226
    let pendingCalls = try alloc::allocSlice(arena, @sizeOf(PendingCall), @alignOf(PendingCall), MAX_PENDING);
227
    let pendingJumps = try alloc::allocSlice(arena, @sizeOf(PendingJump), @alignOf(PendingJump), MAX_PENDING);
228
    let pendingAddrLoads = try alloc::allocSlice(arena, @sizeOf(PendingAddrLoad), @alignOf(PendingAddrLoad), MAX_PENDING);
229
    let blockOffsets = try alloc::allocSlice(arena, @sizeOf(i32), @alignOf(i32), labels::MAX_BLOCKS_PER_FN);
230
    let funcEntries = try alloc::allocSlice(arena, @sizeOf(dict::Entry), @alignOf(dict::Entry), labels::FUNC_TABLE_SIZE);
231
    let funcs = try alloc::allocSlice(arena, @sizeOf(types::FuncAddr), @alignOf(types::FuncAddr), MAX_FUNCS);
232
233
    let mut debugEntries: *mut [types::DebugEntry] = &mut [];
234
    if debug {
235
        set debugEntries = try alloc::allocSlice(
236
            arena, @sizeOf(types::DebugEntry), @alignOf(types::DebugEntry), MAX_DEBUG_ENTRIES
237
        ) as *mut [types::DebugEntry];
238
    }
239
    return Storage {
240
        code: code as *mut [u32],
241
        pendingBranches: pendingBranches as *mut [PendingBranch],
242
        pendingCalls: pendingCalls as *mut [PendingCall],
243
        pendingJumps: pendingJumps as *mut [PendingJump],
244
        pendingAddrLoads: pendingAddrLoads as *mut [PendingAddrLoad],
245
        blockOffsets: blockOffsets as *mut [i32],
246
        funcEntries: funcEntries as *mut [dict::Entry],
247
        funcs: funcs as *mut [types::FuncAddr],
248
        debugEntries,
249
    };
250
}
251
252
/// Create an emitter from caller-owned storage.
253
export fn emitterWithStorage(storage: Storage, debug: bool) -> Emitter {
254
    let case Storage {
255
        code, pendingBranches, pendingCalls, pendingJumps, pendingAddrLoads,
256
        blockOffsets, funcEntries, funcs, debugEntries,
257
    } = storage else panic "expected emitter storage";
258
    assert code.len >= MAX_INSTRS, "emitterWithStorage: code buffer too small";
259
    assert pendingBranches.len >= MAX_PENDING, "emitterWithStorage: branch buffer too small";
260
    assert pendingCalls.len >= MAX_PENDING, "emitterWithStorage: call buffer too small";
261
    assert pendingJumps.len >= MAX_PENDING, "emitterWithStorage: jump buffer too small";
262
    assert pendingAddrLoads.len >= MAX_PENDING, "emitterWithStorage: address buffer too small";
263
    assert blockOffsets.len >= labels::MAX_BLOCKS_PER_FN, "emitterWithStorage: block buffer too small";
264
    assert funcEntries.len >= labels::FUNC_TABLE_SIZE, "emitterWithStorage: label table too small";
265
    assert funcs.len >= MAX_FUNCS, "emitterWithStorage: function buffer too small";
266
    let mut activeDebugEntries: *mut [types::DebugEntry] = &mut [];
267
    if debug {
268
        assert debugEntries.len >= MAX_DEBUG_ENTRIES, "emitterWithStorage: debug buffer too small";
269
        set activeDebugEntries = debugEntries;
270
    }
271
    return Emitter {
272
        error: nil,
273
        sharedData: false,
274
        code,
275
        codeLen: 0,
276
        pendingBranches,
277
        pendingBranchesLen: 0,
278
        pendingCalls,
279
        pendingCallsLen: 0,
280
        pendingJumps,
281
        pendingJumpsLen: 0,
282
        pendingAddrLoads,
283
        pendingAddrLoadsLen: 0,
284
        labels: labels::init(blockOffsets, funcEntries),
285
        funcs,
286
        funcsLen: 0,
287
        debugEntries: activeDebugEntries,
288
        debugEntriesLen: 0,
289
    };
290
}
291
292
///////////////////////
293
// Emission Helpers  //
294
///////////////////////
295
296
/// Emit a single instruction.
297
export fn emit(e: &mut Emitter, instr: u32) {
298
    if e.error <> nil {
299
        return;
300
    }
301
    if e.codeLen == e.code.len {
302
        set e.error = super::Error::Capacity;
303
        return;
304
    }
305
    set e.code[e.codeLen] = instr;
306
    set e.codeLen += 1;
307
}
308
309
/// Compute branch offset to a function by name.
310
export fn branchOffsetToFunc(e: &mut Emitter, srcIndex: u32, name: *[u8]) -> i32 {
311
    if e.error <> nil {
312
        return 0;
313
    }
314
    let target = dict::get(&e.labels.funcs, name) else {
315
        set e.error = super::Error::Symbol; return 0;
316
    };
317
    return target - srcIndex as i32 * super::INSTR_SIZE;
318
}
319
320
/// Patch an instruction at a given index.
321
export fn patch(e: &mut Emitter, index: u32, instr: u32) {
322
    if e.error <> nil {
323
        return;
324
    }
325
    if index >= e.codeLen {
326
        set e.error = super::Error::Capacity;
327
        return;
328
    }
329
    set e.code[index] = instr;
330
}
331
332
/// Record a block's address for branch resolution.
333
export fn recordBlock(e: &mut Emitter, blockIdx: u32) {
334
    if e.error <> nil {
335
        return;
336
    }
337
    if e.codeLen > MAX_CODE_LEN or blockIdx >= e.labels.blockOffsets.len {
338
        set e.error = super::Error::Capacity; return;
339
    }
340
    labels::recordBlock(&mut e.labels, blockIdx, e.codeLen as i32 * super::INSTR_SIZE);
341
}
342
343
/// Record a function's code offset for call resolution.
344
export fn recordFuncOffset(e: &mut Emitter, name: *[u8]) {
345
    let codeLen = e.codeLen;
346
    recordFuncOffsetAt(e, name, codeLen);
347
}
348
349
/// Record a function's code offset at `index` for call resolution.
350
export fn recordFuncOffsetAt(e: &mut Emitter, name: *[u8], index: u32) {
351
    if e.error <> nil {
352
        return;
353
    }
354
    if index > MAX_CODE_LEN {
355
        set e.error = super::Error::Capacity;
356
        return;
357
    }
358
    if name.len == 0 {
359
        set e.error = super::Error::Symbol;
360
        return;
361
    }
362
    if e.labels.funcs.count >= e.labels.funcs.entries.len / 2 and dict::get(&e.labels.funcs, name) == nil {
363
        set e.error = super::Error::Capacity; return;
364
    }
365
    dict::insert(&mut e.labels.funcs, name, index as i32 * super::INSTR_SIZE);
366
}
367
368
/// Record a function's start position for printing.
369
export fn recordFunc(e: &mut Emitter, name: *[u8]) {
370
    let codeLen = e.codeLen;
371
    recordFuncAt(e, name, codeLen);
372
}
373
374
/// Record a function's start position at `index` for printing.
375
export fn recordFuncAt(e: &mut Emitter, name: *[u8], index: u32) {
376
    if e.error <> nil {
377
        return;
378
    }
379
    if e.funcsLen == e.funcs.len {
380
        set e.error = super::Error::Capacity;
381
        return;
382
    }
383
    set e.funcs[e.funcsLen] = types::FuncAddr { name, index };
384
    set e.funcsLen += 1;
385
}
386
387
/// Record a local branch needing later patching.
388
/// Unconditional jumps use a single slot (J-type, +-1MB range).
389
/// Conditional branches use two slots (B-type has only +-4KB range,
390
/// so large functions may need the inverted-branch + JAL fallback).
391
export fn recordBranch(e: &mut Emitter, targetBlock: u32, kind: BranchKind) {
392
    if e.error <> nil {
393
        return;
394
    }
395
    if e.pendingBranchesLen == e.pendingBranches.len {
396
        set e.error = super::Error::Capacity;
397
        return;
398
    }
399
    set e.pendingBranches[e.pendingBranchesLen] = PendingBranch {
400
        index: e.codeLen,
401
        target: targetBlock,
402
        kind,
403
    };
404
    set e.pendingBranchesLen += 1;
405
406
    emit(e, encode::nop()); // First slot, always needed.
407
408
    match kind {
409
        case BranchKind::Jump => {},
410
        else => emit(e, encode::nop()), // Second slot for conditional branches.
411
    }
412
}
413
414
/// Record a function call needing later patching.
415
/// Emits placeholder instructions that will be patched later.
416
/// Uses two slots to support long-distance calls.
417
export fn recordCall(e: &mut Emitter, target: *[u8]) {
418
    if e.error <> nil {
419
        return;
420
    }
421
    if e.pendingCallsLen == e.pendingCalls.len {
422
        set e.error = super::Error::Capacity;
423
        return;
424
    }
425
    set e.pendingCalls[e.pendingCallsLen] = PendingCall {
426
        index: e.codeLen,
427
        target,
428
    };
429
    set e.pendingCallsLen += 1;
430
431
    emit(e, encode::nop()); // Placeholder for AUIPC.
432
    emit(e, encode::nop()); // Placeholder for JALR.
433
}
434
435
/// Record a jump emitted by assembly that needs whole-program patching.
436
export fn recordJumpAt(e: &mut Emitter, target: *[u8], rd: gen::Reg, index: u32) {
437
    if e.error <> nil {
438
        return;
439
    }
440
    if e.pendingJumpsLen == e.pendingJumps.len {
441
        set e.error = super::Error::Capacity;
442
        return;
443
    }
444
    set e.pendingJumps[e.pendingJumpsLen] = PendingJump {
445
        index,
446
        target,
447
        rd,
448
    };
449
    set e.pendingJumpsLen += 1;
450
}
451
452
/// Record a function address load needing later patching.
453
/// Emits placeholder instructions that will be patched to load the function's address.
454
/// Uses two slots to compute long-distance addresses.
455
export fn recordAddrLoad(e: &mut Emitter, target: *[u8], rd: gen::Reg) {
456
    let codeLen = e.codeLen;
457
    recordAddrLoadAt(e, target, rd, codeLen);
458
459
    emit(e, encode::nop()); // Placeholder for AUIPC.
460
    emit(e, encode::nop()); // Placeholder for ADDI.
461
}
462
463
/// Record a function address load already reserved by assembly.
464
export fn recordAddrLoadAt(e: &mut Emitter, target: *[u8], rd: gen::Reg, index: u32) {
465
    if e.error <> nil {
466
        return;
467
    }
468
    if e.pendingAddrLoadsLen == e.pendingAddrLoads.len {
469
        set e.error = super::Error::Capacity;
470
        return;
471
    }
472
    set e.pendingAddrLoads[e.pendingAddrLoadsLen] = PendingAddrLoad {
473
        index,
474
        target,
475
        rd,
476
        isData: false,
477
    };
478
    set e.pendingAddrLoadsLen += 1;
479
}
480
481
/// Record a data address load needing later patching.
482
/// Reserves two instructions for a PC-relative address load.
483
export fn recordDataAddrLoad(e: &mut Emitter, target: *[u8], rd: gen::Reg) {
484
    if e.error <> nil {
485
        return;
486
    }
487
    if e.pendingAddrLoadsLen == e.pendingAddrLoads.len {
488
        set e.error = super::Error::Capacity;
489
        return;
490
    }
491
    set e.pendingAddrLoads[e.pendingAddrLoadsLen] = PendingAddrLoad {
492
        index: e.codeLen,
493
        target,
494
        rd,
495
        isData: true,
496
    };
497
    set e.pendingAddrLoadsLen += 1;
498
499
    emit(e, encode::nop()); // Address-load upper instruction.
500
    emit(e, encode::nop()); // Address-load lower instruction.
501
    if e.sharedData {
502
        emit(e, encode::nop()); // Package offset addition.
503
        emit(e, encode::nop()); // Package offset low bits.
504
    }
505
}
506
507
/// Patch local branches and clear the pending list.
508
///
509
/// Called after each function.
510
///
511
/// Uses two-instruction sequences: short branches use `branch` and `nop`,
512
/// long branches use inverted branch  and `jal` or `auipc` and `jalr`.
513
export fn patchLocalBranches(e: &mut Emitter) {
514
    if e.error <> nil {
515
        return;
516
    }
517
    for i in 0..e.pendingBranchesLen {
518
        let p = e.pendingBranches[i];
519
        if p.target >= e.labels.blockCount {
520
            set e.error = super::Error::Symbol;
521
            return;
522
        }
523
        let offset = labels::branchToBlock(&e.labels, p.index, p.target, super::INSTR_SIZE);
524
        match p.kind {
525
            case BranchKind::Cond { op, rs1, rs2 } => {
526
                if encode::isBranchImm(offset) {
527
                    patch(e, p.index, encodeCondBranch(op, rs1, rs2, offset));
528
                    patch(e, p.index + 1, encode::nop());
529
                } else {
530
                    let adj = offset - super::INSTR_SIZE;
531
                    if not encode::isJumpImm(adj) {
532
                        set e.error = super::Error::Relocation;
533
                        return;
534
                    }
535
                    patch(e, p.index, encodeInvertedBranch(op, rs1, rs2, super::INSTR_SIZE * 2));
536
                    patch(e, p.index + 1, encode::jal(super::ZERO, adj));
537
                }
538
            },
539
            case BranchKind::InvertedCond { op, rs1, rs2 } => {
540
                if encode::isBranchImm(offset) {
541
                    patch(e, p.index, encodeInvertedBranch(op, rs1, rs2, offset));
542
                    patch(e, p.index + 1, encode::nop());
543
                } else {
544
                    let adj = offset - super::INSTR_SIZE;
545
                    if not encode::isJumpImm(adj) {
546
                        set e.error = super::Error::Relocation;
547
                        return;
548
                    }
549
                    patch(e, p.index, encodeCondBranch(op, rs1, rs2, super::INSTR_SIZE * 2));
550
                    patch(e, p.index + 1, encode::jal(super::ZERO, adj));
551
                }
552
            },
553
            case BranchKind::Jump => {
554
                // Single-slot jump (J-type, +-1MB range).
555
                if not encode::isJumpImm(offset) {
556
                    set e.error = super::Error::Relocation;
557
                    return;
558
                }
559
                patch(e, p.index, encode::jal(super::ZERO, offset));
560
            },
561
        }
562
    }
563
    set e.pendingBranchesLen = 0;
564
}
565
566
/// Encode a conditional branch instruction.
567
fn encodeCondBranch(op: il::CmpOp, rs1: gen::Reg, rs2: gen::Reg, offset: i32) -> u32 {
568
    match op {
569
        case il::CmpOp::Eq => return encode::beq(rs1, rs2, offset),
570
        case il::CmpOp::Ne => return encode::bne(rs1, rs2, offset),
571
        case il::CmpOp::Slt => return encode::blt(rs1, rs2, offset),
572
        case il::CmpOp::Ult => return encode::bltu(rs1, rs2, offset),
573
    }
574
}
575
576
/// Encode an inverted conditional branch instruction.
577
fn encodeInvertedBranch(op: il::CmpOp, rs1: gen::Reg, rs2: gen::Reg, offset: i32) -> u32 {
578
    match op {
579
        case il::CmpOp::Eq => return encode::bne(rs1, rs2, offset),
580
        case il::CmpOp::Ne => return encode::beq(rs1, rs2, offset),
581
        case il::CmpOp::Slt => return encode::bge(rs1, rs2, offset),
582
        case il::CmpOp::Ult => return encode::bgeu(rs1, rs2, offset),
583
    }
584
}
585
586
/// Patch all pending function calls.
587
/// Called after all functions have been generated.
588
export fn patchCalls(e: &mut Emitter) {
589
    for i in 0..e.pendingCallsLen {
590
        let p = e.pendingCalls[i];
591
        let offset = branchOffsetToFunc(e, p.index, p.target);
592
        if offset > 0x7ffff7ff {
593
            set e.error = super::Error::Relocation;
594
            return;
595
        }
596
        let s = splitImm(offset);
597
598
        // `AUIPC scratch, hi(offset)`.
599
        patch(e, p.index, encode::auipc(super::SCRATCH1, s.hi));
600
        // `JALR ra, scratch, lo(offset)`.
601
        patch(e, p.index + 1, encode::jalr(super::RA, super::SCRATCH1, s.lo));
602
    }
603
}
604
605
/// Patch all pending assembly jumps.
606
export fn patchJumps(e: &mut Emitter) {
607
    for i in 0..e.pendingJumpsLen {
608
        let p = e.pendingJumps[i];
609
        let offset = branchOffsetToFunc(e, p.index, p.target);
610
611
        if not encode::isJumpImm(offset) {
612
            set e.error = super::Error::Relocation;
613
            return;
614
        }
615
        patch(e, p.index, encode::jal(p.rd, offset));
616
    }
617
}
618
619
/// Patch all pending function and data address loads.
620
/// Called after all functions have been generated and data layout is known.
621
export fn patchAddrLoads(e: &mut Emitter, dataSymMap: &data::DataSymMap, codeBase: u64) throws (super::Error) {
622
    try check(e);
623
    for i in 0..e.pendingAddrLoadsLen {
624
        let p = e.pendingAddrLoads[i];
625
        if p.isData {
626
            let addr = data::lookupAddr(dataSymMap, p.target) else {
627
                throw super::Error::Symbol;
628
            };
629
            let offset = super::image::displacement(codeBase + p.index as u64 * 4, addr) else {
630
                throw super::Error::Relocation;
631
            };
632
            let s = splitImm(offset);
633
634
            patch(e, p.index, encode::auipc(p.rd, s.hi));
635
            patch(e, p.index + 1, encode::addi(p.rd, p.rd, s.lo));
636
637
            continue;
638
        }
639
640
        let offset = branchOffsetToFunc(e, p.index, p.target);
641
        if offset > 0x7ffff7ff {
642
            set e.error = super::Error::Relocation;
643
            return;
644
        }
645
        let s = splitImm(offset);
646
        // `AUIPC rd, hi(offset)`.
647
        patch(e, p.index, encode::auipc(p.rd, s.hi));
648
        // `ADDI rd, rd, lo(offset)`.
649
        patch(e, p.index + 1, encode::addi(p.rd, p.rd, s.lo));
650
    }
651
    try check(e);
652
}
653
654
/////////////////////////
655
// Immediate Handling  //
656
/////////////////////////
657
658
/// Split immediate into `hi` and `lo` bits.
659
export record SplitImm: Copy {
660
    /// Upper 20 bits.
661
    hi: i32,
662
    /// Lower 12 bits.
663
    lo: i32,
664
}
665
666
/// Split a 32-bit immediate for `AUIPC, ADDI` / `JALR` sequences.
667
/// Handles sign extension: if *lo* is negative, increment *hi*.
668
export fn splitImm(imm: i32) -> SplitImm {
669
    let lo = imm & 0xFFF;
670
    let mut hi = (imm >> 12) & 0xFFFFF;
671
    // If `lo`'s sign bit is set, it will be sign-extended to negative.
672
    // Compensate by incrementing `hi`.
673
    if (lo & 0x800) <> 0 {
674
        set hi += 1;
675
        return SplitImm { hi, lo: lo | 0xFFFFF000 as i32 };
676
    }
677
    return SplitImm { hi, lo };
678
}
679
680
/// Adjust a large offset by loading *hi* bits into [`super::ADDR_SCRATCH`].
681
/// Returns adjusted base register and remaining offset.
682
///
683
/// When the offset fits a 12-bit signed immediate, returns it unchanged.
684
/// Otherwise uses [`super::ADDR_SCRATCH`] for the LUI+ADD decomposition.
685
fn adjustOffset(e: &mut Emitter, base: gen::Reg, offset: i32) -> AdjustedOffset {
686
    if offset >= super::MIN_IMM and offset <= super::MAX_IMM {
687
        return AdjustedOffset { base, offset };
688
    }
689
    let s = splitImm(offset);
690
    emit(e, encode::lui(super::ADDR_SCRATCH, s.hi));
691
    emit(e, encode::add(super::ADDR_SCRATCH, super::ADDR_SCRATCH, base));
692
693
    return AdjustedOffset { base: super::ADDR_SCRATCH, offset: s.lo };
694
}
695
696
/// Load an immediate value into a register.
697
/// Handles the full range of 64-bit immediates.
698
/// For values fitting in 12 bits, uses a single `ADDI`.
699
/// For values fitting in 32 bits, uses `LUI` + `ADDIW`.
700
/// For wider values, loads upper and lower halves then combines with shift and add.
701
export fn loadImm(e: &mut Emitter, rd: gen::Reg, imm: i64) {
702
    let immMin = super::MIN_IMM as i64;
703
    let immMax = super::MAX_IMM as i64;
704
705
    if imm >= immMin and imm <= immMax {
706
        emit(e, encode::addi(rd, super::ZERO, imm as i32));
707
        return;
708
    }
709
    // Check if the value fits in 32 bits (sign-extended).
710
    let lo32 = imm as i32;
711
    if lo32 as i64 == imm {
712
        let s = splitImm(lo32);
713
        emit(e, encode::lui(rd, s.hi));
714
        if s.lo <> 0 {
715
            emit(e, encode::addiw(rd, rd, s.lo));
716
        }
717
        return;
718
    }
719
    // Full 64-bit immediate: use only rd, no scratch registers.
720
    // Load upper 32 bits first via the 32-bit path (LUI+ADDIW),
721
    // then shift and add lower bits in 11-bit groups to avoid
722
    // sign-extension issues with ADDI's 12-bit signed immediate.
723
    let hi32 = (imm >> 32) as i32;
724
    let lower = imm as i32;
725
726
    // Load upper 32 bits.
727
    loadImm(e, rd, hi32 as i64);
728
    // Shift left by 11, add bits [31:21].
729
    emit(e, encode::slli(rd, rd, 11));
730
    let chunkHi = (lower >> 21) & 0x7FF;
731
    if chunkHi <> 0 {
732
        emit(e, encode::addi(rd, rd, chunkHi));
733
    }
734
    // Shift left by 11, add bits [20:10].
735
    emit(e, encode::slli(rd, rd, 11));
736
    let chunkMid = (lower >> 10) & 0x7FF;
737
    if chunkMid <> 0 {
738
        emit(e, encode::addi(rd, rd, chunkMid));
739
    }
740
    // Shift left by 10, add bits [9:0].
741
    emit(e, encode::slli(rd, rd, 10));
742
    let chunkLo = lower & 0x3FF;
743
    if chunkLo <> 0 {
744
        emit(e, encode::addi(rd, rd, chunkLo));
745
    }
746
}
747
748
/// Emit add-immediate, handling large immediates.
749
export fn emitAddImm(e: &mut Emitter, rd: gen::Reg, rs: gen::Reg, imm: i32) {
750
    if imm >= super::MIN_IMM and imm <= super::MAX_IMM {
751
        emit(e, encode::addi(rd, rs, imm));
752
    } else {
753
        loadImm(e, super::SCRATCH1, imm as i64);
754
        emit(e, encode::add(rd, rs, super::SCRATCH1));
755
    }
756
}
757
758
////////////////////////
759
// Load/Store Helpers //
760
////////////////////////
761
762
/// Emit unsigned load with automatic offset adjustment.
763
export fn emitLoad(e: &mut Emitter, rd: gen::Reg, base: gen::Reg, offset: i32, typ: il::Type) {
764
    let adj = adjustOffset(e, base, offset);
765
    match typ {
766
        case il::Type::W8 => emit(e, encode::lbu(rd, adj.base, adj.offset)),
767
        case il::Type::W16 => emit(e, encode::lhu(rd, adj.base, adj.offset)),
768
        case il::Type::W32 => emit(e, encode::lwu(rd, adj.base, adj.offset)),
769
        case il::Type::W64 => emit(e, encode::ld(rd, adj.base, adj.offset)),
770
    }
771
}
772
773
/// Emit signed load with automatic offset adjustment.
774
export fn emitSload(e: &mut Emitter, rd: gen::Reg, base: gen::Reg, offset: i32, typ: il::Type) {
775
    let adj = adjustOffset(e, base, offset);
776
    match typ {
777
        case il::Type::W8 => emit(e, encode::lb(rd, adj.base, adj.offset)),
778
        case il::Type::W16 => emit(e, encode::lh(rd, adj.base, adj.offset)),
779
        case il::Type::W32 => emit(e, encode::lw(rd, adj.base, adj.offset)),
780
        case il::Type::W64 => emit(e, encode::ld(rd, adj.base, adj.offset)),
781
    }
782
}
783
784
/// Emit store with automatic offset adjustment.
785
export fn emitStore(e: &mut Emitter, rs: gen::Reg, base: gen::Reg, offset: i32, typ: il::Type) {
786
    let adj = adjustOffset(e, base, offset);
787
    match typ {
788
        case il::Type::W8 => emit(e, encode::sb(rs, adj.base, adj.offset)),
789
        case il::Type::W16 => emit(e, encode::sh(rs, adj.base, adj.offset)),
790
        case il::Type::W32 => emit(e, encode::sw(rs, adj.base, adj.offset)),
791
        case il::Type::W64 => emit(e, encode::sd(rs, adj.base, adj.offset)),
792
    }
793
}
794
795
/// Emit 64-bit load with automatic offset adjustment.
796
export fn emitLd(e: &mut Emitter, rd: gen::Reg, base: gen::Reg, offset: i32) {
797
    let adj = adjustOffset(e, base, offset);
798
    emit(e, encode::ld(rd, adj.base, adj.offset));
799
}
800
801
/// Emit 64-bit store with automatic offset adjustment.
802
export fn emitSd(e: &mut Emitter, rs: gen::Reg, base: gen::Reg, offset: i32) {
803
    let adj = adjustOffset(e, base, offset);
804
    emit(e, encode::sd(rs, adj.base, adj.offset));
805
}
806
807
/// Emit 32-bit load with automatic offset adjustment.
808
export fn emitLw(e: &mut Emitter, rd: gen::Reg, base: gen::Reg, offset: i32) {
809
    let adj = adjustOffset(e, base, offset);
810
    emit(e, encode::lw(rd, adj.base, adj.offset));
811
}
812
813
/// Emit 32-bit store with automatic offset adjustment.
814
export fn emitSw(e: &mut Emitter, rs: gen::Reg, base: gen::Reg, offset: i32) {
815
    let adj = adjustOffset(e, base, offset);
816
    emit(e, encode::sw(rs, adj.base, adj.offset));
817
}
818
819
/// Emit 8-bit load with automatic offset adjustment.
820
export fn emitLb(e: &mut Emitter, rd: gen::Reg, base: gen::Reg, offset: i32) {
821
    let adj = adjustOffset(e, base, offset);
822
    emit(e, encode::lb(rd, adj.base, adj.offset));
823
}
824
825
/// Emit 8-bit store with automatic offset adjustment.
826
export fn emitSb(e: &mut Emitter, rs: gen::Reg, base: gen::Reg, offset: i32) {
827
    let adj = adjustOffset(e, base, offset);
828
    emit(e, encode::sb(rs, adj.base, adj.offset));
829
}
830
831
//////////////////////////
832
// Prologue / Epilogue  //
833
//////////////////////////
834
835
/// Emit function prologue.
836
/// Allocate the frame and save registers. Save FP only for dynamic frames.
837
export fn emitPrologue(e: &mut Emitter, frame: &Frame) {
838
    // Fast path: leaf function with no locals.
839
    if frame.totalSize == 0 {
840
        return;
841
    }
842
    let totalSize = frame.totalSize;
843
844
    // Allocate stack frame.
845
    let negFrame = 0 - totalSize;
846
    if negFrame >= super::MIN_IMM {
847
        emit(e, encode::addi(super::SP, super::SP, negFrame));
848
    } else {
849
        loadImm(e, super::SCRATCH1, totalSize as i64);
850
        emit(e, encode::sub(super::SP, super::SP, super::SCRATCH1));
851
    }
852
    // Save return address.
853
    if not frame.isLeaf {
854
        emitSd(e, super::RA, super::SP, totalSize - super::DWORD_SIZE);
855
    }
856
    // Save and set FP only when dynamic allocations can move SP.
857
    if frame.isDynamic {
858
        emitSd(e, super::FP, super::SP, totalSize - super::DWORD_SIZE * 2);
859
        emitAddImm(e, super::FP, super::SP, totalSize);
860
    }
861
    // Save callee-saved registers.
862
    for i in 0..frame.savedRegsLen {
863
        let sr = frame.savedRegs[i];
864
        emitSd(e, sr.reg, super::SP, sr.offset);
865
    }
866
}
867
868
/// Emit a return: jump to epilogue, or emit `ret` directly for leaf functions.
869
export fn emitReturn(e: &mut Emitter, frame: &Frame) {
870
    if frame.totalSize == 0 {
871
        // Leaf function: no frame to tear down, emit ret directly.
872
        emit(e, encode::ret());
873
        return;
874
    }
875
    recordBranch(e, frame.epilogueBlock, BranchKind::Jump);
876
}
877
878
/// Emit function epilogue.
879
/// Restore saved registers and release the frame. Restore FP only for dynamic frames.
880
export fn emitEpilogue(e: &mut Emitter, frame: &Frame) {
881
    // Record epilogue block address for return jumps.
882
    recordBlock(e, frame.epilogueBlock);
883
884
    // Fast path: leaf function with no locals.
885
    if frame.totalSize == 0 {
886
        emit(e, encode::ret());
887
        return;
888
    }
889
    let totalSize = frame.totalSize;
890
891
    // Restore SP to post-prologue value. Only needed when dynamic stack
892
    // allocation may have moved SP.
893
    if frame.isDynamic {
894
        emitAddImm(e, super::SP, super::FP, 0 - totalSize);
895
    }
896
    // Restore callee-saved registers.
897
    for i in 0..frame.savedRegsLen {
898
        let sr = frame.savedRegs[i];
899
        emitLd(e, sr.reg, super::SP, sr.offset);
900
    }
901
    // Restore FP only if the prologue saved and changed it.
902
    if frame.isDynamic {
903
        emitLd(e, super::FP, super::SP, totalSize - super::DWORD_SIZE * 2);
904
    }
905
    // Restore return address.
906
    if not frame.isLeaf {
907
        emitLd(e, super::RA, super::SP, totalSize - super::DWORD_SIZE);
908
    }
909
    // Deallocate stack frame.
910
    emitAddImm(e, super::SP, super::SP, totalSize);
911
    emit(e, encode::ret());
912
}
913
914
//////////////////
915
// Code Access  //
916
//////////////////
917
918
/// Get emitted code as a slice.
919
export fn getCode 'code (e: &'code Emitter) -> &'code [u32] {
920
    return &e.code[..e.codeLen];
921
}
922
923
/// Record a debug entry mapping the current PC to a source location.
924
/// Deduplicates consecutive entries with the same location.
925
export fn recordSrcLoc(e: &mut Emitter, loc: il::SrcLoc) {
926
    if e.error <> nil {
927
        return;
928
    }
929
    let pc = e.codeLen * super::INSTR_SIZE as u32;
930
931
    // Skip if this is the same location as the previous entry.
932
    if e.debugEntriesLen > 0 {
933
        let prev = &e.debugEntries[e.debugEntriesLen - 1];
934
        if prev.offset == loc.offset and prev.moduleId == loc.moduleId {
935
            return;
936
        }
937
    }
938
    if e.debugEntriesLen == e.debugEntries.len {
939
        set e.error = super::Error::Capacity;
940
        return;
941
    }
942
    set e.debugEntries[e.debugEntriesLen] = types::DebugEntry {
943
        pc,
944
        moduleId: loc.moduleId,
945
        offset: loc.offset,
946
    };
947
    set e.debugEntriesLen += 1;
948
}
949
950
/// Get debug entries as a slice.
951
export fn getDebugEntries 'code (e: &'code Emitter) -> &'code [types::DebugEntry] {
952
    return &e.debugEntries[..e.debugEntriesLen];
953
}
954
955
/// Return the first emission failure before any generated output is published.
956
export fn check(e: &Emitter) throws (super::Error) {
957
    if let error = e.error {
958
        throw error;
959
    }
960
}