lib/std/lang/il/images/native.rad 18.5 KiB raw
1
//! Shared-backend image code, read-only initializers, and trusted linker glue.
2
3
use std::collections::dict;
4
use std::lang::alloc;
5
use std::lang::strings;
6
use std::lang::il;
7
use std::lang::gen::data;
8
use std::lang::gen::bitset;
9
use std::lang::gen::regalloc;
10
use std::arch::rv64;
11
use std::arch::rv64::emit;
12
use std::arch::rv64::encode;
13
use super::graph;
14
15
/// One repeated private pointer initializer, expressed in state-relative bytes.
16
record Fixup: Copy {
17
    /// First pointer slot's private byte offset.
18
    offset: u32,
19
    /// Target declaration's private byte offset.
20
    target: u32,
21
    /// Number of consecutive eight-byte pointer slots.
22
    count: u32,
23
}
24
25
/// Persistent read-only definitions and scratch relocation runs.
26
export record Initializer: Copy {
27
    /// Shared initializer and immutable data definitions.
28
    items: *[il::Data],
29
    /// Private pointer runs applied to each new instance.
30
    fixups: *[Fixup],
31
}
32
33
/// Retain initializer payloads and keep ordinary RO/function relocations intact.
34
fn value(value: il::DataValue, plan: *graph::Plan, names: *graph::Names, arena: *mut alloc::Arena) -> il::DataValue throws (il::binary::Error) {
35
    let mut item = value.item;
36
    match item {
37
        case il::DataItem::Sym(symbol) => {
38
            let index = try graph::index(&plan.dataIndex, symbol);
39
            set item = il::DataItem::Sym(names.data[index]) if plan.items[index].readOnly
40
                else il::DataItem::Val { typ: il::Type::W64, val: plan.offsets[index] as i64 };
41
        }
42
        case il::DataItem::Fn(symbol) => set item = il::DataItem::Fn(names.functions[try graph::index(&plan.fnIndex, symbol)]),
43
        case il::DataItem::Str(bytes) => set item = il::DataItem::Str(try graph::copy(arena, bytes)),
44
        else => {},
45
    }
46
    return il::DataValue { item, count: 0 if graph::width(item) == 0 else value.count };
47
}
48
49
/// Flatten private declarations into one RO initializer using their native layout.
50
export fn initializer(plan: *graph::Plan, names: *graph::Names, stateName: *[u8], arena: *mut alloc::Arena, scratch: *mut alloc::Arena) -> Initializer throws (il::binary::Error) {
51
    let mut shared: u32 = 0;
52
    let mut count: u64 = plan.order.len as u64 * 2;
53
    let mut fixupCount: u64 = 0;
54
    for item in plan.items {
55
        if item.readOnly { set shared += 1; continue; }
56
        if item.isZeroInit { continue; }
57
        set count += item.values.len as u64;
58
        for v in item.values {
59
            if v.count == 0 { continue; }
60
            if let case il::DataItem::Sym(symbol) = v.item {
61
                if not plan.items[try graph::index(&plan.dataIndex, symbol)].readOnly { set fixupCount += 1; }
62
            }
63
        }
64
    }
65
    if count > 0x7FFFFFFF or fixupCount > 0x7FFFFFFF { throw il::binary::error(0, "too many image initializer values"); }
66
    let values = try graph::storage(arena, @sizeOf(il::DataValue), @alignOf(il::DataValue), count as u32) as *mut [il::DataValue];
67
    let fixups = try graph::storage(scratch, @sizeOf(Fixup), @alignOf(Fixup), fixupCount as u32) as *mut [Fixup];
68
    let items = try graph::storage(arena, @sizeOf(il::Data), @alignOf(il::Data), shared + 1) as *mut [il::Data];
69
    let mut valueCount: u32 = 0;
70
    let mut fixupIndex: u32 = 0;
71
    let mut offset: u32 = 0;
72
    for symbol in plan.order {
73
        let item = plan.items[try graph::index(&plan.dataIndex, symbol.name)];
74
        if symbol.addr > offset {
75
            set values[valueCount] = il::DataValue { item: il::DataItem::Undef, count: symbol.addr - offset };
76
            set valueCount += 1;
77
            set offset = symbol.addr;
78
        }
79
        if item.isZeroInit {
80
            set values[valueCount] = il::DataValue { item: il::DataItem::Undef, count: item.size };
81
            set valueCount += 1;
82
            set offset += item.size;
83
            continue;
84
        }
85
        for v in item.values {
86
            if v.count > 0 {
87
                if let case il::DataItem::Sym(target) = v.item {
88
                    let targetIndex = try graph::index(&plan.dataIndex, target);
89
                    if not plan.items[targetIndex].readOnly {
90
                        set fixups[fixupIndex] = Fixup { offset, target: plan.offsets[targetIndex], count: v.count };
91
                        set fixupIndex += 1;
92
                    }
93
                }
94
            }
95
            set values[valueCount] = try value(v, plan, names, arena);
96
            set valueCount += 1;
97
            set offset += graph::width(v.item) * v.count;
98
        }
99
    }
100
    set items[0] = il::Data { name: stateName, size: plan.size, alignment: plan.alignment,
101
        readOnly: true, isZeroInit: false, values: &values[..valueCount] };
102
    let mut itemIndex: u32 = 1;
103
    for item, i in plan.items {
104
        if not item.readOnly { continue; }
105
        let count = 1 if item.isZeroInit else item.values.len;
106
        let copied = try graph::storage(arena, @sizeOf(il::DataValue), @alignOf(il::DataValue), count) as *mut [il::DataValue];
107
        if item.isZeroInit { set copied[0] = il::DataValue { item: il::DataItem::Undef, count: item.size }; }
108
        else { for v, j in item.values { set copied[j] = try value(v, plan, names, arena); } }
109
        set items[itemIndex] = il::Data { name: names.data[i], size: item.size, alignment: item.alignment,
110
            readOnly: true, isZeroInit: false, values: copied };
111
        set itemIndex += 1;
112
    }
113
    return Initializer { items, fixups };
114
}
115
116
/// Reserve the caller's growable list using checked allocation before any append.
117
export fn appendData(target: *mut *mut [il::Data], items: *[il::Data], arena: *mut alloc::Arena) throws (il::binary::Error) {
118
    let total = (*target).len as u64 + items.len as u64;
119
    if total > data::MAX_DATA_SYMS as u64 { throw il::binary::error(0, "native data symbol capacity exceeded"); }
120
    let mut slice = *target;
121
    if total > slice.cap as u64 {
122
        let mut capacity: u32 = 16;
123
        while capacity < total as u32 { set capacity *= 2; }
124
        let grown = try graph::storage(arena, @sizeOf(il::Data), @alignOf(il::Data), capacity) as *mut [il::Data];
125
        for item, i in slice { set grown[i] = item; }
126
        set slice = @sliceOf(grown.ptr, slice.len, capacity);
127
    }
128
    let allocator = alloc::arenaAllocator(arena);
129
    for item in items { slice.append(item, allocator); }
130
    set *target = slice;
131
}
132
133
/// Check hard backend limits and a conservative expansion bound for one function.
134
fn bounds(e: *emit::Emitter, function: *il::Fn) -> u32 throws (il::binary::Error) {
135
    if function.params.len > rv64::ARG_REGS.len { throw il::binary::error(0, "image function has too many arguments"); }
136
    let mut blocks = function.blocks.len as u64 + 1;
137
    let mut words: u64 = 256;
138
    let mut reserve: u64 = 0;
139
    let mut operands: u64 = 0;
140
    let mut calls: u64 = 0;
141
    let mut branches: u64 = 0;
142
    for block in function.blocks {
143
        if block.params.len > 16 { throw il::binary::error(0, "image block has too many parameters"); }
144
        for item in block.instrs {
145
            set words += 64;
146
            set operands += 5;
147
            set branches += 2;
148
            match item {
149
                case il::Instr::Call { func, args, .. } => {
150
                    match func {
151
                        case il::Val::FnAddr(_), il::Val::Reg(_) => {},
152
                        else => throw il::binary::error(0, "invalid image native call target"),
153
                    }
154
                    if args.len > rv64::ARG_REGS.len { throw il::binary::error(0, "image call has too many arguments"); }
155
                    set words += args.len as u64 * 32;
156
                    set operands += args.len as u64;
157
                    set calls += 1;
158
                }
159
                case il::Instr::Jmp { args, .. } => {
160
                    set words += args.len as u64 * 32;
161
                    set operands += args.len as u64;
162
                }
163
                case il::Instr::Br { thenArgs, elseArgs, .. } => {
164
                    if thenArgs.len > 0 and elseArgs.len > 0 { throw il::binary::error(0, "image branch requires split argument edges"); }
165
                    set words += (thenArgs.len as u64 + elseArgs.len as u64) * 32;
166
                    set operands += thenArgs.len as u64 + elseArgs.len as u64;
167
                }
168
                case il::Instr::Switch { defaultArgs, cases, .. } => {
169
                    set words += defaultArgs.len as u64 * 32;
170
                    set operands += defaultArgs.len as u64;
171
                    for c in cases {
172
                        set words += 32 + c.args.len as u64 * 32;
173
                        set operands += c.args.len as u64;
174
                        set branches += 2;
175
                        if c.args.len > 0 { set blocks += 1; }
176
                    }
177
                }
178
                case il::Instr::Reserve { size, alignment, .. } => {
179
                    if alignment == 0 or alignment & (alignment - 1) <> 0 or alignment > graph::MAX_DATA {
180
                        throw il::binary::error(0, "image stack alignment exceeds capacity");
181
                    }
182
                    match size {
183
                        case il::Val::Imm(bytes) => {
184
                            if bytes < 0 or bytes > graph::MAX_DATA as i64 { throw il::binary::error(0, "image stack reservation exceeds capacity"); }
185
                            let a = alignment as u64;
186
                            set reserve = ((reserve + a - 1) & ~(a - 1)) + bytes as u64;
187
                            if reserve > graph::MAX_DATA as u64 { throw il::binary::error(0, "image stack frame exceeds capacity"); }
188
                        }
189
                        case il::Val::Reg(_) => {
190
                            if alignment > 2048 { throw il::binary::error(0, "dynamic image stack alignment exceeds backend capacity"); }
191
                        }
192
                        else => throw il::binary::error(0, "invalid image stack reservation"),
193
                    }
194
                }
195
                case il::Instr::Blit { size, alignment, .. } => {
196
                    let case il::Val::Imm(bytes) = size else { throw il::binary::error(0, "image blit requires an immediate size"); };
197
                    if bytes < 0 or bytes > graph::MAX_DATA as i64 or alignment == 0 or alignment & (alignment - 1) <> 0 {
198
                        throw il::binary::error(0, "image blit exceeds backend capacity");
199
                    }
200
                    // A spilled base can force an unrolled copy with adjusted addresses.
201
                    let width = il::typeSize(il::copyType(bytes as u32, alignment)) if bytes > 0 else 1;
202
                    set words += (bytes as u64 / width as u64 + 8) * 12;
203
                }
204
                else => {},
205
            }
206
        }
207
    }
208
    if blocks > e.labels.blockOffsets.len as u64 { throw il::binary::error(0, "image function exceeds block capacity"); }
209
    // Local jumps use one JAL slot and must stay strictly inside its positive range.
210
    if words >= 0x100000 / rv64::INSTR_SIZE as u64 { throw il::binary::error(0, "image function exceeds local branch reach"); }
211
    if words > (e.code.len - e.codeLen) as u64 { throw il::binary::error(0, "native code capacity exceeded"); }
212
    if e.labels.funcs.count >= e.labels.funcs.entries.len / 2 { throw il::binary::error(0, "native function symbol capacity exceeded"); }
213
    if dict::get(&e.labels.funcs, function.name) <> nil { throw il::binary::error(0, "duplicate native image function"); }
214
    if e.funcs.len >= e.funcs.cap or operands + e.pendingAddrLoads.len as u64 > e.pendingAddrLoads.cap as u64
215
        or calls + e.pendingCalls.len as u64 > e.pendingCalls.cap as u64
216
        or branches + e.pendingBranches.len as u64 > e.pendingBranches.cap as u64 {
217
        throw il::binary::error(0, "native image relocation capacity exceeded");
218
    }
219
    return reserve as u32;
220
}
221
222
/// Shared liveness state used to bound the spill-candidate buffer.
223
record Pressure: Copy {
224
    /// Registers live at the inspected instruction.
225
    live: bitset::Bitset,
226
    /// Number of live registers.
227
    count: u32,
228
}
229
230
/// Add one existing IL operand to the current live register set.
231
fn liveUse(reg: il::Reg, context: *mut opaque) {
232
    let pressure = context as *mut Pressure;
233
    if not bitset::contains(&pressure.live, reg.n) {
234
        bitset::put(&mut pressure.live, reg.n);
235
        set pressure.count += 1;
236
    }
237
}
238
239
/// Run the existing allocation stages, checking their fixed pressure capacity.
240
fn allocate(function: *il::Fn, arena: *mut alloc::Arena) -> regalloc::AllocResult throws (il::binary::Error) {
241
    let config = rv64::targetConfig();
242
    let live = try regalloc::liveness::analyze(function, arena)
243
        catch { throw il::binary::error(0, "image liveness arena exhausted"); };
244
    let saved = alloc::save(arena);
245
    let words = try graph::storage(arena, @sizeOf(u32), @alignOf(u32), bitset::wordsFor(live.maxReg)) as *mut [u32];
246
    let mut pressure = Pressure { live: bitset::init(words), count: 0 };
247
    for block, b in function.blocks {
248
        bitset::copy(&mut pressure.live, &live.liveOut[b]);
249
        set pressure.count = bitset::count(&pressure.live);
250
        if pressure.count > 256 { throw il::binary::error(0, "image exceeds live register capacity"); }
251
        let mut index = block.instrs.len;
252
        while index > 0 {
253
            set index -= 1;
254
            let item = block.instrs[index];
255
            if let dst = il::instrDst(item) {
256
                if bitset::contains(&pressure.live, dst.n) {
257
                    bitset::clear(&mut pressure.live, dst.n);
258
                    set pressure.count -= 1;
259
                }
260
            }
261
            il::forEachReg(item, liveUse, &mut pressure as *mut opaque);
262
            if pressure.count > 256 { throw il::binary::error(0, "image exceeds live register capacity"); }
263
        }
264
    }
265
    alloc::restore(arena, saved);
266
    let spill = try regalloc::spill::analyze(function, &live, config.allocatable.len, config.calleeSaved.len, config.slotSize, arena)
267
        catch { throw il::binary::error(0, "image spill arena exhausted"); };
268
    let assignment = try regalloc::assign::assign(function, &live, &spill, &config, arena)
269
        catch { throw il::binary::error(0, "image register assignment arena exhausted"); };
270
    return regalloc::AllocResult { assignments: assignment.assignments, spill, usedCalleeSaved: assignment.usedCalleeSaved };
271
}
272
273
/// Emit through the existing allocator and selector, propagating scratch exhaustion.
274
export fn function(generator: *mut rv64::Generator, function: *il::Fn, scratch: *mut alloc::Arena) throws (il::binary::Error) {
275
    let reserve = try bounds(&generator.e, function);
276
    let saved = alloc::save(scratch);
277
    let allocation = try allocate(function, scratch) catch error {
278
        alloc::restore(scratch, saved);
279
        throw error;
280
    };
281
    let frame = emit::computeFrame(allocation.spill.frameSize + reserve as i32, allocation.usedCalleeSaved,
282
        function.blocks.len, function.isLeaf, false);
283
    if frame.totalSize > 64 * 1024 {
284
        alloc::restore(scratch, saved);
285
        throw il::binary::error(0, "image function frame exceeds native stack");
286
    }
287
    rv64::isel::selectFn(&mut generator.e, &allocation, function);
288
    alloc::restore(scratch, saved);
289
}
290
291
/// Reserve a trusted leaf helper in the same global symbol and relocation tables.
292
fn helper(e: *mut emit::Emitter, name: *[u8], words: u32) throws (il::binary::Error) {
293
    if dict::get(&e.labels.funcs, name) <> nil { throw il::binary::error(0, "duplicate image helper symbol"); }
294
    if e.labels.funcs.count >= e.labels.funcs.entries.len / 2 { throw il::binary::error(0, "native function symbol capacity exceeded"); }
295
    if words > e.code.len - e.codeLen { throw il::binary::error(0, "native code capacity exceeded"); }
296
    if e.funcs.len >= e.funcs.cap or e.pendingCalls.len == e.pendingCalls.cap
297
        or e.pendingAddrLoads.len == e.pendingAddrLoads.cap {
298
        throw il::binary::error(0, "native image helper capacity exceeded");
299
    }
300
    emit::recordFunc(e, name);
301
    emit::recordFuncOffset(e, name);
302
}
303
304
/// Tail-call one explicitly bound primitive without limiting its physical address.
305
export fn trampoline(generator: *mut rv64::Generator, name: *[u8], address: u64) throws (il::binary::Error) {
306
    let e = &mut generator.e;
307
    try helper(e, name, 9);
308
    emit::loadImm(e, rv64::SCRATCH1, address as i64);
309
    emit::emit(e, encode::jalr(rv64::ZERO, rv64::SCRATCH1, 0));
310
}
311
312
/// Build trusted startup, address getters, and private-pointer relocation routines.
313
export fn helpers(generator: *mut rv64::Generator, id: u32, plan: *graph::Plan, names: *graph::Names,
314
    initial: *Initializer, pool: *mut strings::Pool, arena: *mut alloc::Arena) throws (il::binary::Error) {
315
    let stateName = initial.items[0].name;
316
    let entry = try graph::name("images::native_", id, "", pool, arena);
317
    let entryGetter = try graph::name("images::entry_", id, "", pool, arena);
318
    let initialGetter = try graph::name("images::initial_", id, "", pool, arena);
319
    let relocate = try graph::name("images::relocate_", id, "", pool, arena);
320
    let e = &mut generator.e;
321
    try helper(e, entry, 16);
322
    emit::emit(e, encode::mv(rv64::TP, rv64::A0));
323
    emit::emitLd(e, rv64::SP, rv64::A0, 72);
324
    emit::recordCall(e, names.functions[plan.entry]);
325
    if not plan.returnsStatus { emit::emit(e, encode::mv(rv64::A0, rv64::ZERO)); }
326
    else {
327
        emit::emit(e, encode::slli(rv64::A0, rv64::A0, 32));
328
        emit::emit(e, encode::srli(rv64::A0, rv64::A0, 32));
329
    }
330
    emit::loadImm(e, rv64::A7, 49);
331
    emit::emit(e, encode::ecall());
332
    emit::emit(e, encode::ebreak());
333
    try helper(e, entryGetter, 3);
334
    emit::recordAddrLoad(e, entry, rv64::A0);
335
    emit::emit(e, encode::ret());
336
    try helper(e, initialGetter, 3);
337
    emit::recordDataAddrLoad(e, stateName, rv64::A0);
338
    emit::emit(e, encode::ret());
339
    let words = initial.fixups.len as u64 * 32 + 1;
340
    if words > 0x7FFFFFFF { throw il::binary::error(0, "image relocation helper exceeds capacity"); }
341
    try helper(e, relocate, words as u32);
342
    for fixup in initial.fixups {
343
        emit::emitAddImm(e, rv64::T0, rv64::A0, fixup.offset as i32);
344
        emit::emitAddImm(e, rv64::T1, rv64::A0, fixup.target as i32);
345
        emit::loadImm(e, rv64::T2, fixup.count as i64);
346
        let start = e.codeLen;
347
        if fixup.offset % 8 == 0 { emit::emitSd(e, rv64::T1, rv64::T0, 0); }
348
        else {
349
            emit::emit(e, encode::mv(rv64::T3, rv64::T1));
350
            for byte in 0..8 {
351
                emit::emitSb(e, rv64::T3, rv64::T0, byte as i32);
352
                if byte < 7 { emit::emit(e, encode::srli(rv64::T3, rv64::T3, 8)); }
353
            }
354
        }
355
        if fixup.count > 1 {
356
            emit::emit(e, encode::addi(rv64::T0, rv64::T0, 8));
357
            emit::emit(e, encode::addi(rv64::T2, rv64::T2, -1));
358
            emit::emit(e, encode::bne(rv64::T2, rv64::ZERO, (start as i32 - e.codeLen as i32) * rv64::INSTR_SIZE));
359
        }
360
    }
361
    emit::emit(e, encode::ret());
362
}