lib/std/lang/gen/regalloc/spill.rad 10.9 KiB raw
1
//! Spilling pass - determines which SSA values need stack slots.
2
//!
3
//! This pass analyzes register pressure and determines which values must be
4
//! spilled to memory. It does not modify the IL.
5
//!
6
//! The instruction selector uses this information to emit load/store
7
//! instructions for spilled values.
8
//!
9
//! # Algorithm
10
//!
11
//! 1. Cost calculation: for each value, compute spill cost:
12
//!
13
//!        cost = (defs + uses) * 2^min(loopDepth, 10)
14
//!
15
//!    Values used in inner loops are expensive to spill.
16
//!
17
//! 2. Pressure analysis: walk instructions backwards, tracking live set.
18
//!
19
//!    When `|live| > numRegs`:
20
//!
21
//!    * Sort live values by cost (ascending).
22
//!    * Mark lowest-cost values as spilled until pressure is acceptable.
23
//!
24
//! 3. Slot assignment: assign stack offsets to spilled values.
25
//!
26
//! # Output
27
//!
28
//! Spill info containing:
29
//!
30
//! * Spill slot assignments: SSA reg -> stack offset, `-1` if not spilled.
31
//! * Total frame size needed for spills.
32
33
use std::lang::il;
34
use std::lang::alloc;
35
use std::lang::gen::bitset;
36
use std::lang::gen::regalloc::liveness;
37
38
/// Maximum number of candidates for spill sorting.
39
constant MAX_CANDIDATES: u32 = 256;
40
/// Maximum loop depth for cost weighting (2^10 = 1024).
41
constant MAX_LOOP_WEIGHT: u32 = 10;
42
43
/// Spill cost for a single SSA register.
44
record SpillCost: Copy {
45
    /// Number of definitions (weighted by loop depth).
46
    defs: u32,
47
    /// Number of uses (weighted by loop depth).
48
    uses: u32,
49
}
50
51
/// Spill decision for a function.
52
export record SpillInfo: 'scratch + Copy {
53
    /// SSA register mapped to stack slot offset. `-1` means not spilled.
54
    slots: &'scratch [i32],
55
    /// Total spill frame size needed in bytes.
56
    frameSize: i32,
57
    /// Values that must be allocated in callee-saved registers.
58
    calleeClass: &'scratch [u32],
59
    /// Maximum SSA register number.
60
    maxReg: u32,
61
}
62
63
/// Candidate buffer for spill decisions.
64
record Candidates: Copy {
65
    entries: [CostEntry; MAX_CANDIDATES],
66
    n: u32,
67
}
68
69
/// Entry for cost sorting.
70
record CostEntry: Copy {
71
    reg: u32,
72
    cost: u32,
73
}
74
75
/// Analyze a function and determine which values need spill slots.
76
export unsafe fn analyze 'scratch (
77
    func: &il::Fn,
78
    live: &liveness::LiveInfo 'scratch,
79
    numRegs: u32,
80
    numCalleeSaved: u32,
81
    slotSize: u32,
82
    storage: &Session 'scratch
83
) -> SpillInfo 'scratch throws (alloc::AllocError) {
84
    let maxReg = live.maxReg;
85
    if maxReg == 0 {
86
        let calleeClass = try bitset::allocate(storage, 0);
87
        return SpillInfo 'scratch {
88
            slots: try storage.fill(-1 as i32, 0),
89
            frameSize: 0,
90
            calleeClass: &calleeClass[..],
91
            maxReg: 0,
92
        };
93
    }
94
    // Allocate spill slots array.
95
    let slots = try storage.fill(-1 as i32, maxReg);
96
    // Allocate cost array.
97
    let costs = try storage.fill(SpillCost { defs: 0, uses: 0 }, maxReg);
98
    // Phase 1: Calculate spill costs.
99
    fillCosts(func, costs);
100
101
    // Phase 2: Find values that exceed register pressure.
102
    let spilled = try bitset::allocate(storage, maxReg);
103
    let calleeClass = try bitset::allocate(storage, maxReg);
104
    let scratch = try bitset::allocate(storage, maxReg);
105
106
    for b in 0..func.blocks.len {
107
        let block = &func.blocks[b];
108
109
        // Start with live-out set.
110
        bitset::copy(scratch, liveness::liveOutRow(live, b));
111
112
        // Walk instructions backwards.
113
        let mut i = block.instrs.len;
114
        while i > 0 {
115
            set i -= 1;
116
            let instr = block.instrs[i];
117
118
            // Limit register pressure before processing this instruction.
119
            try limitPressure(scratch, spilled, costs, numRegs);
120
121
            // Enforce cross-call pressure at call sites.
122
            if il::isCall(instr) {
123
                try limitCrossCallPressure(
124
                    scratch, spilled, costs,
125
                    calleeClass, numCalleeSaved, il::instrDst(instr)
126
                );
127
            }
128
            // Remove definition from live set.
129
            if let dst = il::instrDst(instr) {
130
                bitset::clear(scratch, dst.n);
131
            }
132
            // Add uses to live set.
133
            let mut registers = il::registers(&instr);
134
            while let reg = il::nextReg(&mut registers, &instr) {
135
                bitset::put(scratch, reg.n);
136
            }
137
        }
138
        // Also limit pressure at block entry.
139
        try limitPressure(scratch, spilled, costs, numRegs);
140
    }
141
142
    // Phase 3: Enforce global callee-class limit.
143
    // The per-call-site limit may leave the callee-class set larger than
144
    // `numCalleeSaved` when different call sites keep different subsets.
145
    // Spill excess values to guarantee the assignment phase always finds
146
    // a callee-saved register for cross-call values.
147
    let calleeCount = bitset::count(calleeClass);
148
    if calleeCount > numCalleeSaved {
149
        let mut it = bitset::iter(calleeClass);
150
        for _ in 0..(calleeCount - numCalleeSaved) {
151
            if let reg = bitset::iterNext(&mut it, calleeClass) {
152
                bitset::clear(calleeClass, reg);
153
                bitset::put(spilled, reg);
154
            } else {
155
                panic "spill: count > 0 but no set bits found";
156
            }
157
        }
158
    }
159
160
    // Phase 4: Assign stack slots to spilled values.
161
    let mut frameSize: i32 = 0;
162
    let mut it = bitset::iter(spilled);
163
164
    while let n = bitset::iterNext(&mut it, spilled) {
165
        set slots[n] = frameSize;
166
        set frameSize += slotSize as i32;
167
    }
168
    return SpillInfo 'scratch { slots: &slots[..], frameSize, calleeClass: &calleeClass[..], maxReg };
169
}
170
171
/// Calculate spill costs for all registers, weighted by loop depth.
172
unsafe fn fillCosts(func: &il::Fn, costs: &mut [SpillCost]) {
173
    for b in 0..func.blocks.len {
174
        let block = &func.blocks[b];
175
176
        // Exponential weight for loop depth, capped to avoid overflow.
177
        let depth = MAX_LOOP_WEIGHT if block.loopDepth > MAX_LOOP_WEIGHT else block.loopDepth;
178
        let weight: u32 = 1 << depth;
179
180
        // Count block parameter definitions.
181
        for p in block.params {
182
            if p.value.n < costs.len {
183
                set costs[p.value.n].defs = costs[p.value.n].defs + weight;
184
            }
185
        }
186
        // Count instruction defs and uses.
187
        for instr in block.instrs {
188
189
            // Count definition.
190
            if let dst = il::instrDst(instr) {
191
                if dst.n < costs.len {
192
                    set costs[dst.n].defs = costs[dst.n].defs + weight;
193
                }
194
            }
195
            // Count uses.
196
            let mut registers = il::registers(&instr);
197
            while let reg = il::nextReg(&mut registers, &instr) {
198
                countRegUse(reg, &mut costs[..], weight);
199
            }
200
        }
201
    }
202
}
203
204
/// Sort candidates by cost (ascending) using insertion sort, then spill
205
/// the cheapest `excess` values: set in `spilled`, clear in `source`.
206
fn spillCheapest(
207
    c: &mut Candidates,
208
    excess: u32,
209
    source: &mut [u32],
210
    spilled: &mut [u32]
211
) {
212
    // Insertion sort ascending by cost.
213
    for i in 1..c.n {
214
        let key = c.entries[i];
215
        let mut j: u32 = i;
216
        while j > 0 and c.entries[j - 1].cost > key.cost {
217
            set c.entries[j] = c.entries[j - 1];
218
            set j -= 1;
219
        }
220
        set c.entries[j] = key;
221
    }
222
    let toSpill = c.n if excess > c.n else excess;
223
    for i in 0..toSpill {
224
        bitset::put(spilled, c.entries[i].reg);
225
        bitset::clear(source, c.entries[i].reg);
226
    }
227
}
228
229
/// Collect all values from a bitset into a candidates buffer with their costs.
230
unsafe fn collectCandidates(bs: &[u32], costs: &[SpillCost]) -> Candidates throws (alloc::AllocError) {
231
    let mut c = Candidates { entries: undefined, n: 0 };
232
    let mut it = bitset::iter(bs);
233
    while let reg = bitset::iterNext(&mut it, bs) {
234
        if c.n == MAX_CANDIDATES {
235
            throw alloc::AllocError::OutOfMemory;
236
        }
237
        if reg < costs.len {
238
            set c.entries[c.n] = CostEntry { reg, cost: costs[reg].defs + costs[reg].uses };
239
            set c.n += 1;
240
        }
241
    }
242
    return c;
243
}
244
245
/// Limit register pressure by marking low-cost values as spilled.
246
unsafe fn limitPressure(
247
    live: &mut [u32],
248
    spilled: &mut [u32],
249
    costs: &[SpillCost],
250
    numRegs: u32
251
) throws (alloc::AllocError) {
252
    let liveCount = bitset::count(live);
253
    if liveCount <= numRegs {
254
        return;
255
    }
256
    let mut c = try collectCandidates(live, costs);
257
    spillCheapest(&mut c, liveCount - numRegs, live, spilled);
258
}
259
260
/// Check whether an SSA register is the destination of a call instruction.
261
fn isCallDst(callDst: ?il::Reg, n: u32) -> bool {
262
    if let d = callDst {
263
        return d.n == n;
264
    }
265
    return false;
266
}
267
268
/// Limit cross-call pressure by spilling values that exceed callee-saved capacity.
269
///
270
/// At a call site, every live value must survive the call in a callee-saved
271
/// register. If the count exceeds `numCalleeSaved`, spill the cheapest
272
/// crossing values.
273
unsafe fn limitCrossCallPressure(
274
    live: &mut [u32],
275
    spilled: &mut [u32],
276
    costs: &[SpillCost],
277
    calleeClass: &mut [u32],
278
    numCalleeSaved: u32,
279
    callDst: ?il::Reg
280
) throws (alloc::AllocError) {
281
    // Collect crossing candidates: live values excluding the call destination.
282
    let mut candidates: [CostEntry; MAX_CANDIDATES] = undefined;
283
    let mut numCandidates: u32 = 0;
284
    let mut it = bitset::iter(live);
285
    while let n = bitset::iterNext(&mut it, live) {
286
        if not isCallDst(callDst, n) and n < costs.len {
287
            if numCandidates == MAX_CANDIDATES {
288
                throw alloc::AllocError::OutOfMemory;
289
            }
290
            set candidates[numCandidates] = CostEntry {
291
                reg: n,
292
                cost: costs[n].defs + costs[n].uses,
293
            };
294
            set numCandidates += 1;
295
        }
296
    }
297
    // Spill cheapest candidates if crossing count exceeds callee-saved capacity.
298
    if numCandidates > numCalleeSaved {
299
        let mut c = Candidates { entries: candidates, n: numCandidates };
300
        spillCheapest(&mut c, numCandidates - numCalleeSaved, live, spilled);
301
    }
302
    // Mark crossing values that remain after spilling as callee-saved class.
303
    let mut it2 = bitset::iter(live);
304
    while let n = bitset::iterNext(&mut it2, live) {
305
        if not isCallDst(callDst, n) {
306
            bitset::put(calleeClass, n);
307
        }
308
    }
309
}
310
311
/// Add the block weight to the register use count.
312
fn countRegUse(reg: il::Reg, costs: &mut [SpillCost], weight: u32) {
313
    assert reg.n < costs.len, "countRegUse: register out of bounds";
314
    set costs[reg.n].uses += weight;
315
}
316
317
/// Check if a register is spilled.
318
export fn isSpilled 'scratch (info: &SpillInfo 'scratch, reg: il::Reg) -> bool {
319
    if reg.n >= info.maxReg {
320
        return false;
321
    }
322
    return info.slots[reg.n] >= 0;
323
}
324
325
/// Get spill slot offset for a register, or `nil` if not spilled.
326
export fn spillSlot 'scratch (info: &SpillInfo 'scratch, reg: il::Reg) -> ?i32 {
327
    if isSpilled(info, reg) {
328
        return info.slots[reg.n];
329
    }
330
    return nil;
331
}