emulator: Speed up execution

064c3426ad28d10dd695f0ca71c7d3385df15b89db49678cdfc043ef4dfaea8a
Every translated block returned to C after each branch, and each had
its own prologue and epilogue. Startup also cleared all of guest
memory, which touched every page of the configured memory size, 376
MiB for compiler runs, and dominated short runs: 96 small compiles
took 2.8 s with 24 in parallel.

Blocks now chain. A shared entry trampoline and exit stub replace the
per-block prologue and epilogue. Direct jumps are patched to the
target block once C has translated it, and indirect jumps go through
a direct-mapped jump cache filled by C. A budget register keeps
instruction counting exact: each block checks that budget remains,
then charges its instruction count, so -count-instructions and
-max-steps still see every guest instruction.

Guest memory is static storage and cpu_init runs once, before the
program is loaded, so the memory is already zero. cpu_init no longer
clears it; the same 96 compiles take 0.3 s.

Drop JIT state that nothing reads: jit_flush, which nothing calls,
the end PC, instruction count and code size of each block, and the
block statistics.

Assisted-by: Claude:sonnet-5.5
Assisted-by: Claude:opus-5.5
Alexis Sellier committed ago 1 parent d9bc0eae
emulator.c +20 -9
1203 1203
    cpu->ebreak      = false;
1204 1204
    cpu->modified    = ZERO;
1205 1205
    last_executed_pc = 0;
1206 1206
}
1207 1207
1208 -
/* Initialize CPU and memory to a clean state. */
1208 +
/* Initialize the CPU to a clean state.  Guest memory is not cleared: this runs
1209 +
 * once at startup, when the static memory is still zero, and clearing it would
1210 +
 * touch every page. */
1209 1211
static void cpu_init(struct cpu *cpu) {
1210 -
    memset(memory, 0, g_opts.memory_size);
1211 1212
    cpu->program     = (instr_t *)memory;
1212 1213
    cpu->programsize = 0;
1213 1214
    trace_reset();
1214 1215
    guest_fd_table_init();
1215 1216
    cpu_reset(cpu);
2575 2576
            use_jit = jit_init(&jit);
2576 2577
        }
2577 2578
2578 2579
        if (use_jit) {
2579 2580
            /* ---- JIT execution loop ---- */
2581 +
            u8 *patch = NULL; /* Direct jump to link to the next block. */
2582 +
2580 2583
            while (cpu.running && steps < max_steps) {
2581 2584
                struct jit_block *block = jit_get_block(
2582 2585
                    &jit, cpu.pc, memory, program_base, program_bytes
2583 2586
                );
2584 2587
                if (!block) {
2587 2590
                    while (cpu.running && steps++ < max_steps) {
2588 2591
                        cpu_execute(&cpu, display, true);
2589 2592
                    }
2590 2593
                    break;
2591 2594
                }
2592 -
                u32 next_pc = 0;
2595 +
                if (patch)
2596 +
                    jit_patch_chain(patch, block);
2597 +
                jit_cache_jump(&jit, block);
2598 +
2599 +
                /* Generated code charges each block's instructions against
2600 +
                 * the budget, so `steps` advances by what it consumed. */
2601 +
                u64 left   = max_steps - steps;
2602 +
                i64 budget = left > INT64_MAX ? INT64_MAX : (i64)left;
2593 2603
                int exit_reason =
2594 -
                    jit_exec_block(block, cpu.regs, memory, &next_pc);
2595 -
                steps += block->insn_count;
2596 -
                jit.blocks_executed++;
2597 -
                jit.insns_executed += block->insn_count;
2604 +
                    jit_run(&jit, block, cpu.regs, memory, budget);
2605 +
                steps  += (u64)budget - (u64)jit.io.budget;
2606 +
                cpu.pc  = jit.io.pc;
2607 +
                patch   = NULL;
2598 2608
2599 2609
                switch (exit_reason) {
2600 2610
                case JIT_EXIT_BRANCH:
2611 +
                    break;
2612 +
2601 2613
                case JIT_EXIT_CHAIN:
2602 -
                    cpu.pc = next_pc;
2614 +
                    patch = jit.io.patch;
2603 2615
                    break;
2604 2616
2605 2617
                case JIT_EXIT_RET:
2606 2618
                    cpu.running = false;
2607 2619
                    break;
2608 2620
2609 2621
                default:
2610 2622
                    /* ECALL, EBREAK, FAULT -- interpreter handles it.
2611 2623
                     * The instruction is already counted above,
2612 2624
                     * so don't increment steps again. */
2613 -
                    cpu.pc = next_pc;
2614 2625
                    cpu_execute(&cpu, display, true);
2615 2626
                    break;
2616 2627
                }
2617 2628
            }
2618 2629
            jit_destroy(&jit);
jit.c +232 -94
1 1
/*
2 2
 * JIT compiler: RV64I basic-block -> x86-64 native code.
3 3
 *
4 4
 * Strategy:
5 -
 *   - Guest registers live in a u64[32] array pointed to by the first arg
6 -
 *     (rdi on entry).  Memory base is in rsi.  The third arg (rdx) is a
7 -
 *     pointer where we write the next guest PC on exit.
5 +
 *   - One entry trampoline and one exit stub are generated at init.  The
6 +
 *     trampoline saves the callee-saved registers, pins the registers below
7 +
 *     and jumps into a block.  The exit stub stores the remaining budget,
8 +
 *     restores the registers and returns the exit reason in eax.  Blocks
9 +
 *     have no prologue or epilogue and never call.
8 10
 *   - During block execution we pin:
9 11
 *       r12 = &regs[0]   (guest register file)
10 12
 *       r13 = memory base
11 -
 *       r14 = pc_out pointer
13 +
 *       r14 = &jit->io   (exit information; pc is its first field)
14 +
 *       r15 = budget     (guest instructions left, signed)
12 15
 *   - Guest registers are loaded/stored on demand from/to the array.
13 -
 *   - At block boundaries we store the next guest PC into *pc_out and
14 -
 *     return an exit-reason code in eax.
16 +
 *   - Each block starts with "if budget <= 0 exit at this PC; budget -=
17 +
 *     insn_count".  Every exit is the block's last instruction, so a block
18 +
 *     always accounts exactly the instructions it executes.
19 +
 *   - Direct jumps (jal, both sides of a branch, block split) are a jmp
20 +
 *     rel32 that first falls into a stub returning JIT_EXIT_CHAIN.  C then
21 +
 *     patches the jmp to the target block, so later runs stay in the cache.
22 +
 *   - Indirect jumps (jalr) probe a direct-mapped jump cache filled by C and
23 +
 *     jump to the hit; a miss returns JIT_EXIT_BRANCH.
15 24
 *
16 25
 *  We emit raw x86-64 bytes into the code cache.
17 26
 */
18 27
28 +
#include <stddef.h>
29 +
#include <stdint.h>
19 30
#include <string.h>
20 31
#include <sys/mman.h>
21 32
22 33
#include "jit.h"
23 34
#include "riscv.h"
29 40
struct emitter {
30 41
    u8  *buf;      /* Start of buffer. */
31 42
    u32  pos;      /* Current write position. */
32 43
    u32  capacity; /* Maximum bytes. */
33 44
    bool overflow; /* Set if we ran out of space. */
45 +
46 +
    struct jit_state *jit;  /* For the exit stub and jump cache addresses. */
47 +
    u8               *dest; /* Address the buffer will be copied to. */
34 48
};
35 49
36 50
static inline void emit_u8(struct emitter *e, u8 b) {
37 51
    if (e->pos < e->capacity)
38 52
        e->buf[e->pos++] = b;
86 100
static inline u8 sib(u8 scale, u8 index, u8 base) {
87 101
    return (u8)((scale << 6) | ((index & 7) << 3) | (base & 7));
88 102
}
89 103
90 104
/* Pinned host registers. */
91 -
#define HREGS  X_R12 /* &regs[0]  */
92 -
#define HMEM   X_R13 /* memory    */
93 -
#define HPCOUT X_R14 /* pc_out    */
105 +
#define HREGS   X_R12 /* &regs[0]  */
106 +
#define HMEM    X_R13 /* memory    */
107 +
#define HIO     X_R14 /* &jit->io  */
108 +
#define HBUDGET X_R15 /* budget    */
94 109
95 110
/* Scratch registers for codegen (caller-saved, not pinned). */
96 111
#define HTMP1 X_RAX
97 112
#define HTMP2 X_RCX
98 113
170 185
    emit_u8(e, 0xC7);
171 186
    emit_u8(e, modrm(3, 0, dst));
172 187
    emit_u32(e, (u32)imm);
173 188
}
174 189
175 -
/* Careful prologue using known encodings. */
176 -
static void emit_prologue(struct emitter *e) {
177 -
    emit_u8(e, 0x55); /* push rbp */
178 -
    emit_u8(e, 0x41);
179 -
    emit_u8(e, 0x54); /* push r12 */
180 -
    emit_u8(e, 0x41);
181 -
    emit_u8(e, 0x55); /* push r13 */
182 -
    emit_u8(e, 0x41);
183 -
    emit_u8(e, 0x56); /* push r14 */
184 -
    emit_u8(e, 0x49);
185 -
    emit_u8(e, 0x89);
186 -
    emit_u8(e, 0xFC); /* mov r12, rdi */
187 -
    emit_u8(e, 0x49);
188 -
    emit_u8(e, 0x89);
189 -
    emit_u8(e, 0xF5); /* mov r13, rsi */
190 -
    emit_u8(e, 0x49);
191 -
    emit_u8(e, 0x89);
192 -
    emit_u8(e, 0xD6); /* mov r14, rdx */
193 -
}
194 -
195 -
/* Emit function epilogue + ret with exit reason. */
196 -
static void emit_epilogue_with_exit(struct emitter *e, u32 exit_reason) {
197 -
    emit_u8(e, 0xB8);
198 -
    emit_u32(e, exit_reason); /* mov eax, reason */
199 -
    emit_u8(e, 0x41);
200 -
    emit_u8(e, 0x5E); /* pop r14 */
201 -
    emit_u8(e, 0x41);
202 -
    emit_u8(e, 0x5D); /* pop r13 */
203 -
    emit_u8(e, 0x41);
204 -
    emit_u8(e, 0x5C); /* pop r12 */
205 -
    emit_u8(e, 0x5D); /* pop rbp */
206 -
    emit_u8(e, 0xC3); /* ret */
207 -
}
208 -
209 -
/* Write next-PC to *r14, then epilogue+ret. */
210 -
static void emit_block_exit(
211 -
    struct emitter *e, enum jit_exit reason, u32 next_pc
212 -
) {
213 -
    /* mov dword [r14], next_pc */
214 -
    emit_u8(e, 0x41);
215 -
    emit_u8(e, 0xC7);
216 -
    emit_u8(e, modrm(0, 0, X_R14 & 7));
217 -
    emit_u32(e, next_pc);
218 -
    emit_epilogue_with_exit(e, (u32)reason);
219 -
}
220 -
221 190
/* ---------- Jump patching ---------------------------------------------- */
222 191
223 192
/* Emit jnz rel32 (0F 85), return position for patching. */
224 193
static u32 emit_jnz_placeholder(struct emitter *e) {
225 194
    u32 pos = e->pos;
227 196
    emit_u8(e, 0x85);
228 197
    emit_u32(e, 0);
229 198
    return pos;
230 199
}
231 200
201 +
/* Emit jle rel32 (0F 8E), return position for patching. */
202 +
static u32 emit_jle_placeholder(struct emitter *e) {
203 +
    u32 pos = e->pos;
204 +
    emit_u8(e, 0x0F);
205 +
    emit_u8(e, 0x8E);
206 +
    emit_u32(e, 0);
207 +
    return pos;
208 +
}
209 +
232 210
/* Emit jne rel32 (0F 85), return position for patching. */
233 211
#define emit_jne_placeholder emit_jnz_placeholder
234 212
235 213
/* Emit jmp rel32 (E9), return position for patching. */
236 214
static u32 emit_jmp_placeholder(struct emitter *e) {
256 234
    e->buf[jmp_pos + 2] = (u8)(rel >> 8);
257 235
    e->buf[jmp_pos + 3] = (u8)(rel >> 16);
258 236
    e->buf[jmp_pos + 4] = (u8)(rel >> 24);
259 237
}
260 238
239 +
/* Offsets of struct jit_io fields from r14. */
240 +
#define IO_BUDGET ((u8)offsetof(struct jit_io, budget))
241 +
#define IO_PATCH  ((u8)offsetof(struct jit_io, patch))
242 +
243 +
/* jmp rel32 to an absolute address in the code cache. */
244 +
static void emit_jmp_abs(struct emitter *e, u8 *target) {
245 +
    emit_u8(e, 0xE9);
246 +
    emit_u32(e, (u32)(target - (e->dest + e->pos + 4)));
247 +
}
248 +
249 +
/* mov dword [r14], imm32 -- store the next guest PC. */
250 +
static void emit_store_pc_imm(struct emitter *e, u32 pc) {
251 +
    emit_u8(e, 0x41);
252 +
    emit_u8(e, 0xC7);
253 +
    emit_u8(e, modrm(0, 0, HIO));
254 +
    emit_u32(e, pc);
255 +
}
256 +
257 +
/* mov eax, reason; jmp exit_stub. */
258 +
static void emit_exit(struct emitter *e, u32 exit_reason) {
259 +
    emit_u8(e, 0xB8);
260 +
    emit_u32(e, exit_reason); /* mov eax, reason */
261 +
    emit_jmp_abs(e, e->jit->exit_stub);
262 +
}
263 +
264 +
/* Store next-PC to *r14 and leave the cache with `reason`. */
265 +
static void emit_block_exit(
266 +
    struct emitter *e, enum jit_exit reason, u32 next_pc
267 +
) {
268 +
    emit_store_pc_imm(e, next_pc);
269 +
    emit_exit(e, (u32)reason);
270 +
}
271 +
272 +
/* Jump to the guest block at the constant PC `target`.  Emits a jmp rel32
273 +
 * that initially falls into a stub returning JIT_EXIT_CHAIN with the address
274 +
 * of its rel32 field in io.patch; see jit_patch_chain. */
275 +
static void emit_chain_exit(struct emitter *e, u32 target) {
276 +
    u32 site = e->pos;
277 +
    emit_u8(e, 0xE9);
278 +
    emit_u32(e, 0); /* jmp .stub (next instruction) */
279 +
280 +
    emit_store_pc_imm(e, target);
281 +
282 +
    /* lea rax, [rip + (site + 1) - next] */
283 +
    emit_u8(e, 0x48);
284 +
    emit_u8(e, 0x8D);
285 +
    emit_u8(e, modrm(0, X_RAX, 5));
286 +
    emit_u32(e, (site + 1) - (e->pos + 4));
287 +
288 +
    /* mov [r14 + IO_PATCH], rax */
289 +
    emit_u8(e, 0x49);
290 +
    emit_u8(e, 0x89);
291 +
    emit_u8(e, modrm(1, X_RAX, HIO));
292 +
    emit_u8(e, IO_PATCH);
293 +
294 +
    emit_exit(e, JIT_EXIT_CHAIN);
295 +
}
296 +
297 +
/* Jump to the guest PC in rax (already even).  Stores it to io.pc, probes the
298 +
 * jump cache and jumps to the cached block on a hit, else exits to C. */
299 +
static void emit_indirect_jump(struct emitter *e) {
300 +
    /* mov dword [r14], eax */
301 +
    emit_u8(e, 0x41);
302 +
    emit_u8(e, 0x89);
303 +
    emit_u8(e, modrm(0, X_RAX, HIO));
304 +
305 +
    /* mov ecx, eax; and ecx, mask << 2; shl ecx, 2 -- byte offset of the
306 +
     * entry, ((pc >> 2) & mask) * sizeof(struct jit_jump_entry). */
307 +
    emit_u8(e, 0x89);
308 +
    emit_u8(e, modrm(3, X_RAX, X_RCX));
309 +
    emit_u8(e, 0x81);
310 +
    emit_u8(e, modrm(3, 4, X_RCX));
311 +
    emit_u32(e, (JIT_JUMP_CACHE_SIZE - 1) << 2);
312 +
    emit_u8(e, 0xC1);
313 +
    emit_u8(e, modrm(3, 4, X_RCX));
314 +
    emit_u8(e, 2);
315 +
316 +
    emit_mov_imm64(e, X_RDX, (u64)(uintptr_t)e->jit->jump_cache);
317 +
318 +
    /* cmp eax, [rdx + rcx] */
319 +
    emit_u8(e, 0x3B);
320 +
    emit_u8(e, modrm(0, X_RAX, 4));
321 +
    emit_u8(e, sib(0, X_RCX, X_RDX));
322 +
323 +
    u32 jne_pos = emit_jnz_placeholder(e);
324 +
325 +
    /* jmp [rdx + rcx + offsetof(code)] */
326 +
    emit_u8(e, 0xFF);
327 +
    emit_u8(e, modrm(1, 4, 4));
328 +
    emit_u8(e, sib(0, X_RCX, X_RDX));
329 +
    emit_u8(e, offsetof(struct jit_jump_entry, code));
330 +
331 +
    /* Miss: io.pc is set, let C dispatch. */
332 +
    patch_jcc(e, jne_pos, e->pos);
333 +
    emit_exit(e, JIT_EXIT_BRANCH);
334 +
}
335 +
261 336
/* ---------- Per-instruction translation helpers ------------------------ */
262 337
263 338
/* Load rs1 into rax. */
264 339
static void emit_load_rs1(struct emitter *e, u32 rs1) {
265 340
    emit_load_guest(e, HTMP1, rs1);
697 772
        u32 target = pc + (u32)get_j_imm(ins);
698 773
        if (ins.j.rd != 0) {
699 774
            emit_mov_imm64(e, HTMP1, (u64)pc_next);
700 775
            emit_store_guest(e, ins.j.rd, HTMP1);
701 776
        }
702 -
        emit_block_exit(e, JIT_EXIT_BRANCH, target);
777 +
        emit_chain_exit(e, target);
703 778
        return true;
704 779
    }
705 780
706 781
    case OP_JALR: {
707 782
        i32 imm = get_i_imm(ins);
725 800
            emit_rexw(e);
726 801
            emit_u8(e, 0x83);
727 802
            emit_u8(e, modrm(3, 4, X_RAX));
728 803
            emit_u8(e, 0xFE); /* and rax, ~1 */
729 804
730 -
            /* mov dword [r14], eax -- write PC */
731 -
            emit_u8(e, 0x41);
732 -
            emit_u8(e, 0x89);
733 -
            emit_u8(e, modrm(0, X_RAX, X_R14 & 7));
734 -
735 -
            emit_epilogue_with_exit(e, (u32)JIT_EXIT_BRANCH);
805 +
            emit_indirect_jump(e);
736 806
            return true;
737 807
        }
738 808
739 809
        /* General JALR: target = (rs1 + imm) & ~1. */
740 810
        emit_load_rs1(e, ins.i.rs1);
742 812
        emit_rexw(e);
743 813
        emit_u8(e, 0x83);
744 814
        emit_u8(e, modrm(3, 4, X_RAX));
745 815
        emit_u8(e, 0xFE); /* and rax, ~1 */
746 816
747 -
        /* mov rcx, rax (save target) */
748 -
        emit_rexw(e);
749 -
        emit_u8(e, 0x89);
750 -
        emit_u8(e, modrm(3, X_RAX, X_RCX));
751 -
752 817
        if (ins.i.rd != 0) {
753 -
            emit_mov_imm64(e, HTMP1, (u64)pc_next);
754 -
            emit_store_guest(e, ins.i.rd, HTMP1);
818 +
            emit_mov_imm64(e, HTMP2, (u64)pc_next);
819 +
            emit_store_guest(e, ins.i.rd, HTMP2);
755 820
        }
756 821
757 -
        /* mov dword [r14], ecx */
758 -
        emit_u8(e, 0x41);
759 -
        emit_u8(e, 0x89);
760 -
        emit_u8(e, modrm(0, X_RCX, X_R14 & 7));
761 -
762 -
        emit_epilogue_with_exit(e, (u32)JIT_EXIT_BRANCH);
822 +
        emit_indirect_jump(e);
763 823
        return true;
764 824
    }
765 825
766 826
    case OP_BRANCH: {
767 827
        u32 target = pc + (u32)get_b_imm(ins);
800 860
        emit_u8(e, 0x0F);
801 861
        emit_u8(e, cc);
802 862
        emit_u32(e, 0);
803 863
804 864
        /* Not taken: fall through. */
805 -
        emit_block_exit(e, JIT_EXIT_BRANCH, pc_next);
865 +
        emit_chain_exit(e, pc_next);
806 866
807 867
        /* .taken: */
808 868
        patch_jcc(e, jcc_pos, e->pos);
809 -
        emit_block_exit(e, JIT_EXIT_BRANCH, target);
869 +
        emit_chain_exit(e, target);
810 870
        return true;
811 871
    }
812 872
813 873
    case OP_LOAD: {
814 874
        i32 imm = get_i_imm(ins);
1144 1204
    struct emitter em = {
1145 1205
        .buf      = emit_buf,
1146 1206
        .pos      = 0,
1147 1207
        .capacity = JIT_EMIT_BUF_SIZE,
1148 1208
        .overflow = false,
1209 +
        .jit      = jit,
1210 +
        .dest     = jit->code_cache + jit->code_cache_used,
1149 1211
    };
1150 1212
1151 -
    emit_prologue(&em);
1213 +
    /* if (budget <= 0) goto .exhausted */
1214 +
    emit_u8(&em, 0x4D);
1215 +
    emit_u8(&em, 0x85);
1216 +
    emit_u8(&em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */
1217 +
    u32 jle_pos = emit_jle_placeholder(&em);
1218 +
1219 +
    /* sub r15, insn_count -- imm32 is patched once the count is known. */
1220 +
    emit_u8(&em, 0x49);
1221 +
    emit_u8(&em, 0x81);
1222 +
    emit_u8(&em, modrm(3, 5, HBUDGET));
1223 +
    u32 count_pos = em.pos;
1224 +
    emit_u32(&em, 0);
1152 1225
1153 1226
    u32 pc          = guest_pc;
1154 1227
    u32 insn_count  = 0;
1155 1228
    u32 prog_end_pc = prog_base + prog_bytes;
1156 1229
1168 1241
1169 1242
        if (ends_block)
1170 1243
            break;
1171 1244
1172 1245
        if (em.pos + JIT_MAX_INSN_BYTES + 64 > em.capacity) {
1173 -
            emit_block_exit(&em, JIT_EXIT_BRANCH, pc);
1246 +
            emit_chain_exit(&em, pc);
1174 1247
            break;
1175 1248
        }
1176 1249
    }
1177 1250
1178 1251
    if (insn_count >= JIT_MAX_BLOCK_INSNS) {
1179 -
        emit_block_exit(&em, JIT_EXIT_BRANCH, pc);
1252 +
        emit_chain_exit(&em, pc);
1180 1253
    }
1181 1254
1255 +
    /* .exhausted: out of budget, leave before running anything. */
1256 +
    patch_jcc(&em, jle_pos, em.pos);
1257 +
    emit_block_exit(&em, JIT_EXIT_BRANCH, guest_pc);
1258 +
1182 1259
    if (em.overflow)
1183 1260
        return NULL;
1261 +
    for (u32 i = 0; i < 4; i++)
1262 +
        em.buf[count_pos + i] = (u8)(insn_count >> (8 * i));
1184 1263
1185 1264
    u32 code_size = em.pos;
1186 1265
    if (jit->code_cache_used + code_size > JIT_CODE_CACHE_SIZE)
1187 1266
        return NULL;
1188 1267
1192 1271
1193 1272
    struct jit_block *block = jit_alloc_block(jit);
1194 1273
    if (!block)
1195 1274
        return NULL;
1196 1275
1197 -
    block->guest_pc     = guest_pc;
1198 -
    block->guest_end_pc = pc;
1199 -
    block->insn_count   = insn_count;
1200 -
    block->code         = dest;
1201 -
    block->code_size    = code_size;
1202 -
    block->hash_next    = NULL;
1276 +
    block->guest_pc  = guest_pc;
1277 +
    block->code      = dest;
1278 +
    block->hash_next = NULL;
1203 1279
1204 1280
    jit_insert_block(jit, block);
1205 -
    jit->blocks_compiled++;
1206 1281
1207 1282
    return block;
1208 1283
}
1209 1284
1210 -
/* ---------- Init / Destroy / Flush ------------------------------------ */
1285 +
/* ---------- Chaining ---------------------------------------------------- */
1286 +
1287 +
void jit_patch_chain(u8 *site, struct jit_block *target) {
1288 +
    u32 rel = (u32)(target->code - (site + 4));
1289 +
    for (u32 i = 0; i < 4; i++)
1290 +
        site[i] = (u8)(rel >> (8 * i));
1291 +
}
1292 +
1293 +
/* ---------- Init / Destroy -------------------------------------------- */
1294 +
1295 +
/* Emit the shared exit stub and entry trampoline at the cache start. */
1296 +
static void jit_emit_trampolines(struct jit_state *jit) {
1297 +
    u8             buf[128];
1298 +
    struct emitter e = {
1299 +
        .buf      = buf,
1300 +
        .pos      = 0,
1301 +
        .capacity = sizeof(buf),
1302 +
        .overflow = false,
1303 +
        .jit      = jit,
1304 +
        .dest     = jit->code_cache,
1305 +
    };
1306 +
1307 +
    /* Exit stub: reason is in eax. */
1308 +
    jit->exit_stub = jit->code_cache;
1309 +
    emit_u8(&e, 0x4D);
1310 +
    emit_u8(&e, 0x89);
1311 +
    emit_u8(&e, modrm(1, HBUDGET, HIO));
1312 +
    emit_u8(&e, IO_BUDGET); /* mov [r14 + IO_BUDGET], r15 */
1313 +
    emit_u8(&e, 0x41);
1314 +
    emit_u8(&e, 0x5F); /* pop r15 */
1315 +
    emit_u8(&e, 0x41);
1316 +
    emit_u8(&e, 0x5E); /* pop r14 */
1317 +
    emit_u8(&e, 0x41);
1318 +
    emit_u8(&e, 0x5D); /* pop r13 */
1319 +
    emit_u8(&e, 0x41);
1320 +
    emit_u8(&e, 0x5C); /* pop r12 */
1321 +
    emit_u8(&e, 0xC3); /* ret */
1322 +
1323 +
    /* Entry: (rdi=regs, rsi=memory, rdx=io, rcx=code, r8=budget).  Blocks
1324 +
     * make no calls, so the stack alignment is irrelevant. */
1325 +
    jit->enter = (jit_enter_fn)(void *)(jit->code_cache + e.pos);
1326 +
    emit_u8(&e, 0x41);
1327 +
    emit_u8(&e, 0x54); /* push r12 */
1328 +
    emit_u8(&e, 0x41);
1329 +
    emit_u8(&e, 0x55); /* push r13 */
1330 +
    emit_u8(&e, 0x41);
1331 +
    emit_u8(&e, 0x56); /* push r14 */
1332 +
    emit_u8(&e, 0x41);
1333 +
    emit_u8(&e, 0x57); /* push r15 */
1334 +
    emit_u8(&e, 0x49);
1335 +
    emit_u8(&e, 0x89);
1336 +
    emit_u8(&e, 0xFC); /* mov r12, rdi */
1337 +
    emit_u8(&e, 0x49);
1338 +
    emit_u8(&e, 0x89);
1339 +
    emit_u8(&e, 0xF5); /* mov r13, rsi */
1340 +
    emit_u8(&e, 0x49);
1341 +
    emit_u8(&e, 0x89);
1342 +
    emit_u8(&e, 0xD6); /* mov r14, rdx */
1343 +
    emit_u8(&e, 0x4D);
1344 +
    emit_u8(&e, 0x89);
1345 +
    emit_u8(&e, 0xC7); /* mov r15, r8 */
1346 +
    emit_u8(&e, 0xFF);
1347 +
    emit_u8(&e, 0xE1); /* jmp rcx */
1348 +
1349 +
    memcpy(jit->code_cache, buf, e.pos);
1350 +
    jit->code_cache_used = e.pos;
1351 +
}
1211 1352
1212 1353
bool jit_init(struct jit_state *jit) {
1213 1354
    memset(jit, 0, sizeof(*jit));
1214 1355
1215 1356
#if defined(__x86_64__) || defined(_M_X64)
1224 1365
    if (jit->code_cache == MAP_FAILED) {
1225 1366
        jit->code_cache = NULL;
1226 1367
        jit->available  = false;
1227 1368
        return false;
1228 1369
    }
1370 +
    jit_emit_trampolines(jit);
1371 +
    for (u32 i = 0; i < JIT_JUMP_CACHE_SIZE; i++)
1372 +
        jit->jump_cache[i].guest_pc = JIT_JUMP_CACHE_EMPTY;
1229 1373
    jit->available = true;
1230 1374
    return true;
1231 1375
#else
1232 1376
    jit->available = false;
1233 1377
    return false;
1239 1383
        munmap(jit->code_cache, JIT_CODE_CACHE_SIZE);
1240 1384
        jit->code_cache = NULL;
1241 1385
    }
1242 1386
    jit->block_count = 0;
1243 1387
}
1244 -
1245 -
void jit_flush(struct jit_state *jit) {
1246 -
    memset(jit->block_hash, 0, sizeof(jit->block_hash));
1247 -
    jit->block_count     = 0;
1248 -
    jit->code_cache_used = 0;
1249 -
}
jit.h +67 -24
36 36
#define JIT_BLOCK_HASH_SIZE (64 * 1024)
37 37
38 38
/* Maximum number of translated blocks. */
39 39
#define JIT_MAX_BLOCKS (256 * 1024)
40 40
41 +
/* Number of jump cache entries (must be power of 2). */
42 +
#define JIT_JUMP_CACHE_SIZE 16384
43 +
44 +
/* Guest PC that marks an empty jump cache entry.  Indirect jump targets are
45 +
 * always even, so this never matches. */
46 +
#define JIT_JUMP_CACHE_EMPTY 0xFFFFFFFFu
47 +
41 48
/* Exit reasons returned by generated code. */
42 49
enum jit_exit {
43 -
    JIT_EXIT_BRANCH = 0, /* End of block -- branch/jump, update PC. */
50 +
    JIT_EXIT_BRANCH = 0, /* Return to C, which continues at io.pc. */
44 51
    JIT_EXIT_ECALL  = 1, /* ECALL -- needs interpreter handling. */
45 52
    JIT_EXIT_EBREAK = 2, /* EBREAK -- needs interpreter handling. */
46 53
    JIT_EXIT_FAULT  = 3, /* Memory fault -- needs interpreter handling. */
47 -
    JIT_EXIT_CHAIN  = 4, /* Chain to next block (PC already set). */
54 +
    JIT_EXIT_CHAIN  = 4, /* Direct jump to a known PC; io.patch is its site. */
48 55
    JIT_EXIT_RET    = 5, /* Program returned (jalr x0, ra, 0 with ra==0). */
49 56
};
50 57
51 58
/* A translated block of native code. */
52 59
struct jit_block {
53 -
    u32               guest_pc;     /* Guest PC this block starts at. */
54 -
    u32               guest_end_pc; /* Guest PC *after* last instruction. */
55 -
    u32               insn_count; /* Number of guest instructions translated.*/
56 -
    u8               *code;       /* Pointer into code cache. */
57 -
    u32               code_size;  /* Size of generated x86-64 code. */
58 -
    struct jit_block *hash_next;  /* Hash chain. */
60 +
    u32               guest_pc;  /* Guest PC this block starts at. */
61 +
    u8               *code;      /* Pointer into code cache. */
62 +
    struct jit_block *hash_next; /* Hash chain. */
63 +
};
64 +
65 +
/* Exchange area between generated code and C.  Generated code keeps a pointer
66 +
 * to it in r14. */
67 +
struct jit_io {
68 +
    u32 pc;     /* Next guest PC on exit. */
69 +
    i64 budget; /* Remaining guest instructions; in on entry, out on exit. */
70 +
    u8 *patch;  /* On JIT_EXIT_CHAIN, the rel32 field of the jmp to patch. */
59 71
};
60 72
73 +
/* One entry of the indirect jump cache.  Generated code indexes the cache
74 +
 * assuming 16-byte entries. */
75 +
struct jit_jump_entry {
76 +
    u32 guest_pc; /* Guest PC, or JIT_JUMP_CACHE_EMPTY. */
77 +
    u8 *code;     /* Entry point of the block at that PC. */
78 +
};
79 +
80 +
/* Signature of the shared entry trampoline.  It saves host registers, pins
81 +
 * regs/memory/io/budget and jumps to `code`; the shared exit stub returns the
82 +
 * exit reason. */
83 +
typedef int (*jit_enter_fn)(
84 +
    u64 *regs, u8 *memory, struct jit_io *io, u8 *code, i64 budget
85 +
);
86 +
61 87
/* Complete JIT state. */
62 88
struct jit_state {
63 89
    /* Executable code cache (mmap'd RWX). */
64 90
    u8 *code_cache;
65 91
    u32 code_cache_used;
69 95
70 96
    /* Block storage (static pool). */
71 97
    struct jit_block blocks[JIT_MAX_BLOCKS];
72 98
    u32              block_count;
73 99
74 -
    /* Statistics. */
75 -
    u64 blocks_compiled;
76 -
    u64 blocks_executed;
77 -
    u64 insns_executed;
100 +
    /* Shared entry trampoline and exit stub, at the start of the cache. */
101 +
    jit_enter_fn enter;
102 +
    u8          *exit_stub;
103 +
104 +
    /* Exit information for the current run. */
105 +
    struct jit_io io;
106 +
107 +
    /* Direct-mapped cache of indirect jump targets, read by generated code
108 +
     * and filled by C. */
109 +
    struct jit_jump_entry jump_cache[JIT_JUMP_CACHE_SIZE];
78 110
79 111
    /* Whether JIT is available (mmap succeeded, x86-64 host). */
80 112
    bool available;
81 113
};
82 114
90 122
bool jit_init(struct jit_state *jit);
91 123
92 124
/* Release JIT resources. */
93 125
void jit_destroy(struct jit_state *jit);
94 126
95 -
/* Flush the entire translation cache (e.g. after self-modifying code). */
96 -
void jit_flush(struct jit_state *jit);
97 -
98 127
/* Compile (but don't look up) a block starting at `guest_pc`.
99 128
 * Returns NULL if the code cache is full or compilation fails. */
100 129
struct jit_block *jit_compile_block(
101 130
    struct jit_state *jit,
102 131
    u32               guest_pc,
134 163
    return jit_compile_block(
135 164
        jit, guest_pc, memory, program_base, program_bytes
136 165
    );
137 166
}
138 167
139 -
/* Signature of generated block entry points.
140 -
 * Args: pointer to cpu->regs[], pointer to memory[].
141 -
 * Returns: exit reason (enum jit_exit). */
142 -
typedef int (*jit_block_fn)(u64 *regs, u8 *memory, u32 *pc_out);
168 +
/* Record `block` as the target of indirect jumps to its PC. */
169 +
static inline void jit_cache_jump(
170 +
    struct jit_state *jit, struct jit_block *block
171 +
) {
172 +
    struct jit_jump_entry *ent =
173 +
        &jit->jump_cache[(block->guest_pc >> 2) & (JIT_JUMP_CACHE_SIZE - 1)];
174 +
    ent->guest_pc = block->guest_pc;
175 +
    ent->code     = block->code;
176 +
}
177 +
178 +
/* Point the direct jump whose rel32 field is at `site` to `target`. */
179 +
void jit_patch_chain(u8 *site, struct jit_block *target);
143 180
144 -
/* Execute a translated block.  Returns the exit reason. */
145 -
static inline int jit_exec_block(
146 -
    struct jit_block *block, u64 *regs, u8 *memory, u32 *pc_out
181 +
/* Run translated code starting at `block`, with room for `budget` more guest
182 +
 * instructions.  Generated code follows direct and cached indirect jumps
183 +
 * without returning.  Returns the exit reason; the next PC, the remaining
184 +
 * budget (which may be negative) and any patch site are left in jit->io. */
185 +
static inline int jit_run(
186 +
    struct jit_state *jit,
187 +
    struct jit_block *block,
188 +
    u64              *regs,
189 +
    u8               *memory,
190 +
    i64               budget
147 191
) {
148 -
    jit_block_fn fn = (jit_block_fn)(void *)block->code;
149 -
    return fn(regs, memory, pc_out);
192 +
    return jit->enter(regs, memory, &jit->io, block->code, budget);
150 193
}
151 194
152 195
#endif /* JIT_H */