emulator: Speed up execution
064c3426ad28d10dd695f0ca71c7d3385df15b89db49678cdfc043ef4dfaea8a
Every translated block returned to C after each branch, and each had its own prologue and epilogue. Startup also cleared all of guest memory, which touched every page of the configured memory size, 376 MiB for compiler runs, and dominated short runs: 96 small compiles took 2.8 s with 24 in parallel. Blocks now chain. A shared entry trampoline and exit stub replace the per-block prologue and epilogue. Direct jumps are patched to the target block once C has translated it, and indirect jumps go through a direct-mapped jump cache filled by C. A budget register keeps instruction counting exact: each block checks that budget remains, then charges its instruction count, so -count-instructions and -max-steps still see every guest instruction. Guest memory is static storage and cpu_init runs once, before the program is loaded, so the memory is already zero. cpu_init no longer clears it; the same 96 compiles take 0.3 s. Drop JIT state that nothing reads: jit_flush, which nothing calls, the end PC, instruction count and code size of each block, and the block statistics. Assisted-by: Claude:sonnet-5.5 Assisted-by: Claude:opus-5.5
1 parent
d9bc0eae
emulator.c
+20 -9
| 1203 | 1203 | cpu->ebreak = false; |
|
| 1204 | 1204 | cpu->modified = ZERO; |
|
| 1205 | 1205 | last_executed_pc = 0; |
|
| 1206 | 1206 | } |
|
| 1207 | 1207 | ||
| 1208 | - | /* Initialize CPU and memory to a clean state. */ |
|
| 1208 | + | /* Initialize the CPU to a clean state. Guest memory is not cleared: this runs |
|
| 1209 | + | * once at startup, when the static memory is still zero, and clearing it would |
|
| 1210 | + | * touch every page. */ |
|
| 1209 | 1211 | static void cpu_init(struct cpu *cpu) { |
|
| 1210 | - | memset(memory, 0, g_opts.memory_size); |
|
| 1211 | 1212 | cpu->program = (instr_t *)memory; |
|
| 1212 | 1213 | cpu->programsize = 0; |
|
| 1213 | 1214 | trace_reset(); |
|
| 1214 | 1215 | guest_fd_table_init(); |
|
| 1215 | 1216 | cpu_reset(cpu); |
| 2575 | 2576 | use_jit = jit_init(&jit); |
|
| 2576 | 2577 | } |
|
| 2577 | 2578 | ||
| 2578 | 2579 | if (use_jit) { |
|
| 2579 | 2580 | /* ---- JIT execution loop ---- */ |
|
| 2581 | + | u8 *patch = NULL; /* Direct jump to link to the next block. */ |
|
| 2582 | + | ||
| 2580 | 2583 | while (cpu.running && steps < max_steps) { |
|
| 2581 | 2584 | struct jit_block *block = jit_get_block( |
|
| 2582 | 2585 | &jit, cpu.pc, memory, program_base, program_bytes |
|
| 2583 | 2586 | ); |
|
| 2584 | 2587 | if (!block) { |
| 2587 | 2590 | while (cpu.running && steps++ < max_steps) { |
|
| 2588 | 2591 | cpu_execute(&cpu, display, true); |
|
| 2589 | 2592 | } |
|
| 2590 | 2593 | break; |
|
| 2591 | 2594 | } |
|
| 2592 | - | u32 next_pc = 0; |
|
| 2595 | + | if (patch) |
|
| 2596 | + | jit_patch_chain(patch, block); |
|
| 2597 | + | jit_cache_jump(&jit, block); |
|
| 2598 | + | ||
| 2599 | + | /* Generated code charges each block's instructions against |
|
| 2600 | + | * the budget, so `steps` advances by what it consumed. */ |
|
| 2601 | + | u64 left = max_steps - steps; |
|
| 2602 | + | i64 budget = left > INT64_MAX ? INT64_MAX : (i64)left; |
|
| 2593 | 2603 | int exit_reason = |
|
| 2594 | - | jit_exec_block(block, cpu.regs, memory, &next_pc); |
|
| 2595 | - | steps += block->insn_count; |
|
| 2596 | - | jit.blocks_executed++; |
|
| 2597 | - | jit.insns_executed += block->insn_count; |
|
| 2604 | + | jit_run(&jit, block, cpu.regs, memory, budget); |
|
| 2605 | + | steps += (u64)budget - (u64)jit.io.budget; |
|
| 2606 | + | cpu.pc = jit.io.pc; |
|
| 2607 | + | patch = NULL; |
|
| 2598 | 2608 | ||
| 2599 | 2609 | switch (exit_reason) { |
|
| 2600 | 2610 | case JIT_EXIT_BRANCH: |
|
| 2611 | + | break; |
|
| 2612 | + | ||
| 2601 | 2613 | case JIT_EXIT_CHAIN: |
|
| 2602 | - | cpu.pc = next_pc; |
|
| 2614 | + | patch = jit.io.patch; |
|
| 2603 | 2615 | break; |
|
| 2604 | 2616 | ||
| 2605 | 2617 | case JIT_EXIT_RET: |
|
| 2606 | 2618 | cpu.running = false; |
|
| 2607 | 2619 | break; |
|
| 2608 | 2620 | ||
| 2609 | 2621 | default: |
|
| 2610 | 2622 | /* ECALL, EBREAK, FAULT -- interpreter handles it. |
|
| 2611 | 2623 | * The instruction is already counted above, |
|
| 2612 | 2624 | * so don't increment steps again. */ |
|
| 2613 | - | cpu.pc = next_pc; |
|
| 2614 | 2625 | cpu_execute(&cpu, display, true); |
|
| 2615 | 2626 | break; |
|
| 2616 | 2627 | } |
|
| 2617 | 2628 | } |
|
| 2618 | 2629 | jit_destroy(&jit); |
jit.c
+232 -94
| 1 | 1 | /* |
|
| 2 | 2 | * JIT compiler: RV64I basic-block -> x86-64 native code. |
|
| 3 | 3 | * |
|
| 4 | 4 | * Strategy: |
|
| 5 | - | * - Guest registers live in a u64[32] array pointed to by the first arg |
|
| 6 | - | * (rdi on entry). Memory base is in rsi. The third arg (rdx) is a |
|
| 7 | - | * pointer where we write the next guest PC on exit. |
|
| 5 | + | * - One entry trampoline and one exit stub are generated at init. The |
|
| 6 | + | * trampoline saves the callee-saved registers, pins the registers below |
|
| 7 | + | * and jumps into a block. The exit stub stores the remaining budget, |
|
| 8 | + | * restores the registers and returns the exit reason in eax. Blocks |
|
| 9 | + | * have no prologue or epilogue and never call. |
|
| 8 | 10 | * - During block execution we pin: |
|
| 9 | 11 | * r12 = ®s[0] (guest register file) |
|
| 10 | 12 | * r13 = memory base |
|
| 11 | - | * r14 = pc_out pointer |
|
| 13 | + | * r14 = &jit->io (exit information; pc is its first field) |
|
| 14 | + | * r15 = budget (guest instructions left, signed) |
|
| 12 | 15 | * - Guest registers are loaded/stored on demand from/to the array. |
|
| 13 | - | * - At block boundaries we store the next guest PC into *pc_out and |
|
| 14 | - | * return an exit-reason code in eax. |
|
| 16 | + | * - Each block starts with "if budget <= 0 exit at this PC; budget -= |
|
| 17 | + | * insn_count". Every exit is the block's last instruction, so a block |
|
| 18 | + | * always accounts exactly the instructions it executes. |
|
| 19 | + | * - Direct jumps (jal, both sides of a branch, block split) are a jmp |
|
| 20 | + | * rel32 that first falls into a stub returning JIT_EXIT_CHAIN. C then |
|
| 21 | + | * patches the jmp to the target block, so later runs stay in the cache. |
|
| 22 | + | * - Indirect jumps (jalr) probe a direct-mapped jump cache filled by C and |
|
| 23 | + | * jump to the hit; a miss returns JIT_EXIT_BRANCH. |
|
| 15 | 24 | * |
|
| 16 | 25 | * We emit raw x86-64 bytes into the code cache. |
|
| 17 | 26 | */ |
|
| 18 | 27 | ||
| 28 | + | #include <stddef.h> |
|
| 29 | + | #include <stdint.h> |
|
| 19 | 30 | #include <string.h> |
|
| 20 | 31 | #include <sys/mman.h> |
|
| 21 | 32 | ||
| 22 | 33 | #include "jit.h" |
|
| 23 | 34 | #include "riscv.h" |
| 29 | 40 | struct emitter { |
|
| 30 | 41 | u8 *buf; /* Start of buffer. */ |
|
| 31 | 42 | u32 pos; /* Current write position. */ |
|
| 32 | 43 | u32 capacity; /* Maximum bytes. */ |
|
| 33 | 44 | bool overflow; /* Set if we ran out of space. */ |
|
| 45 | + | ||
| 46 | + | struct jit_state *jit; /* For the exit stub and jump cache addresses. */ |
|
| 47 | + | u8 *dest; /* Address the buffer will be copied to. */ |
|
| 34 | 48 | }; |
|
| 35 | 49 | ||
| 36 | 50 | static inline void emit_u8(struct emitter *e, u8 b) { |
|
| 37 | 51 | if (e->pos < e->capacity) |
|
| 38 | 52 | e->buf[e->pos++] = b; |
| 86 | 100 | static inline u8 sib(u8 scale, u8 index, u8 base) { |
|
| 87 | 101 | return (u8)((scale << 6) | ((index & 7) << 3) | (base & 7)); |
|
| 88 | 102 | } |
|
| 89 | 103 | ||
| 90 | 104 | /* Pinned host registers. */ |
|
| 91 | - | #define HREGS X_R12 /* ®s[0] */ |
|
| 92 | - | #define HMEM X_R13 /* memory */ |
|
| 93 | - | #define HPCOUT X_R14 /* pc_out */ |
|
| 105 | + | #define HREGS X_R12 /* ®s[0] */ |
|
| 106 | + | #define HMEM X_R13 /* memory */ |
|
| 107 | + | #define HIO X_R14 /* &jit->io */ |
|
| 108 | + | #define HBUDGET X_R15 /* budget */ |
|
| 94 | 109 | ||
| 95 | 110 | /* Scratch registers for codegen (caller-saved, not pinned). */ |
|
| 96 | 111 | #define HTMP1 X_RAX |
|
| 97 | 112 | #define HTMP2 X_RCX |
|
| 98 | 113 |
| 170 | 185 | emit_u8(e, 0xC7); |
|
| 171 | 186 | emit_u8(e, modrm(3, 0, dst)); |
|
| 172 | 187 | emit_u32(e, (u32)imm); |
|
| 173 | 188 | } |
|
| 174 | 189 | ||
| 175 | - | /* Careful prologue using known encodings. */ |
|
| 176 | - | static void emit_prologue(struct emitter *e) { |
|
| 177 | - | emit_u8(e, 0x55); /* push rbp */ |
|
| 178 | - | emit_u8(e, 0x41); |
|
| 179 | - | emit_u8(e, 0x54); /* push r12 */ |
|
| 180 | - | emit_u8(e, 0x41); |
|
| 181 | - | emit_u8(e, 0x55); /* push r13 */ |
|
| 182 | - | emit_u8(e, 0x41); |
|
| 183 | - | emit_u8(e, 0x56); /* push r14 */ |
|
| 184 | - | emit_u8(e, 0x49); |
|
| 185 | - | emit_u8(e, 0x89); |
|
| 186 | - | emit_u8(e, 0xFC); /* mov r12, rdi */ |
|
| 187 | - | emit_u8(e, 0x49); |
|
| 188 | - | emit_u8(e, 0x89); |
|
| 189 | - | emit_u8(e, 0xF5); /* mov r13, rsi */ |
|
| 190 | - | emit_u8(e, 0x49); |
|
| 191 | - | emit_u8(e, 0x89); |
|
| 192 | - | emit_u8(e, 0xD6); /* mov r14, rdx */ |
|
| 193 | - | } |
|
| 194 | - | ||
| 195 | - | /* Emit function epilogue + ret with exit reason. */ |
|
| 196 | - | static void emit_epilogue_with_exit(struct emitter *e, u32 exit_reason) { |
|
| 197 | - | emit_u8(e, 0xB8); |
|
| 198 | - | emit_u32(e, exit_reason); /* mov eax, reason */ |
|
| 199 | - | emit_u8(e, 0x41); |
|
| 200 | - | emit_u8(e, 0x5E); /* pop r14 */ |
|
| 201 | - | emit_u8(e, 0x41); |
|
| 202 | - | emit_u8(e, 0x5D); /* pop r13 */ |
|
| 203 | - | emit_u8(e, 0x41); |
|
| 204 | - | emit_u8(e, 0x5C); /* pop r12 */ |
|
| 205 | - | emit_u8(e, 0x5D); /* pop rbp */ |
|
| 206 | - | emit_u8(e, 0xC3); /* ret */ |
|
| 207 | - | } |
|
| 208 | - | ||
| 209 | - | /* Write next-PC to *r14, then epilogue+ret. */ |
|
| 210 | - | static void emit_block_exit( |
|
| 211 | - | struct emitter *e, enum jit_exit reason, u32 next_pc |
|
| 212 | - | ) { |
|
| 213 | - | /* mov dword [r14], next_pc */ |
|
| 214 | - | emit_u8(e, 0x41); |
|
| 215 | - | emit_u8(e, 0xC7); |
|
| 216 | - | emit_u8(e, modrm(0, 0, X_R14 & 7)); |
|
| 217 | - | emit_u32(e, next_pc); |
|
| 218 | - | emit_epilogue_with_exit(e, (u32)reason); |
|
| 219 | - | } |
|
| 220 | - | ||
| 221 | 190 | /* ---------- Jump patching ---------------------------------------------- */ |
|
| 222 | 191 | ||
| 223 | 192 | /* Emit jnz rel32 (0F 85), return position for patching. */ |
|
| 224 | 193 | static u32 emit_jnz_placeholder(struct emitter *e) { |
|
| 225 | 194 | u32 pos = e->pos; |
| 227 | 196 | emit_u8(e, 0x85); |
|
| 228 | 197 | emit_u32(e, 0); |
|
| 229 | 198 | return pos; |
|
| 230 | 199 | } |
|
| 231 | 200 | ||
| 201 | + | /* Emit jle rel32 (0F 8E), return position for patching. */ |
|
| 202 | + | static u32 emit_jle_placeholder(struct emitter *e) { |
|
| 203 | + | u32 pos = e->pos; |
|
| 204 | + | emit_u8(e, 0x0F); |
|
| 205 | + | emit_u8(e, 0x8E); |
|
| 206 | + | emit_u32(e, 0); |
|
| 207 | + | return pos; |
|
| 208 | + | } |
|
| 209 | + | ||
| 232 | 210 | /* Emit jne rel32 (0F 85), return position for patching. */ |
|
| 233 | 211 | #define emit_jne_placeholder emit_jnz_placeholder |
|
| 234 | 212 | ||
| 235 | 213 | /* Emit jmp rel32 (E9), return position for patching. */ |
|
| 236 | 214 | static u32 emit_jmp_placeholder(struct emitter *e) { |
| 256 | 234 | e->buf[jmp_pos + 2] = (u8)(rel >> 8); |
|
| 257 | 235 | e->buf[jmp_pos + 3] = (u8)(rel >> 16); |
|
| 258 | 236 | e->buf[jmp_pos + 4] = (u8)(rel >> 24); |
|
| 259 | 237 | } |
|
| 260 | 238 | ||
| 239 | + | /* Offsets of struct jit_io fields from r14. */ |
|
| 240 | + | #define IO_BUDGET ((u8)offsetof(struct jit_io, budget)) |
|
| 241 | + | #define IO_PATCH ((u8)offsetof(struct jit_io, patch)) |
|
| 242 | + | ||
| 243 | + | /* jmp rel32 to an absolute address in the code cache. */ |
|
| 244 | + | static void emit_jmp_abs(struct emitter *e, u8 *target) { |
|
| 245 | + | emit_u8(e, 0xE9); |
|
| 246 | + | emit_u32(e, (u32)(target - (e->dest + e->pos + 4))); |
|
| 247 | + | } |
|
| 248 | + | ||
| 249 | + | /* mov dword [r14], imm32 -- store the next guest PC. */ |
|
| 250 | + | static void emit_store_pc_imm(struct emitter *e, u32 pc) { |
|
| 251 | + | emit_u8(e, 0x41); |
|
| 252 | + | emit_u8(e, 0xC7); |
|
| 253 | + | emit_u8(e, modrm(0, 0, HIO)); |
|
| 254 | + | emit_u32(e, pc); |
|
| 255 | + | } |
|
| 256 | + | ||
| 257 | + | /* mov eax, reason; jmp exit_stub. */ |
|
| 258 | + | static void emit_exit(struct emitter *e, u32 exit_reason) { |
|
| 259 | + | emit_u8(e, 0xB8); |
|
| 260 | + | emit_u32(e, exit_reason); /* mov eax, reason */ |
|
| 261 | + | emit_jmp_abs(e, e->jit->exit_stub); |
|
| 262 | + | } |
|
| 263 | + | ||
| 264 | + | /* Store next-PC to *r14 and leave the cache with `reason`. */ |
|
| 265 | + | static void emit_block_exit( |
|
| 266 | + | struct emitter *e, enum jit_exit reason, u32 next_pc |
|
| 267 | + | ) { |
|
| 268 | + | emit_store_pc_imm(e, next_pc); |
|
| 269 | + | emit_exit(e, (u32)reason); |
|
| 270 | + | } |
|
| 271 | + | ||
| 272 | + | /* Jump to the guest block at the constant PC `target`. Emits a jmp rel32 |
|
| 273 | + | * that initially falls into a stub returning JIT_EXIT_CHAIN with the address |
|
| 274 | + | * of its rel32 field in io.patch; see jit_patch_chain. */ |
|
| 275 | + | static void emit_chain_exit(struct emitter *e, u32 target) { |
|
| 276 | + | u32 site = e->pos; |
|
| 277 | + | emit_u8(e, 0xE9); |
|
| 278 | + | emit_u32(e, 0); /* jmp .stub (next instruction) */ |
|
| 279 | + | ||
| 280 | + | emit_store_pc_imm(e, target); |
|
| 281 | + | ||
| 282 | + | /* lea rax, [rip + (site + 1) - next] */ |
|
| 283 | + | emit_u8(e, 0x48); |
|
| 284 | + | emit_u8(e, 0x8D); |
|
| 285 | + | emit_u8(e, modrm(0, X_RAX, 5)); |
|
| 286 | + | emit_u32(e, (site + 1) - (e->pos + 4)); |
|
| 287 | + | ||
| 288 | + | /* mov [r14 + IO_PATCH], rax */ |
|
| 289 | + | emit_u8(e, 0x49); |
|
| 290 | + | emit_u8(e, 0x89); |
|
| 291 | + | emit_u8(e, modrm(1, X_RAX, HIO)); |
|
| 292 | + | emit_u8(e, IO_PATCH); |
|
| 293 | + | ||
| 294 | + | emit_exit(e, JIT_EXIT_CHAIN); |
|
| 295 | + | } |
|
| 296 | + | ||
| 297 | + | /* Jump to the guest PC in rax (already even). Stores it to io.pc, probes the |
|
| 298 | + | * jump cache and jumps to the cached block on a hit, else exits to C. */ |
|
| 299 | + | static void emit_indirect_jump(struct emitter *e) { |
|
| 300 | + | /* mov dword [r14], eax */ |
|
| 301 | + | emit_u8(e, 0x41); |
|
| 302 | + | emit_u8(e, 0x89); |
|
| 303 | + | emit_u8(e, modrm(0, X_RAX, HIO)); |
|
| 304 | + | ||
| 305 | + | /* mov ecx, eax; and ecx, mask << 2; shl ecx, 2 -- byte offset of the |
|
| 306 | + | * entry, ((pc >> 2) & mask) * sizeof(struct jit_jump_entry). */ |
|
| 307 | + | emit_u8(e, 0x89); |
|
| 308 | + | emit_u8(e, modrm(3, X_RAX, X_RCX)); |
|
| 309 | + | emit_u8(e, 0x81); |
|
| 310 | + | emit_u8(e, modrm(3, 4, X_RCX)); |
|
| 311 | + | emit_u32(e, (JIT_JUMP_CACHE_SIZE - 1) << 2); |
|
| 312 | + | emit_u8(e, 0xC1); |
|
| 313 | + | emit_u8(e, modrm(3, 4, X_RCX)); |
|
| 314 | + | emit_u8(e, 2); |
|
| 315 | + | ||
| 316 | + | emit_mov_imm64(e, X_RDX, (u64)(uintptr_t)e->jit->jump_cache); |
|
| 317 | + | ||
| 318 | + | /* cmp eax, [rdx + rcx] */ |
|
| 319 | + | emit_u8(e, 0x3B); |
|
| 320 | + | emit_u8(e, modrm(0, X_RAX, 4)); |
|
| 321 | + | emit_u8(e, sib(0, X_RCX, X_RDX)); |
|
| 322 | + | ||
| 323 | + | u32 jne_pos = emit_jnz_placeholder(e); |
|
| 324 | + | ||
| 325 | + | /* jmp [rdx + rcx + offsetof(code)] */ |
|
| 326 | + | emit_u8(e, 0xFF); |
|
| 327 | + | emit_u8(e, modrm(1, 4, 4)); |
|
| 328 | + | emit_u8(e, sib(0, X_RCX, X_RDX)); |
|
| 329 | + | emit_u8(e, offsetof(struct jit_jump_entry, code)); |
|
| 330 | + | ||
| 331 | + | /* Miss: io.pc is set, let C dispatch. */ |
|
| 332 | + | patch_jcc(e, jne_pos, e->pos); |
|
| 333 | + | emit_exit(e, JIT_EXIT_BRANCH); |
|
| 334 | + | } |
|
| 335 | + | ||
| 261 | 336 | /* ---------- Per-instruction translation helpers ------------------------ */ |
|
| 262 | 337 | ||
| 263 | 338 | /* Load rs1 into rax. */ |
|
| 264 | 339 | static void emit_load_rs1(struct emitter *e, u32 rs1) { |
|
| 265 | 340 | emit_load_guest(e, HTMP1, rs1); |
| 697 | 772 | u32 target = pc + (u32)get_j_imm(ins); |
|
| 698 | 773 | if (ins.j.rd != 0) { |
|
| 699 | 774 | emit_mov_imm64(e, HTMP1, (u64)pc_next); |
|
| 700 | 775 | emit_store_guest(e, ins.j.rd, HTMP1); |
|
| 701 | 776 | } |
|
| 702 | - | emit_block_exit(e, JIT_EXIT_BRANCH, target); |
|
| 777 | + | emit_chain_exit(e, target); |
|
| 703 | 778 | return true; |
|
| 704 | 779 | } |
|
| 705 | 780 | ||
| 706 | 781 | case OP_JALR: { |
|
| 707 | 782 | i32 imm = get_i_imm(ins); |
| 725 | 800 | emit_rexw(e); |
|
| 726 | 801 | emit_u8(e, 0x83); |
|
| 727 | 802 | emit_u8(e, modrm(3, 4, X_RAX)); |
|
| 728 | 803 | emit_u8(e, 0xFE); /* and rax, ~1 */ |
|
| 729 | 804 | ||
| 730 | - | /* mov dword [r14], eax -- write PC */ |
|
| 731 | - | emit_u8(e, 0x41); |
|
| 732 | - | emit_u8(e, 0x89); |
|
| 733 | - | emit_u8(e, modrm(0, X_RAX, X_R14 & 7)); |
|
| 734 | - | ||
| 735 | - | emit_epilogue_with_exit(e, (u32)JIT_EXIT_BRANCH); |
|
| 805 | + | emit_indirect_jump(e); |
|
| 736 | 806 | return true; |
|
| 737 | 807 | } |
|
| 738 | 808 | ||
| 739 | 809 | /* General JALR: target = (rs1 + imm) & ~1. */ |
|
| 740 | 810 | emit_load_rs1(e, ins.i.rs1); |
| 742 | 812 | emit_rexw(e); |
|
| 743 | 813 | emit_u8(e, 0x83); |
|
| 744 | 814 | emit_u8(e, modrm(3, 4, X_RAX)); |
|
| 745 | 815 | emit_u8(e, 0xFE); /* and rax, ~1 */ |
|
| 746 | 816 | ||
| 747 | - | /* mov rcx, rax (save target) */ |
|
| 748 | - | emit_rexw(e); |
|
| 749 | - | emit_u8(e, 0x89); |
|
| 750 | - | emit_u8(e, modrm(3, X_RAX, X_RCX)); |
|
| 751 | - | ||
| 752 | 817 | if (ins.i.rd != 0) { |
|
| 753 | - | emit_mov_imm64(e, HTMP1, (u64)pc_next); |
|
| 754 | - | emit_store_guest(e, ins.i.rd, HTMP1); |
|
| 818 | + | emit_mov_imm64(e, HTMP2, (u64)pc_next); |
|
| 819 | + | emit_store_guest(e, ins.i.rd, HTMP2); |
|
| 755 | 820 | } |
|
| 756 | 821 | ||
| 757 | - | /* mov dword [r14], ecx */ |
|
| 758 | - | emit_u8(e, 0x41); |
|
| 759 | - | emit_u8(e, 0x89); |
|
| 760 | - | emit_u8(e, modrm(0, X_RCX, X_R14 & 7)); |
|
| 761 | - | ||
| 762 | - | emit_epilogue_with_exit(e, (u32)JIT_EXIT_BRANCH); |
|
| 822 | + | emit_indirect_jump(e); |
|
| 763 | 823 | return true; |
|
| 764 | 824 | } |
|
| 765 | 825 | ||
| 766 | 826 | case OP_BRANCH: { |
|
| 767 | 827 | u32 target = pc + (u32)get_b_imm(ins); |
| 800 | 860 | emit_u8(e, 0x0F); |
|
| 801 | 861 | emit_u8(e, cc); |
|
| 802 | 862 | emit_u32(e, 0); |
|
| 803 | 863 | ||
| 804 | 864 | /* Not taken: fall through. */ |
|
| 805 | - | emit_block_exit(e, JIT_EXIT_BRANCH, pc_next); |
|
| 865 | + | emit_chain_exit(e, pc_next); |
|
| 806 | 866 | ||
| 807 | 867 | /* .taken: */ |
|
| 808 | 868 | patch_jcc(e, jcc_pos, e->pos); |
|
| 809 | - | emit_block_exit(e, JIT_EXIT_BRANCH, target); |
|
| 869 | + | emit_chain_exit(e, target); |
|
| 810 | 870 | return true; |
|
| 811 | 871 | } |
|
| 812 | 872 | ||
| 813 | 873 | case OP_LOAD: { |
|
| 814 | 874 | i32 imm = get_i_imm(ins); |
| 1144 | 1204 | struct emitter em = { |
|
| 1145 | 1205 | .buf = emit_buf, |
|
| 1146 | 1206 | .pos = 0, |
|
| 1147 | 1207 | .capacity = JIT_EMIT_BUF_SIZE, |
|
| 1148 | 1208 | .overflow = false, |
|
| 1209 | + | .jit = jit, |
|
| 1210 | + | .dest = jit->code_cache + jit->code_cache_used, |
|
| 1149 | 1211 | }; |
|
| 1150 | 1212 | ||
| 1151 | - | emit_prologue(&em); |
|
| 1213 | + | /* if (budget <= 0) goto .exhausted */ |
|
| 1214 | + | emit_u8(&em, 0x4D); |
|
| 1215 | + | emit_u8(&em, 0x85); |
|
| 1216 | + | emit_u8(&em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */ |
|
| 1217 | + | u32 jle_pos = emit_jle_placeholder(&em); |
|
| 1218 | + | ||
| 1219 | + | /* sub r15, insn_count -- imm32 is patched once the count is known. */ |
|
| 1220 | + | emit_u8(&em, 0x49); |
|
| 1221 | + | emit_u8(&em, 0x81); |
|
| 1222 | + | emit_u8(&em, modrm(3, 5, HBUDGET)); |
|
| 1223 | + | u32 count_pos = em.pos; |
|
| 1224 | + | emit_u32(&em, 0); |
|
| 1152 | 1225 | ||
| 1153 | 1226 | u32 pc = guest_pc; |
|
| 1154 | 1227 | u32 insn_count = 0; |
|
| 1155 | 1228 | u32 prog_end_pc = prog_base + prog_bytes; |
|
| 1156 | 1229 |
| 1168 | 1241 | ||
| 1169 | 1242 | if (ends_block) |
|
| 1170 | 1243 | break; |
|
| 1171 | 1244 | ||
| 1172 | 1245 | if (em.pos + JIT_MAX_INSN_BYTES + 64 > em.capacity) { |
|
| 1173 | - | emit_block_exit(&em, JIT_EXIT_BRANCH, pc); |
|
| 1246 | + | emit_chain_exit(&em, pc); |
|
| 1174 | 1247 | break; |
|
| 1175 | 1248 | } |
|
| 1176 | 1249 | } |
|
| 1177 | 1250 | ||
| 1178 | 1251 | if (insn_count >= JIT_MAX_BLOCK_INSNS) { |
|
| 1179 | - | emit_block_exit(&em, JIT_EXIT_BRANCH, pc); |
|
| 1252 | + | emit_chain_exit(&em, pc); |
|
| 1180 | 1253 | } |
|
| 1181 | 1254 | ||
| 1255 | + | /* .exhausted: out of budget, leave before running anything. */ |
|
| 1256 | + | patch_jcc(&em, jle_pos, em.pos); |
|
| 1257 | + | emit_block_exit(&em, JIT_EXIT_BRANCH, guest_pc); |
|
| 1258 | + | ||
| 1182 | 1259 | if (em.overflow) |
|
| 1183 | 1260 | return NULL; |
|
| 1261 | + | for (u32 i = 0; i < 4; i++) |
|
| 1262 | + | em.buf[count_pos + i] = (u8)(insn_count >> (8 * i)); |
|
| 1184 | 1263 | ||
| 1185 | 1264 | u32 code_size = em.pos; |
|
| 1186 | 1265 | if (jit->code_cache_used + code_size > JIT_CODE_CACHE_SIZE) |
|
| 1187 | 1266 | return NULL; |
|
| 1188 | 1267 |
| 1192 | 1271 | ||
| 1193 | 1272 | struct jit_block *block = jit_alloc_block(jit); |
|
| 1194 | 1273 | if (!block) |
|
| 1195 | 1274 | return NULL; |
|
| 1196 | 1275 | ||
| 1197 | - | block->guest_pc = guest_pc; |
|
| 1198 | - | block->guest_end_pc = pc; |
|
| 1199 | - | block->insn_count = insn_count; |
|
| 1200 | - | block->code = dest; |
|
| 1201 | - | block->code_size = code_size; |
|
| 1202 | - | block->hash_next = NULL; |
|
| 1276 | + | block->guest_pc = guest_pc; |
|
| 1277 | + | block->code = dest; |
|
| 1278 | + | block->hash_next = NULL; |
|
| 1203 | 1279 | ||
| 1204 | 1280 | jit_insert_block(jit, block); |
|
| 1205 | - | jit->blocks_compiled++; |
|
| 1206 | 1281 | ||
| 1207 | 1282 | return block; |
|
| 1208 | 1283 | } |
|
| 1209 | 1284 | ||
| 1210 | - | /* ---------- Init / Destroy / Flush ------------------------------------ */ |
|
| 1285 | + | /* ---------- Chaining ---------------------------------------------------- */ |
|
| 1286 | + | ||
| 1287 | + | void jit_patch_chain(u8 *site, struct jit_block *target) { |
|
| 1288 | + | u32 rel = (u32)(target->code - (site + 4)); |
|
| 1289 | + | for (u32 i = 0; i < 4; i++) |
|
| 1290 | + | site[i] = (u8)(rel >> (8 * i)); |
|
| 1291 | + | } |
|
| 1292 | + | ||
| 1293 | + | /* ---------- Init / Destroy -------------------------------------------- */ |
|
| 1294 | + | ||
| 1295 | + | /* Emit the shared exit stub and entry trampoline at the cache start. */ |
|
| 1296 | + | static void jit_emit_trampolines(struct jit_state *jit) { |
|
| 1297 | + | u8 buf[128]; |
|
| 1298 | + | struct emitter e = { |
|
| 1299 | + | .buf = buf, |
|
| 1300 | + | .pos = 0, |
|
| 1301 | + | .capacity = sizeof(buf), |
|
| 1302 | + | .overflow = false, |
|
| 1303 | + | .jit = jit, |
|
| 1304 | + | .dest = jit->code_cache, |
|
| 1305 | + | }; |
|
| 1306 | + | ||
| 1307 | + | /* Exit stub: reason is in eax. */ |
|
| 1308 | + | jit->exit_stub = jit->code_cache; |
|
| 1309 | + | emit_u8(&e, 0x4D); |
|
| 1310 | + | emit_u8(&e, 0x89); |
|
| 1311 | + | emit_u8(&e, modrm(1, HBUDGET, HIO)); |
|
| 1312 | + | emit_u8(&e, IO_BUDGET); /* mov [r14 + IO_BUDGET], r15 */ |
|
| 1313 | + | emit_u8(&e, 0x41); |
|
| 1314 | + | emit_u8(&e, 0x5F); /* pop r15 */ |
|
| 1315 | + | emit_u8(&e, 0x41); |
|
| 1316 | + | emit_u8(&e, 0x5E); /* pop r14 */ |
|
| 1317 | + | emit_u8(&e, 0x41); |
|
| 1318 | + | emit_u8(&e, 0x5D); /* pop r13 */ |
|
| 1319 | + | emit_u8(&e, 0x41); |
|
| 1320 | + | emit_u8(&e, 0x5C); /* pop r12 */ |
|
| 1321 | + | emit_u8(&e, 0xC3); /* ret */ |
|
| 1322 | + | ||
| 1323 | + | /* Entry: (rdi=regs, rsi=memory, rdx=io, rcx=code, r8=budget). Blocks |
|
| 1324 | + | * make no calls, so the stack alignment is irrelevant. */ |
|
| 1325 | + | jit->enter = (jit_enter_fn)(void *)(jit->code_cache + e.pos); |
|
| 1326 | + | emit_u8(&e, 0x41); |
|
| 1327 | + | emit_u8(&e, 0x54); /* push r12 */ |
|
| 1328 | + | emit_u8(&e, 0x41); |
|
| 1329 | + | emit_u8(&e, 0x55); /* push r13 */ |
|
| 1330 | + | emit_u8(&e, 0x41); |
|
| 1331 | + | emit_u8(&e, 0x56); /* push r14 */ |
|
| 1332 | + | emit_u8(&e, 0x41); |
|
| 1333 | + | emit_u8(&e, 0x57); /* push r15 */ |
|
| 1334 | + | emit_u8(&e, 0x49); |
|
| 1335 | + | emit_u8(&e, 0x89); |
|
| 1336 | + | emit_u8(&e, 0xFC); /* mov r12, rdi */ |
|
| 1337 | + | emit_u8(&e, 0x49); |
|
| 1338 | + | emit_u8(&e, 0x89); |
|
| 1339 | + | emit_u8(&e, 0xF5); /* mov r13, rsi */ |
|
| 1340 | + | emit_u8(&e, 0x49); |
|
| 1341 | + | emit_u8(&e, 0x89); |
|
| 1342 | + | emit_u8(&e, 0xD6); /* mov r14, rdx */ |
|
| 1343 | + | emit_u8(&e, 0x4D); |
|
| 1344 | + | emit_u8(&e, 0x89); |
|
| 1345 | + | emit_u8(&e, 0xC7); /* mov r15, r8 */ |
|
| 1346 | + | emit_u8(&e, 0xFF); |
|
| 1347 | + | emit_u8(&e, 0xE1); /* jmp rcx */ |
|
| 1348 | + | ||
| 1349 | + | memcpy(jit->code_cache, buf, e.pos); |
|
| 1350 | + | jit->code_cache_used = e.pos; |
|
| 1351 | + | } |
|
| 1211 | 1352 | ||
| 1212 | 1353 | bool jit_init(struct jit_state *jit) { |
|
| 1213 | 1354 | memset(jit, 0, sizeof(*jit)); |
|
| 1214 | 1355 | ||
| 1215 | 1356 | #if defined(__x86_64__) || defined(_M_X64) |
| 1224 | 1365 | if (jit->code_cache == MAP_FAILED) { |
|
| 1225 | 1366 | jit->code_cache = NULL; |
|
| 1226 | 1367 | jit->available = false; |
|
| 1227 | 1368 | return false; |
|
| 1228 | 1369 | } |
|
| 1370 | + | jit_emit_trampolines(jit); |
|
| 1371 | + | for (u32 i = 0; i < JIT_JUMP_CACHE_SIZE; i++) |
|
| 1372 | + | jit->jump_cache[i].guest_pc = JIT_JUMP_CACHE_EMPTY; |
|
| 1229 | 1373 | jit->available = true; |
|
| 1230 | 1374 | return true; |
|
| 1231 | 1375 | #else |
|
| 1232 | 1376 | jit->available = false; |
|
| 1233 | 1377 | return false; |
| 1239 | 1383 | munmap(jit->code_cache, JIT_CODE_CACHE_SIZE); |
|
| 1240 | 1384 | jit->code_cache = NULL; |
|
| 1241 | 1385 | } |
|
| 1242 | 1386 | jit->block_count = 0; |
|
| 1243 | 1387 | } |
|
| 1244 | - | ||
| 1245 | - | void jit_flush(struct jit_state *jit) { |
|
| 1246 | - | memset(jit->block_hash, 0, sizeof(jit->block_hash)); |
|
| 1247 | - | jit->block_count = 0; |
|
| 1248 | - | jit->code_cache_used = 0; |
|
| 1249 | - | } |
jit.h
+67 -24
| 36 | 36 | #define JIT_BLOCK_HASH_SIZE (64 * 1024) |
|
| 37 | 37 | ||
| 38 | 38 | /* Maximum number of translated blocks. */ |
|
| 39 | 39 | #define JIT_MAX_BLOCKS (256 * 1024) |
|
| 40 | 40 | ||
| 41 | + | /* Number of jump cache entries (must be power of 2). */ |
|
| 42 | + | #define JIT_JUMP_CACHE_SIZE 16384 |
|
| 43 | + | ||
| 44 | + | /* Guest PC that marks an empty jump cache entry. Indirect jump targets are |
|
| 45 | + | * always even, so this never matches. */ |
|
| 46 | + | #define JIT_JUMP_CACHE_EMPTY 0xFFFFFFFFu |
|
| 47 | + | ||
| 41 | 48 | /* Exit reasons returned by generated code. */ |
|
| 42 | 49 | enum jit_exit { |
|
| 43 | - | JIT_EXIT_BRANCH = 0, /* End of block -- branch/jump, update PC. */ |
|
| 50 | + | JIT_EXIT_BRANCH = 0, /* Return to C, which continues at io.pc. */ |
|
| 44 | 51 | JIT_EXIT_ECALL = 1, /* ECALL -- needs interpreter handling. */ |
|
| 45 | 52 | JIT_EXIT_EBREAK = 2, /* EBREAK -- needs interpreter handling. */ |
|
| 46 | 53 | JIT_EXIT_FAULT = 3, /* Memory fault -- needs interpreter handling. */ |
|
| 47 | - | JIT_EXIT_CHAIN = 4, /* Chain to next block (PC already set). */ |
|
| 54 | + | JIT_EXIT_CHAIN = 4, /* Direct jump to a known PC; io.patch is its site. */ |
|
| 48 | 55 | JIT_EXIT_RET = 5, /* Program returned (jalr x0, ra, 0 with ra==0). */ |
|
| 49 | 56 | }; |
|
| 50 | 57 | ||
| 51 | 58 | /* A translated block of native code. */ |
|
| 52 | 59 | struct jit_block { |
|
| 53 | - | u32 guest_pc; /* Guest PC this block starts at. */ |
|
| 54 | - | u32 guest_end_pc; /* Guest PC *after* last instruction. */ |
|
| 55 | - | u32 insn_count; /* Number of guest instructions translated.*/ |
|
| 56 | - | u8 *code; /* Pointer into code cache. */ |
|
| 57 | - | u32 code_size; /* Size of generated x86-64 code. */ |
|
| 58 | - | struct jit_block *hash_next; /* Hash chain. */ |
|
| 60 | + | u32 guest_pc; /* Guest PC this block starts at. */ |
|
| 61 | + | u8 *code; /* Pointer into code cache. */ |
|
| 62 | + | struct jit_block *hash_next; /* Hash chain. */ |
|
| 63 | + | }; |
|
| 64 | + | ||
| 65 | + | /* Exchange area between generated code and C. Generated code keeps a pointer |
|
| 66 | + | * to it in r14. */ |
|
| 67 | + | struct jit_io { |
|
| 68 | + | u32 pc; /* Next guest PC on exit. */ |
|
| 69 | + | i64 budget; /* Remaining guest instructions; in on entry, out on exit. */ |
|
| 70 | + | u8 *patch; /* On JIT_EXIT_CHAIN, the rel32 field of the jmp to patch. */ |
|
| 59 | 71 | }; |
|
| 60 | 72 | ||
| 73 | + | /* One entry of the indirect jump cache. Generated code indexes the cache |
|
| 74 | + | * assuming 16-byte entries. */ |
|
| 75 | + | struct jit_jump_entry { |
|
| 76 | + | u32 guest_pc; /* Guest PC, or JIT_JUMP_CACHE_EMPTY. */ |
|
| 77 | + | u8 *code; /* Entry point of the block at that PC. */ |
|
| 78 | + | }; |
|
| 79 | + | ||
| 80 | + | /* Signature of the shared entry trampoline. It saves host registers, pins |
|
| 81 | + | * regs/memory/io/budget and jumps to `code`; the shared exit stub returns the |
|
| 82 | + | * exit reason. */ |
|
| 83 | + | typedef int (*jit_enter_fn)( |
|
| 84 | + | u64 *regs, u8 *memory, struct jit_io *io, u8 *code, i64 budget |
|
| 85 | + | ); |
|
| 86 | + | ||
| 61 | 87 | /* Complete JIT state. */ |
|
| 62 | 88 | struct jit_state { |
|
| 63 | 89 | /* Executable code cache (mmap'd RWX). */ |
|
| 64 | 90 | u8 *code_cache; |
|
| 65 | 91 | u32 code_cache_used; |
| 69 | 95 | ||
| 70 | 96 | /* Block storage (static pool). */ |
|
| 71 | 97 | struct jit_block blocks[JIT_MAX_BLOCKS]; |
|
| 72 | 98 | u32 block_count; |
|
| 73 | 99 | ||
| 74 | - | /* Statistics. */ |
|
| 75 | - | u64 blocks_compiled; |
|
| 76 | - | u64 blocks_executed; |
|
| 77 | - | u64 insns_executed; |
|
| 100 | + | /* Shared entry trampoline and exit stub, at the start of the cache. */ |
|
| 101 | + | jit_enter_fn enter; |
|
| 102 | + | u8 *exit_stub; |
|
| 103 | + | ||
| 104 | + | /* Exit information for the current run. */ |
|
| 105 | + | struct jit_io io; |
|
| 106 | + | ||
| 107 | + | /* Direct-mapped cache of indirect jump targets, read by generated code |
|
| 108 | + | * and filled by C. */ |
|
| 109 | + | struct jit_jump_entry jump_cache[JIT_JUMP_CACHE_SIZE]; |
|
| 78 | 110 | ||
| 79 | 111 | /* Whether JIT is available (mmap succeeded, x86-64 host). */ |
|
| 80 | 112 | bool available; |
|
| 81 | 113 | }; |
|
| 82 | 114 |
| 90 | 122 | bool jit_init(struct jit_state *jit); |
|
| 91 | 123 | ||
| 92 | 124 | /* Release JIT resources. */ |
|
| 93 | 125 | void jit_destroy(struct jit_state *jit); |
|
| 94 | 126 | ||
| 95 | - | /* Flush the entire translation cache (e.g. after self-modifying code). */ |
|
| 96 | - | void jit_flush(struct jit_state *jit); |
|
| 97 | - | ||
| 98 | 127 | /* Compile (but don't look up) a block starting at `guest_pc`. |
|
| 99 | 128 | * Returns NULL if the code cache is full or compilation fails. */ |
|
| 100 | 129 | struct jit_block *jit_compile_block( |
|
| 101 | 130 | struct jit_state *jit, |
|
| 102 | 131 | u32 guest_pc, |
| 134 | 163 | return jit_compile_block( |
|
| 135 | 164 | jit, guest_pc, memory, program_base, program_bytes |
|
| 136 | 165 | ); |
|
| 137 | 166 | } |
|
| 138 | 167 | ||
| 139 | - | /* Signature of generated block entry points. |
|
| 140 | - | * Args: pointer to cpu->regs[], pointer to memory[]. |
|
| 141 | - | * Returns: exit reason (enum jit_exit). */ |
|
| 142 | - | typedef int (*jit_block_fn)(u64 *regs, u8 *memory, u32 *pc_out); |
|
| 168 | + | /* Record `block` as the target of indirect jumps to its PC. */ |
|
| 169 | + | static inline void jit_cache_jump( |
|
| 170 | + | struct jit_state *jit, struct jit_block *block |
|
| 171 | + | ) { |
|
| 172 | + | struct jit_jump_entry *ent = |
|
| 173 | + | &jit->jump_cache[(block->guest_pc >> 2) & (JIT_JUMP_CACHE_SIZE - 1)]; |
|
| 174 | + | ent->guest_pc = block->guest_pc; |
|
| 175 | + | ent->code = block->code; |
|
| 176 | + | } |
|
| 177 | + | ||
| 178 | + | /* Point the direct jump whose rel32 field is at `site` to `target`. */ |
|
| 179 | + | void jit_patch_chain(u8 *site, struct jit_block *target); |
|
| 143 | 180 | ||
| 144 | - | /* Execute a translated block. Returns the exit reason. */ |
|
| 145 | - | static inline int jit_exec_block( |
|
| 146 | - | struct jit_block *block, u64 *regs, u8 *memory, u32 *pc_out |
|
| 181 | + | /* Run translated code starting at `block`, with room for `budget` more guest |
|
| 182 | + | * instructions. Generated code follows direct and cached indirect jumps |
|
| 183 | + | * without returning. Returns the exit reason; the next PC, the remaining |
|
| 184 | + | * budget (which may be negative) and any patch site are left in jit->io. */ |
|
| 185 | + | static inline int jit_run( |
|
| 186 | + | struct jit_state *jit, |
|
| 187 | + | struct jit_block *block, |
|
| 188 | + | u64 *regs, |
|
| 189 | + | u8 *memory, |
|
| 190 | + | i64 budget |
|
| 147 | 191 | ) { |
|
| 148 | - | jit_block_fn fn = (jit_block_fn)(void *)block->code; |
|
| 149 | - | return fn(regs, memory, pc_out); |
|
| 192 | + | return jit->enter(regs, memory, &jit->io, block->code, budget); |
|
| 150 | 193 | } |
|
| 151 | 194 | ||
| 152 | 195 | #endif /* JIT_H */ |