jit: Speed up translated calls and register access
a503d772dc87f89f24f3f56c9fcf45f7e10edd3ce1d05dbe6fe0148d49b2b118
Use native calls and returns for direct guest calls, and charge the instruction budget at block exits. Share a base register for the guest register file and JIT exchange area to hold another guest register. Radiance self-compilation runs 20% faster with identical output and guest instruction counts. Assisted-By: Codex:gpt-6.2-sol Change-Id: I31c39b37ad5d63319e55069d6d9974ab93a23e89
1 parent
4799a8d7
jit.c
+278 -242
| 4 | 4 | * Strategy: |
|
| 5 | 5 | * - One entry trampoline and one exit stub are generated at init. The |
|
| 6 | 6 | * trampoline saves the callee-saved registers, pins the registers below |
|
| 7 | 7 | * and jumps into a block. The exit stub stores the remaining budget, |
|
| 8 | 8 | * restores the registers and returns the exit reason in eax. Blocks |
|
| 9 | - | * have no prologue or epilogue and never call. |
|
| 9 | + | * can call other blocks directly. |
|
| 10 | 10 | * - During block execution we pin: |
|
| 11 | - | * r12 = ®s[0] (guest register file) |
|
| 12 | 11 | * r13 = memory base |
|
| 13 | - | * r14 = &jit->io (exit information; pc is its first field) |
|
| 12 | + | * r14 = &jit->io (exit information and register file base) |
|
| 14 | 13 | * r15 = budget (guest instructions left, signed) |
|
| 15 | 14 | * - Guest registers are loaded/stored on demand from/to the array. |
|
| 16 | 15 | * - Vector operations check their element width, group size, active length |
|
| 17 | 16 | * and restart index before access. |
|
| 18 | - | * - Each block starts with "if budget <= 0 exit at this PC; budget -= |
|
| 19 | - | * insn_count". Every exit is the block's last instruction, so a block |
|
| 20 | - | * always accounts exactly the instructions it executes. |
|
| 17 | + | * - Each block starts with a budget check. Every exit charges the |
|
| 18 | + | * instructions on its path. A jump to another block carries the flags |
|
| 19 | + | * from the subtraction, so the target needs only a conditional jump. |
|
| 21 | 20 | * - Direct jumps (jal, both sides of a branch, block split) are a jmp |
|
| 22 | 21 | * rel32 that first falls into a stub returning JIT_EXIT_CHAIN. C then |
|
| 23 | 22 | * patches the jmp to the target block, so later runs stay in the cache. |
|
| 24 | 23 | * - Indirect jumps (jalr) probe a direct-mapped jump cache filled by C and |
|
| 25 | 24 | * jump to the hit; a miss returns JIT_EXIT_BRANCH. |
|
| 25 | + | * - Direct calls save the guest return address on the host stack. A return |
|
| 26 | + | * to that address uses a native ret. The overflow flag at a return |
|
| 27 | + | * continuation marks an unwind to C. |
|
| 26 | 28 | * |
|
| 27 | 29 | * We emit raw x86-64 bytes into the code cache. |
|
| 28 | 30 | */ |
|
| 29 | 31 | ||
| 30 | 32 | #include <stddef.h> |
| 51 | 53 | /* The register file holds every guest register and host registers may be |
|
| 52 | 54 | * clobbered: set from an instruction that addresses the register file |
|
| 53 | 55 | * directly until the held registers are reloaded. */ |
|
| 54 | 56 | bool spilled; |
|
| 55 | 57 | u32 insns; /* Instructions through the current operation. */ |
|
| 56 | - | /* Each indexed load can fail three checks for each of four lanes. */ |
|
| 57 | - | u32 refunds[JIT_MAX_BLOCK_INSNS * 12]; /* Budget patch offsets. */ |
|
| 58 | - | u32 consumed[JIT_MAX_BLOCK_INSNS * 12]; /* Instructions at each patch. */ |
|
| 59 | - | u32 refunds_len; /* Number of guarded exits. */ |
|
| 60 | 58 | ||
| 61 | - | /* Exits from inside the block, on a branch or a checked return, emitted |
|
| 62 | - | * after its main path. */ |
|
| 59 | + | /* Conditional branch exits, emitted after the main path. */ |
|
| 63 | 60 | struct { |
|
| 64 | - | u32 jcc; /* Offset of the jcc rel32 instruction. */ |
|
| 65 | - | u32 target; /* Guest PC of the branch target. */ |
|
| 66 | - | u32 insns; /* Instructions executed when it is taken. */ |
|
| 67 | - | bool ret; /* Return to ra instead of jumping to `target`. */ |
|
| 61 | + | u32 jcc; /* Offset of the jcc rel32 instruction. */ |
|
| 62 | + | u32 target; /* Guest PC of the branch target. */ |
|
| 63 | + | u32 insns; /* Instructions executed when it is taken. */ |
|
| 68 | 64 | } sides[JIT_MAX_SIDE_EXITS]; |
|
| 69 | 65 | u32 sides_len; |
|
| 70 | 66 | ||
| 71 | 67 | /* Guest PC after the current instruction. */ |
|
| 72 | 68 | u32 next_pc; |
| 76 | 72 | u32 run_start[JIT_MAX_FOLLOWS + 1]; |
|
| 77 | 73 | u32 run_end[JIT_MAX_FOLLOWS + 1]; |
|
| 78 | 74 | u32 runs; |
|
| 79 | 75 | /* The previous instruction set `auipc_reg` to `auipc_value` with auipc, |
|
| 80 | 76 | * or `auipc_reg` is zero. */ |
|
| 81 | - | u32 auipc_reg; |
|
| 82 | - | u64 auipc_value; |
|
| 83 | - | /* ra holds `ra_value`, set by a call the block followed. */ |
|
| 84 | - | bool ra_known; |
|
| 85 | - | u32 ra_value; |
|
| 86 | - | ||
| 77 | + | u32 auipc_reg; |
|
| 78 | + | u64 auipc_value; |
|
| 87 | 79 | u8 *mem; /* Guest memory, for looking ahead. */ |
|
| 88 | 80 | u32 prog_base; /* Guest code bounds. */ |
|
| 89 | 81 | u32 prog_end; |
|
| 90 | 82 | struct jit_state *jit; /* For the exit stub and jump cache addresses. */ |
|
| 91 | 83 | u8 *dest; /* Address the buffer will be copied to. */ |
| 144 | 136 | static inline u8 sib(u8 scale, u8 index, u8 base) { |
|
| 145 | 137 | return (u8)((scale << 6) | ((index & 7) << 3) | (base & 7)); |
|
| 146 | 138 | } |
|
| 147 | 139 | ||
| 148 | 140 | /* Pinned host registers. */ |
|
| 149 | - | #define HREGS X_R12 /* ®s[0] */ |
|
| 150 | 141 | #define HMEM X_R13 /* memory */ |
|
| 151 | 142 | #define HIO X_R14 /* &jit->io */ |
|
| 152 | 143 | #define HBUDGET X_R15 /* budget */ |
|
| 153 | 144 | ||
| 154 | 145 | /* Scratch registers for codegen (caller-saved, not pinned). */ |
| 166 | 157 | [2] = X_RDI, /* sp */ |
|
| 167 | 158 | [30] = X_R8, /* t5 */ |
|
| 168 | 159 | [7] = X_R9, /* t2 */ |
|
| 169 | 160 | [10] = X_R10, /* a0 */ |
|
| 170 | 161 | [28] = X_R11, /* t3 */ |
|
| 162 | + | [9] = X_R12, /* s1 */ |
|
| 171 | 163 | [11] = X_RDX, /* a1 */ |
|
| 172 | 164 | }; |
|
| 173 | 165 | ||
| 174 | 166 | /* Host register holding `guest`, or zero if it lives in the register file. */ |
|
| 175 | 167 | static inline u8 host_of(struct emitter *e, u32 guest) { |
| 199 | 191 | emit_u8(e, modrm(3, ext, X_RAX)); |
|
| 200 | 192 | } |
|
| 201 | 193 | ||
| 202 | 194 | /* ---------- Load/store guest register from register file --------------- */ |
|
| 203 | 195 | ||
| 204 | - | /* Emit the ModR/M, SIB, and displacement for [r12 + off]. */ |
|
| 205 | - | static void emit_r12_address(struct emitter *e, enum x86reg reg, u32 off) { |
|
| 206 | - | if (off < 128) { |
|
| 207 | - | emit_u8(e, modrm(1, reg, X_R12 & 7)); |
|
| 208 | - | emit_u8(e, sib(0, 4, X_R12 & 7)); |
|
| 209 | - | emit_u8(e, (u8)off); |
|
| 196 | + | /* Address byte offset `off` in the guest register file adjacent to io. */ |
|
| 197 | + | static void emit_reg_address(struct emitter *e, enum x86reg reg, u32 off) { |
|
| 198 | + | i32 disp = (i32)offsetof(struct jit_state, regs) - |
|
| 199 | + | (i32)offsetof(struct jit_state, io) + (i32)off; |
|
| 200 | + | if (disp >= INT8_MIN && disp <= INT8_MAX) { |
|
| 201 | + | emit_u8(e, modrm(1, reg, HIO)); |
|
| 202 | + | emit_u8(e, (u8)disp); |
|
| 210 | 203 | } else { |
|
| 211 | - | emit_u8(e, modrm(2, reg, X_R12 & 7)); |
|
| 212 | - | emit_u8(e, sib(0, 4, X_R12 & 7)); |
|
| 213 | - | emit_u32(e, off); |
|
| 204 | + | emit_u8(e, modrm(2, reg, HIO)); |
|
| 205 | + | emit_u32(e, (u32)disp); |
|
| 214 | 206 | } |
|
| 215 | 207 | } |
|
| 216 | 208 | ||
| 217 | - | /* Emit mov with [r12 + disp] addressing (r12 needs SIB). */ |
|
| 218 | - | static void emit_r12_disp( |
|
| 209 | + | /* Emit a 64-bit operation on a guest register in the register file. */ |
|
| 210 | + | static void emit_reg_disp( |
|
| 219 | 211 | struct emitter *e, u8 opcode, enum x86reg reg, u32 off |
|
| 220 | 212 | ) { |
|
| 221 | - | emit_u8(e, rex(true, reg >= 8, false, true)); /* B=1 for r12 */ |
|
| 213 | + | emit_u8(e, rex(true, reg >= 8, false, true)); |
|
| 222 | 214 | emit_u8(e, opcode); |
|
| 223 | - | emit_r12_address(e, reg, off); |
|
| 215 | + | emit_reg_address(e, reg, off); |
|
| 224 | 216 | } |
|
| 225 | 217 | ||
| 226 | 218 | /* mov dst, src (64-bit), omitted when they are the same register. */ |
|
| 227 | 219 | static void emit_mov_rr(struct emitter *e, u8 dst, u8 src) { |
|
| 228 | 220 | if (dst == src) |
| 233 | 225 | } |
|
| 234 | 226 | ||
| 235 | 227 | /* Load guest register `guest` into host `dst`. */ |
|
| 236 | 228 | static void emit_load_guest(struct emitter *e, enum x86reg dst, u32 guest) { |
|
| 237 | 229 | if (guest == 0) { |
|
| 238 | - | /* x0 is always zero -- xor dst, dst. */ |
|
| 239 | - | emit_u8(e, rex(true, dst >= 8, false, dst >= 8)); |
|
| 230 | + | /* xor dst32, dst32 clears the full register. */ |
|
| 231 | + | if (dst >= 8) |
|
| 232 | + | emit_u8(e, rex(false, true, false, true)); |
|
| 240 | 233 | emit_u8(e, 0x31); |
|
| 241 | 234 | emit_u8(e, modrm(3, dst, dst)); |
|
| 242 | 235 | return; |
|
| 243 | 236 | } |
|
| 244 | 237 | u8 host = host_of(e, guest); |
|
| 245 | 238 | if (host) { |
|
| 246 | 239 | emit_mov_rr(e, dst, host); |
|
| 247 | 240 | return; |
|
| 248 | 241 | } |
|
| 249 | - | emit_r12_disp(e, 0x8B, dst, guest * 8); |
|
| 242 | + | emit_reg_disp(e, 0x8B, dst, guest * 8); |
|
| 250 | 243 | } |
|
| 251 | 244 | ||
| 252 | 245 | /* Load and zero-extend the low word of a guest register. */ |
|
| 253 | 246 | static void emit_load_guest32(struct emitter *e, enum x86reg dst, u32 guest) { |
|
| 254 | 247 | if (guest == 0) { |
| 264 | 257 | emit_u8(e, modrm(3, host, dst)); |
|
| 265 | 258 | return; |
|
| 266 | 259 | } |
|
| 267 | 260 | emit_u8(e, rex(false, dst >= 8, false, true)); |
|
| 268 | 261 | emit_u8(e, 0x8b); |
|
| 269 | - | emit_r12_address(e, dst, guest * 8); |
|
| 262 | + | emit_reg_address(e, dst, guest * 8); |
|
| 270 | 263 | } |
|
| 271 | 264 | ||
| 272 | 265 | /* Store host `src` to guest register `guest`. */ |
|
| 273 | 266 | static void emit_store_guest(struct emitter *e, u32 guest, enum x86reg src) { |
|
| 274 | 267 | if (guest == 0) |
| 276 | 269 | u8 host = host_of(e, guest); |
|
| 277 | 270 | if (host) { |
|
| 278 | 271 | emit_mov_rr(e, host, src); |
|
| 279 | 272 | return; |
|
| 280 | 273 | } |
|
| 281 | - | emit_r12_disp(e, 0x89, src, guest * 8); |
|
| 274 | + | emit_reg_disp(e, 0x89, src, guest * 8); |
|
| 282 | 275 | } |
|
| 283 | 276 | ||
| 284 | 277 | /* Every host register, as a mask for emit_transfer(). */ |
|
| 285 | 278 | #define ALL_HOSTS 0xFFFFu |
|
| 286 | 279 | ||
| 287 | 280 | /* Store (`store`) or load the held guest registers whose host registers are |
|
| 288 | 281 | * in `mask` to or from the register file. */ |
|
| 289 | 282 | static void emit_transfer(struct emitter *e, u32 mask, bool store) { |
|
| 290 | 283 | for (u32 g = 1; g < 32; g++) { |
|
| 291 | 284 | if (guest_host[g] && (mask >> guest_host[g]) & 1) |
|
| 292 | - | emit_r12_disp(e, store ? 0x89 : 0x8B, guest_host[g], g * 8); |
|
| 285 | + | emit_reg_disp(e, store ? 0x89 : 0x8B, guest_host[g], g * 8); |
|
| 293 | 286 | } |
|
| 294 | 287 | } |
|
| 295 | 288 | ||
| 296 | 289 | /* Make the register file hold every guest register before an instruction |
|
| 297 | 290 | * that addresses it directly or clobbers host registers. */ |
| 313 | 306 | emit_u64(e, imm); |
|
| 314 | 307 | } |
|
| 315 | 308 | ||
| 316 | 309 | /* mov reg, imm32 (sign-extended to 64 via mov r/m64, imm32) */ |
|
| 317 | 310 | static void emit_mov_imm32_sx(struct emitter *e, enum x86reg dst, i32 imm) { |
|
| 311 | + | if (imm >= 0) { |
|
| 312 | + | /* A 32-bit destination clears the high word. */ |
|
| 313 | + | if (dst >= 8) |
|
| 314 | + | emit_u8(e, rex(false, false, false, true)); |
|
| 315 | + | emit_u8(e, 0xB8 | (dst & 7)); |
|
| 316 | + | emit_u32(e, (u32)imm); |
|
| 317 | + | return; |
|
| 318 | + | } |
|
| 318 | 319 | emit_u8(e, rex(true, false, false, dst >= 8)); |
|
| 319 | 320 | emit_u8(e, 0xC7); |
|
| 320 | 321 | emit_u8(e, modrm(3, 0, dst)); |
|
| 321 | 322 | emit_u32(e, (u32)imm); |
|
| 322 | 323 | } |
|
| 323 | 324 | ||
| 325 | + | /* Set a guest register to a signed 32-bit constant. */ |
|
| 326 | + | static void emit_guest_imm32(struct emitter *e, u32 guest, i32 imm) { |
|
| 327 | + | u8 host = host_of(e, guest); |
|
| 328 | + | if (host) { |
|
| 329 | + | emit_mov_imm32_sx(e, host, imm); |
|
| 330 | + | return; |
|
| 331 | + | } |
|
| 332 | + | emit_u8(e, 0x49); |
|
| 333 | + | emit_u8(e, 0xC7); |
|
| 334 | + | emit_reg_address(e, 0, guest * 8); |
|
| 335 | + | emit_u32(e, (u32)imm); |
|
| 336 | + | } |
|
| 337 | + | ||
| 324 | 338 | /* ---------- Jump patching ---------------------------------------------- */ |
|
| 325 | 339 | ||
| 326 | 340 | /* Emit jnz rel32 (0F 85), return position for patching. */ |
|
| 327 | 341 | static u32 emit_jnz_placeholder(struct emitter *e) { |
|
| 328 | 342 | u32 pos = e->pos; |
| 386 | 400 | emit_u8(e, 0xC7); |
|
| 387 | 401 | emit_u8(e, modrm(0, 0, HIO)); |
|
| 388 | 402 | emit_u32(e, pc); |
|
| 389 | 403 | } |
|
| 390 | 404 | ||
| 405 | + | /* Charge the instructions on this path and set the budget flags. */ |
|
| 406 | + | static void emit_charge(struct emitter *e, u32 insns) { |
|
| 407 | + | if (insns == 0) |
|
| 408 | + | return; |
|
| 409 | + | bool small = insns <= INT8_MAX; |
|
| 410 | + | emit_u8(e, 0x49); |
|
| 411 | + | emit_u8(e, small ? 0x83 : 0x81); |
|
| 412 | + | emit_u8(e, modrm(3, 5, HBUDGET)); |
|
| 413 | + | if (small) |
|
| 414 | + | emit_u8(e, (u8)insns); |
|
| 415 | + | else |
|
| 416 | + | emit_u32(e, insns); |
|
| 417 | + | } |
|
| 418 | + | ||
| 391 | 419 | /* mov eax, reason; jmp exit_stub. While spilled, the register |
|
| 392 | 420 | * file is already current and the host registers may be stale. */ |
|
| 393 | 421 | static void emit_exit(struct emitter *e, u32 exit_reason) { |
|
| 394 | 422 | emit_u8(e, 0xB8); |
|
| 395 | 423 | emit_u32(e, exit_reason); /* mov eax, reason */ |
| 398 | 426 | ||
| 399 | 427 | /* Store next-PC to *r14 and leave the cache with `reason`. */ |
|
| 400 | 428 | static void emit_block_exit( |
|
| 401 | 429 | struct emitter *e, enum jit_exit reason, u32 next_pc |
|
| 402 | 430 | ) { |
|
| 431 | + | emit_charge(e, e->insns); |
|
| 403 | 432 | emit_store_pc_imm(e, next_pc); |
|
| 404 | 433 | emit_exit(e, (u32)reason); |
|
| 405 | 434 | } |
|
| 406 | 435 | ||
| 407 | - | /* Jump to the guest block at the constant PC `target`. Emits a jmp rel32 |
|
| 408 | - | * that initially falls into a stub returning JIT_EXIT_CHAIN with the address |
|
| 409 | - | * of its rel32 field in io.patch; see jit_patch_chain. */ |
|
| 410 | - | static void emit_chain_exit(struct emitter *e, u32 target) { |
|
| 411 | - | /* A linked jump enters the target with the guest registers held. */ |
|
| 412 | - | bool spilled = e->spilled; |
|
| 413 | - | if (spilled) |
|
| 414 | - | emit_reload(e); |
|
| 415 | - | ||
| 416 | - | u32 site = e->pos; |
|
| 417 | - | emit_u8(e, 0xE9); |
|
| 418 | - | emit_u32(e, 0); /* jmp .stub (next instruction) */ |
|
| 419 | - | ||
| 436 | + | /* Leave for C to link the rel32 branch at `site` to `target`. */ |
|
| 437 | + | static void emit_chain_stub(struct emitter *e, u32 target, u32 site) { |
|
| 420 | 438 | emit_store_pc_imm(e, target); |
|
| 421 | 439 | ||
| 422 | - | /* lea rax, [rip + (site + 1) - next] */ |
|
| 440 | + | /* lea rax, [rip + site - next] */ |
|
| 423 | 441 | emit_u8(e, 0x48); |
|
| 424 | 442 | emit_u8(e, 0x8D); |
|
| 425 | 443 | emit_u8(e, modrm(0, X_RAX, 5)); |
|
| 426 | - | emit_u32(e, (site + 1) - (e->pos + 4)); |
|
| 444 | + | emit_u32(e, site - (e->pos + 4)); |
|
| 427 | 445 | ||
| 428 | 446 | /* mov [r14 + IO_PATCH], rax */ |
|
| 429 | 447 | emit_u8(e, 0x49); |
|
| 430 | 448 | emit_u8(e, 0x89); |
|
| 431 | 449 | emit_u8(e, modrm(1, X_RAX, HIO)); |
|
| 432 | 450 | emit_u8(e, IO_PATCH); |
|
| 433 | 451 | ||
| 434 | 452 | emit_exit(e, JIT_EXIT_CHAIN); |
|
| 453 | + | } |
|
| 454 | + | ||
| 455 | + | /* Jump to the guest block at `target` with the budget flags set. */ |
|
| 456 | + | static void emit_link(struct emitter *e, u32 target) { |
|
| 457 | + | u32 site = emit_jmp_placeholder(e); |
|
| 458 | + | emit_chain_stub(e, target, site + 1); |
|
| 459 | + | } |
|
| 460 | + | ||
| 461 | + | /* Charge the block and jump to `target` with all held registers loaded. */ |
|
| 462 | + | static void emit_chain_exit(struct emitter *e, u32 target) { |
|
| 463 | + | bool spilled = e->spilled; |
|
| 464 | + | if (spilled) |
|
| 465 | + | emit_reload(e); |
|
| 466 | + | emit_charge(e, e->insns); |
|
| 467 | + | emit_link(e, target); |
|
| 435 | 468 | e->spilled = spilled; |
|
| 436 | 469 | } |
|
| 437 | 470 | ||
| 471 | + | /* A native call frame contains the host continuation and the guest link. */ |
|
| 472 | + | #define JIT_CALL_LINK_SIZE 8 |
|
| 473 | + | #define JIT_CALL_STACK_SIZE 4096 |
|
| 474 | + | #define IO_STACK_BASE ((u8)offsetof(struct jit_io, stack_base)) |
|
| 475 | + | #define IO_STACK_FLOOR ((u8)offsetof(struct jit_io, stack_floor)) |
|
| 476 | + | ||
| 477 | + | /* Return from a native call and drop its guest link. */ |
|
| 478 | + | static void emit_native_ret(struct emitter *e) { |
|
| 479 | + | emit_u8(e, 0xC2); /* ret imm16 */ |
|
| 480 | + | emit_u8(e, JIT_CALL_LINK_SIZE); |
|
| 481 | + | emit_u8(e, 0); |
|
| 482 | + | } |
|
| 483 | + | ||
| 484 | + | /* On an unwind, continue returning through the native call stack. */ |
|
| 485 | + | static void emit_unwind_check(struct emitter *e) { |
|
| 486 | + | emit_u8(e, 0x0F); |
|
| 487 | + | emit_u8(e, 0x80); /* jo exit_unwind */ |
|
| 488 | + | emit_u32(e, (u32)(e->jit->exit_unwind - (e->dest + e->pos + 4))); |
|
| 489 | + | } |
|
| 490 | + | ||
| 491 | + | /* Translate a direct call with guest link `next`. If the host call stack is |
|
| 492 | + | * full, charge the block and jump without adding a frame. */ |
|
| 493 | + | static void emit_native_call(struct emitter *e, u32 next, u32 target) { |
|
| 494 | + | /* cmp rsp, [r14 + IO_STACK_FLOOR]; jbe .full */ |
|
| 495 | + | emit_u8(e, 0x49); |
|
| 496 | + | emit_u8(e, 0x3B); |
|
| 497 | + | emit_u8(e, modrm(1, X_RSP, HIO)); |
|
| 498 | + | emit_u8(e, IO_STACK_FLOOR); |
|
| 499 | + | u32 full = e->pos; |
|
| 500 | + | emit_u8(e, 0x0F); |
|
| 501 | + | emit_u8(e, 0x86); |
|
| 502 | + | emit_u32(e, 0); |
|
| 503 | + | ||
| 504 | + | emit_u8(e, 0x68); /* push guest link */ |
|
| 505 | + | emit_u32(e, next); |
|
| 506 | + | emit_charge(e, e->insns); |
|
| 507 | + | u32 call = e->pos; |
|
| 508 | + | emit_u8(e, 0xE8); |
|
| 509 | + | emit_u32(e, 0); |
|
| 510 | + | ||
| 511 | + | /* A guest return jumps to its continuation. An unwind falls through. */ |
|
| 512 | + | u32 resume = e->pos; |
|
| 513 | + | emit_u8(e, 0x0F); |
|
| 514 | + | emit_u8(e, 0x81); /* jno */ |
|
| 515 | + | emit_u32(e, 0); |
|
| 516 | + | emit_jmp_abs(e, e->jit->exit_unwind); |
|
| 517 | + | patch_jcc(e, resume, e->pos); |
|
| 518 | + | emit_chain_stub(e, next, resume + 2); |
|
| 519 | + | ||
| 520 | + | patch_jmp(e, call, e->pos); |
|
| 521 | + | emit_chain_stub(e, target, call + 1); |
|
| 522 | + | ||
| 523 | + | patch_jcc(e, full, e->pos); |
|
| 524 | + | emit_chain_exit(e, target); |
|
| 525 | + | } |
|
| 526 | + | ||
| 438 | 527 | /* Offset of the jump cache from the exchange area in r14. */ |
|
| 439 | 528 | #define IO_JUMP_CACHE \ |
|
| 440 | 529 | ((u32)(offsetof(struct jit_state, jump_cache) - \ |
|
| 441 | 530 | offsetof(struct jit_state, io))) |
|
| 442 | 531 | ||
| 443 | - | /* Jump to the guest PC in rax (already even). Stores it to io.pc, probes the |
|
| 444 | - | * jump cache and jumps to the cached block on a hit, else exits to C. |
|
| 445 | - | * Clobbers only rax and rcx, since the target expects held registers. */ |
|
| 532 | + | /* Jump to the guest PC in rax (already even). On a cache miss, store the PC |
|
| 533 | + | * in io.pc and return to C. Preserve the held guest registers for the target. |
|
| 534 | + | */ |
|
| 446 | 535 | static void emit_indirect_jump(struct emitter *e) { |
|
| 447 | 536 | if (e->spilled) |
|
| 448 | 537 | emit_reload(e); |
|
| 449 | 538 | ||
| 450 | - | /* mov dword [r14], eax */ |
|
| 451 | - | emit_u8(e, 0x41); |
|
| 452 | - | emit_u8(e, 0x89); |
|
| 453 | - | emit_u8(e, modrm(0, X_RAX, HIO)); |
|
| 454 | - | ||
| 455 | 539 | /* mov ecx, eax; and ecx, mask << 2; shl ecx, 2 -- byte offset of the |
|
| 456 | 540 | * entry, ((pc >> 2) & mask) * sizeof(struct jit_jump_entry). */ |
|
| 457 | 541 | emit_u8(e, 0x89); |
|
| 458 | 542 | emit_u8(e, modrm(3, X_RAX, X_RCX)); |
|
| 459 | 543 | emit_u8(e, 0x81); |
| 470 | 554 | emit_u8(e, sib(0, X_RCX, HIO)); |
|
| 471 | 555 | emit_u32(e, IO_JUMP_CACHE); |
|
| 472 | 556 | ||
| 473 | 557 | u32 jne_pos = emit_jnz_placeholder(e); |
|
| 474 | 558 | ||
| 559 | + | emit_charge(e, e->insns); |
|
| 560 | + | ||
| 475 | 561 | /* jmp [r14 + rcx + IO_JUMP_CACHE + offsetof(code)] */ |
|
| 476 | 562 | emit_u8(e, 0x41); |
|
| 477 | 563 | emit_u8(e, 0xFF); |
|
| 478 | 564 | emit_u8(e, modrm(2, 4, 4)); |
|
| 479 | 565 | emit_u8(e, sib(0, X_RCX, HIO)); |
|
| 480 | 566 | emit_u32(e, IO_JUMP_CACHE + offsetof(struct jit_jump_entry, code)); |
|
| 481 | 567 | ||
| 482 | - | /* Miss: io.pc is set, let C dispatch. */ |
|
| 568 | + | /* Miss: save the target PC for C. */ |
|
| 483 | 569 | patch_jcc(e, jne_pos, e->pos); |
|
| 570 | + | emit_charge(e, e->insns); |
|
| 571 | + | emit_u8(e, 0x41); |
|
| 572 | + | emit_u8(e, 0x89); |
|
| 573 | + | emit_u8(e, modrm(0, X_RAX, HIO)); |
|
| 484 | 574 | emit_exit(e, JIT_EXIT_BRANCH); |
|
| 485 | 575 | } |
|
| 486 | 576 | ||
| 487 | 577 | /* ---------- Per-instruction translation helpers ------------------------ */ |
|
| 488 | 578 |
| 584 | 674 | emit_u8(e, 0x83); |
|
| 585 | 675 | emit_u8(e, modrm(3, 7, X_RCX)); |
|
| 586 | 676 | emit_u8(e, 0xFF); |
|
| 587 | 677 | } |
|
| 588 | 678 | ||
| 589 | - | /* mov rax, r11 (via: REX.WR mov rax, r11) */ |
|
| 590 | - | static void emit_mov_rax_r11(struct emitter *e) { |
|
| 591 | - | emit_u8(e, 0x4C); |
|
| 592 | - | emit_u8(e, 0x89); |
|
| 593 | - | emit_u8(e, modrm(3, X_R11 & 7, X_RAX)); |
|
| 594 | - | } |
|
| 595 | - | ||
| 596 | 679 | /* mov rax, rdx (64-bit) */ |
|
| 597 | 680 | static void emit_mov_rax_rdx(struct emitter *e) { |
|
| 598 | 681 | emit_rexw(e); |
|
| 599 | 682 | emit_u8(e, 0x89); |
|
| 600 | 683 | emit_u8(e, modrm(3, X_RDX, X_RAX)); |
| 647 | 730 | emit_u8(e, 0x0F); |
|
| 648 | 731 | emit_u8(e, 0xB6); |
|
| 649 | 732 | emit_u8(e, modrm(3, X_RAX, X_RAX)); |
|
| 650 | 733 | } |
|
| 651 | 734 | ||
| 652 | - | /* and ecx, imm8 (for masking shift amounts). */ |
|
| 653 | - | static void emit_and_ecx_imm8(struct emitter *e, u8 mask) { |
|
| 654 | - | emit_u8(e, 0x83); |
|
| 655 | - | emit_u8(e, modrm(3, 4, X_RCX)); |
|
| 656 | - | emit_u8(e, mask); |
|
| 657 | - | } |
|
| 658 | - | ||
| 659 | 735 | /* ---------- Division helpers ------------------------------------------- */ |
|
| 660 | 736 | ||
| 661 | 737 | /* |
|
| 662 | 738 | * 64-bit div/rem of rax by rcx, leaving the result in rax, with the RISC-V |
|
| 663 | 739 | * results where x86 division would trap: |
| 667 | 743 | * |
|
| 668 | 744 | * `is_signed`: signed division. |
|
| 669 | 745 | * `is_rem`: return the remainder instead of the quotient. |
|
| 670 | 746 | */ |
|
| 671 | 747 | static void emit_div64(struct emitter *e, bool is_signed, bool is_rem) { |
|
| 672 | - | /* For rem: save dividend to r11 (needed if divisor==0). */ |
|
| 673 | - | if (is_rem) { |
|
| 674 | - | /* mov r11, rax */ |
|
| 675 | - | emit_u8(e, rex(true, false, false, true)); |
|
| 676 | - | emit_u8(e, 0x89); |
|
| 677 | - | emit_u8(e, modrm(3, X_RAX, X_R11 & 7)); |
|
| 678 | - | } |
|
| 679 | - | ||
| 680 | 748 | emit_test_rcx(e); |
|
| 681 | 749 | u32 jnz = emit_jnz_placeholder(e); |
|
| 682 | 750 | ||
| 683 | 751 | /* Divisor == 0. */ |
|
| 684 | - | if (is_rem) { |
|
| 685 | - | emit_mov_rax_r11(e); /* result = dividend */ |
|
| 686 | - | } else { |
|
| 752 | + | if (!is_rem) { |
|
| 687 | 753 | emit_mov_imm32_sx(e, HTMP1, -1); /* result = -1 (all ones) */ |
|
| 688 | 754 | } |
|
| 689 | 755 | u32 jmp_end1 = emit_jmp_placeholder(e); |
|
| 690 | 756 | ||
| 691 | 757 | /* .nonzero: */ |
| 731 | 797 | } |
|
| 732 | 798 | } |
|
| 733 | 799 | ||
| 734 | 800 | /* Same pattern for 32-bit division (W-suffix instructions). */ |
|
| 735 | 801 | static void emit_div32(struct emitter *e, bool is_signed, bool is_rem) { |
|
| 736 | - | /* For rem: save dividend in edx. */ |
|
| 737 | - | if (is_rem) { |
|
| 738 | - | /* mov edx, eax */ |
|
| 739 | - | emit_u8(e, 0x89); |
|
| 740 | - | emit_u8(e, modrm(3, X_RAX, X_RDX)); |
|
| 741 | - | } |
|
| 742 | - | ||
| 743 | 802 | emit_test_ecx(e); |
|
| 744 | 803 | u32 jnz = emit_jnz_placeholder(e); |
|
| 745 | 804 | ||
| 746 | 805 | /* Divisor == 0. */ |
|
| 747 | - | if (is_rem) { |
|
| 748 | - | /* mov eax, edx (result = dividend) */ |
|
| 749 | - | emit_u8(e, 0x89); |
|
| 750 | - | emit_u8(e, modrm(3, X_RDX, X_RAX)); |
|
| 751 | - | } else { |
|
| 806 | + | if (!is_rem) { |
|
| 752 | 807 | /* mov eax, -1 */ |
|
| 753 | 808 | emit_u8(e, 0xB8); |
|
| 754 | 809 | emit_u32(e, 0xFFFFFFFF); |
|
| 755 | 810 | } |
|
| 756 | 811 | u32 jmp_end1 = emit_jmp_placeholder(e); |
| 773 | 828 | jmp_end2 = emit_jmp_placeholder(e); |
|
| 774 | 829 | } |
|
| 775 | 830 | ||
| 776 | 831 | /* .safe: */ |
|
| 777 | 832 | u32 safe = e->pos; |
|
| 778 | - | if (is_rem) { |
|
| 779 | - | /* Restore dividend to eax from edx. */ |
|
| 780 | - | emit_u8(e, 0x89); |
|
| 781 | - | emit_u8(e, modrm(3, X_RDX, X_RAX)); |
|
| 782 | - | } |
|
| 783 | 833 | if (is_signed) { |
|
| 784 | 834 | emit_u8(e, 0x99); /* cdq */ |
|
| 785 | 835 | emit_u8(e, 0xF7); |
|
| 786 | 836 | emit_u8(e, modrm(3, 7, X_RCX)); /* idiv ecx */ |
|
| 787 | 837 | } else { |
| 965 | 1015 | emit_load_rs1(e, ZERO); |
|
| 966 | 1016 | } else { |
|
| 967 | 1017 | emit_u8(e, rex(op != BM_ZEXTH, false, false, true)); |
|
| 968 | 1018 | emit_u8(e, 0x0f); |
|
| 969 | 1019 | emit_u8(e, op == BM_SEXTB ? 0xbe : op == BM_SEXTH ? 0xbf : 0xb7); |
|
| 970 | - | emit_r12_address(e, HTMP1, ins.r.rs1 * 8); |
|
| 1020 | + | emit_reg_address(e, HTMP1, ins.r.rs1 * 8); |
|
| 971 | 1021 | } |
|
| 972 | 1022 | emit_store_rd(e, ins.r.rd); |
|
| 973 | 1023 | return; |
|
| 974 | 1024 | } |
|
| 975 | 1025 | if (op == BM_ORCB) { |
| 1020 | 1070 | } |
|
| 1021 | 1071 | emit_u8(e, avx ? 0xc4 : 0xf3); |
|
| 1022 | 1072 | emit_u8(e, avx ? 0xc1 : 0x41); |
|
| 1023 | 1073 | emit_u8(e, avx ? 0x7a : 0x0f); |
|
| 1024 | 1074 | emit_u8(e, 0x7e); |
|
| 1025 | - | emit_r12_address(e, X_RAX, ins.r.rs1 * 8); /* vmovq xmm0, [r12 + off] */ |
|
| 1075 | + | emit_reg_address( |
|
| 1076 | + | e, X_RAX, ins.r.rs1 * 8 |
|
| 1077 | + | ); /* vmovq xmm0, [r14 + disp] */ |
|
| 1026 | 1078 | for (u32 i = 0; i < sizeof(masks[0]); i++) |
|
| 1027 | 1079 | emit_u8(e, masks[avx][i]); |
|
| 1028 | 1080 | emit_u8(e, avx ? 0xc4 : 0x66); |
|
| 1029 | 1081 | emit_u8(e, avx ? 0xc1 : 0x41); |
|
| 1030 | 1082 | emit_u8(e, avx ? 0x79 : 0x0f); |
|
| 1031 | 1083 | emit_u8(e, 0xd6); |
|
| 1032 | - | emit_r12_address(e, X_RAX, ins.r.rd * 8); /* vmovq [r12 + off], xmm0 */ |
|
| 1084 | + | emit_reg_address(e, X_RAX, ins.r.rd * 8); /* vmovq [r14 + disp], xmm0 */ |
|
| 1033 | 1085 | return; |
|
| 1034 | 1086 | } |
|
| 1035 | 1087 | bool word = op == BM_ROLW || op == BM_RORW || op == BM_RORIW; |
|
| 1036 | 1088 | bool word_source = op == BM_ADDUW || op == BM_SH1ADDUW || |
|
| 1037 | 1089 | op == BM_SH2ADDUW || op == BM_SH3ADDUW || |
| 1238 | 1290 | break; |
|
| 1239 | 1291 | } |
|
| 1240 | 1292 | emit_store_rd(e, ins.r.rd); |
|
| 1241 | 1293 | } |
|
| 1242 | 1294 | ||
| 1243 | - | /* Give back the budget of the instructions after the first `insns` of the |
|
| 1244 | - | * block, on a path that leaves after them: add r15, refund. The refund is |
|
| 1245 | - | * patched in once the length of the block is known. */ |
|
| 1246 | - | static void emit_refund(struct emitter *e, u32 insns) { |
|
| 1247 | - | emit_u8(e, 0x49); |
|
| 1248 | - | emit_u8(e, 0x81); |
|
| 1249 | - | emit_u8(e, modrm(3, 0, HBUDGET)); |
|
| 1250 | - | e->refunds[e->refunds_len] = e->pos; |
|
| 1251 | - | e->consumed[e->refunds_len++] = insns; |
|
| 1252 | - | emit_u32(e, 0); |
|
| 1253 | - | } |
|
| 1254 | - | ||
| 1255 | 1295 | /* ---------- Vector translation ----------------------------------------- */ |
|
| 1256 | 1296 | ||
| 1257 | 1297 | /* Return a failed memory operation to the interpreter before its side effects |
|
| 1258 | 1298 | * when the jcc `condition` (the second opcode byte) holds. */ |
|
| 1259 | 1299 | static void emit_fault_if(struct emitter *e, u32 pc, u8 condition) { |
|
| 1260 | - | u32 bad = e->pos; |
|
| 1300 | + | u32 ok = e->pos; |
|
| 1261 | 1301 | emit_u8(e, 0x0f); |
|
| 1262 | - | emit_u8(e, condition); |
|
| 1302 | + | emit_u8(e, condition ^ 1); |
|
| 1263 | 1303 | emit_u32(e, 0); |
|
| 1264 | - | u32 ok = emit_jmp_placeholder(e); |
|
| 1265 | - | patch_jcc(e, bad, e->pos); |
|
| 1266 | - | emit_refund(e, e->insns); |
|
| 1267 | 1304 | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 1268 | - | patch_jmp(e, ok, e->pos); |
|
| 1305 | + | patch_jcc(e, ok, e->pos); |
|
| 1269 | 1306 | } |
|
| 1270 | 1307 | ||
| 1271 | 1308 | /* Load the architectural vector state into rax. */ |
|
| 1272 | 1309 | static void emit_vector_base(struct emitter *e) { |
|
| 1273 | 1310 | emit_u8(e, 0x49); |
| 1889 | 1926 | bad[i] = emit_jne_placeholder(e); |
|
| 1890 | 1927 | } |
|
| 1891 | 1928 | u32 ok = emit_jmp_placeholder(e); |
|
| 1892 | 1929 | for (u32 i = 0; i < 3; i++) |
|
| 1893 | 1930 | patch_jcc(e, bad[i], e->pos); |
|
| 1894 | - | emit_u8(e, 0x49); |
|
| 1895 | - | emit_u8(e, 0x83); |
|
| 1896 | - | emit_u8(e, modrm(3, 5, HBUDGET)); |
|
| 1897 | - | emit_u8(e, 1); |
|
| 1898 | - | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 1931 | + | emit_charge(e, 1); |
|
| 1932 | + | emit_store_pc_imm(e, pc); |
|
| 1933 | + | emit_exit(e, JIT_EXIT_FAULT); |
|
| 1899 | 1934 | patch_jmp(e, ok, e->pos); |
|
| 1900 | 1935 | } |
|
| 1901 | 1936 | ||
| 1902 | 1937 | /* Set the flags as cmp rs1, rs2 would, using held registers in place. */ |
|
| 1903 | 1938 | static void emit_compare(struct emitter *e, u32 rs1, u32 rs2) { |
| 1915 | 1950 | emit_u8(e, rex(true, b >= 8, false, a >= 8)); |
|
| 1916 | 1951 | emit_u8(e, 0x39); |
|
| 1917 | 1952 | emit_u8(e, modrm(3, b, a)); |
|
| 1918 | 1953 | return; |
|
| 1919 | 1954 | } |
|
| 1920 | - | /* cmp a, [r12 + rs2 * 8] */ |
|
| 1921 | - | emit_r12_disp(e, 0x3B, a, rs2 * 8); |
|
| 1955 | + | /* cmp a, [guest register file + rs2 * 8] */ |
|
| 1956 | + | emit_reg_disp(e, 0x3B, a, rs2 * 8); |
|
| 1922 | 1957 | } |
|
| 1923 | 1958 | ||
| 1924 | 1959 | /* Whether the block already holds the instruction at `pc`. */ |
|
| 1925 | 1960 | static bool translated(struct emitter *e, u32 pc) { |
|
| 1926 | 1961 | for (u32 i = 0; i < e->runs; i++) { |
| 1938 | 1973 | e->run_start[e->runs] = target; |
|
| 1939 | 1974 | e->runs++; |
|
| 1940 | 1975 | e->next_pc = target; |
|
| 1941 | 1976 | } |
|
| 1942 | 1977 | ||
| 1943 | - | /* Jump to the constant `target`, linking `next` in `rd`. Continues the |
|
| 1944 | - | * block at the target unless that would loop, and returns whether the block |
|
| 1945 | - | * ends. */ |
|
| 1978 | + | /* Jump to the constant `target`, linking `next` in `rd`. End the block for |
|
| 1979 | + | * a native call or when the target cannot be translated in this block. */ |
|
| 1946 | 1980 | static bool emit_direct_jump(struct emitter *e, u32 rd, u32 next, u32 target) { |
|
| 1947 | 1981 | if (rd != 0) { |
|
| 1948 | - | emit_mov_imm64(e, HTMP1, (u64)next); |
|
| 1949 | - | emit_store_guest(e, rd, HTMP1); |
|
| 1982 | + | if (next <= INT32_MAX) { |
|
| 1983 | + | emit_guest_imm32(e, rd, (i32)next); |
|
| 1984 | + | } else { |
|
| 1985 | + | emit_mov_imm64(e, HTMP1, (u64)next); |
|
| 1986 | + | emit_store_guest(e, rd, HTMP1); |
|
| 1987 | + | } |
|
| 1988 | + | } |
|
| 1989 | + | if (rd == RA && next <= INT32_MAX) { |
|
| 1990 | + | emit_native_call(e, next, target); |
|
| 1991 | + | return true; |
|
| 1950 | 1992 | } |
|
| 1951 | 1993 | if (e->runs > JIT_MAX_FOLLOWS || translated(e, target)) { |
|
| 1952 | 1994 | emit_chain_exit(e, target); |
|
| 1953 | 1995 | return true; |
|
| 1954 | 1996 | } |
|
| 1955 | - | /* A return to `next` can then be checked in place. */ |
|
| 1956 | - | if (rd == RA && next <= INT32_MAX) { |
|
| 1957 | - | e->ra_known = true; |
|
| 1958 | - | e->ra_value = next; |
|
| 1959 | - | } |
|
| 1960 | 1997 | follow(e, next, target); |
|
| 1961 | 1998 | return false; |
|
| 1962 | 1999 | } |
|
| 1963 | 2000 | ||
| 1964 | - | /* Return to ra: leave the program when it is zero, else jump to it. */ |
|
| 2001 | + | /* Return to ra. Use ret when ra matches the top native frame. */ |
|
| 1965 | 2002 | static void emit_ret(struct emitter *e) { |
|
| 1966 | 2003 | emit_load_guest(e, HTMP1, RA); |
|
| 1967 | 2004 | ||
| 2005 | + | /* cmp rax, [rsp + 8]; jne .fallback */ |
|
| 2006 | + | emit_rexw(e); |
|
| 2007 | + | emit_u8(e, 0x3B); |
|
| 2008 | + | emit_u8(e, modrm(1, X_RAX, X_RSP)); |
|
| 2009 | + | emit_u8(e, sib(0, X_RSP, X_RSP)); |
|
| 2010 | + | emit_u8(e, 8); |
|
| 2011 | + | u32 mismatch = emit_jnz_placeholder(e); |
|
| 2012 | + | /* The budget was positive at entry, so this subtraction clears OF. */ |
|
| 2013 | + | emit_charge(e, e->insns); |
|
| 2014 | + | emit_native_ret(e); |
|
| 2015 | + | patch_jcc(e, mismatch, e->pos); |
|
| 2016 | + | ||
| 1968 | 2017 | /* test rax, rax */ |
|
| 1969 | 2018 | emit_rexw(e); |
|
| 1970 | 2019 | emit_u8(e, 0x85); |
|
| 1971 | 2020 | emit_u8(e, modrm(3, X_RAX, X_RAX)); |
|
| 1972 | 2021 |
| 1983 | 2032 | emit_u8(e, 0xFE); /* and rax, ~1 */ |
|
| 1984 | 2033 | ||
| 1985 | 2034 | emit_indirect_jump(e); |
|
| 1986 | 2035 | } |
|
| 1987 | 2036 | ||
| 1988 | - | /* Emit the side exits recorded for taken branches and checked returns. */ |
|
| 2037 | + | /* Emit the side exits recorded for conditional branches. */ |
|
| 1989 | 2038 | static void emit_side_exits(struct emitter *e) { |
|
| 1990 | 2039 | /* Branches are never translated spilled. */ |
|
| 1991 | 2040 | e->spilled = false; |
|
| 1992 | 2041 | for (u32 i = 0; i < e->sides_len; i++) { |
|
| 1993 | 2042 | patch_jcc(e, e->sides[i].jcc, e->pos); |
|
| 1994 | - | emit_refund(e, e->sides[i].insns); |
|
| 1995 | - | if (e->sides[i].ret) |
|
| 1996 | - | emit_ret(e); |
|
| 1997 | - | else |
|
| 1998 | - | emit_chain_exit(e, e->sides[i].target); |
|
| 2043 | + | e->insns = e->sides[i].insns; |
|
| 2044 | + | emit_chain_exit(e, e->sides[i].target); |
|
| 1999 | 2045 | } |
|
| 2000 | 2046 | } |
|
| 2001 | 2047 | ||
| 2002 | - | /* Leave the block through a side exit when the jcc `cc` (its second opcode |
|
| 2003 | - | * byte) holds: to `target`, or by returning to ra when `ret` is set. */ |
|
| 2004 | - | static void emit_side_exit(struct emitter *e, u8 cc, u32 target, bool ret) { |
|
| 2048 | + | /* Leave the block for `target` when the jcc `cc` condition is true. */ |
|
| 2049 | + | static void emit_side_exit(struct emitter *e, u8 cc, u32 target) { |
|
| 2005 | 2050 | e->sides[e->sides_len].jcc = e->pos; |
|
| 2006 | 2051 | e->sides[e->sides_len].target = target; |
|
| 2007 | 2052 | e->sides[e->sides_len].insns = e->insns; |
|
| 2008 | - | e->sides[e->sides_len].ret = ret; |
|
| 2009 | 2053 | e->sides_len++; |
|
| 2010 | 2054 | emit_u8(e, 0x0F); |
|
| 2011 | 2055 | emit_u8(e, cc); |
|
| 2012 | 2056 | emit_u32(e, 0); |
|
| 2013 | 2057 | } |
| 2027 | 2071 | * the r/m operand, held or in the register file. `src` is not x0. */ |
|
| 2028 | 2072 | static void emit_op_guest( |
|
| 2029 | 2073 | struct emitter *e, bool wide, bool escape, u8 opcode, u8 dst, u32 src |
|
| 2030 | 2074 | ) { |
|
| 2031 | 2075 | u8 host = host_of(e, src); |
|
| 2032 | - | /* The register file is addressed through r12, which needs REX.B. */ |
|
| 2076 | + | /* The register file is addressed through r14, which needs REX.B. */ |
|
| 2033 | 2077 | emit_u8(e, rex(wide, dst >= 8, false, host ? host >= 8 : true)); |
|
| 2034 | 2078 | if (escape) |
|
| 2035 | 2079 | emit_u8(e, 0x0F); |
|
| 2036 | 2080 | emit_u8(e, opcode); |
|
| 2037 | 2081 | if (host) |
|
| 2038 | 2082 | emit_u8(e, modrm(3, dst, host)); |
|
| 2039 | 2083 | else |
|
| 2040 | - | emit_r12_address(e, dst, src * 8); |
|
| 2084 | + | emit_reg_address(e, dst, src * 8); |
|
| 2041 | 2085 | } |
|
| 2042 | 2086 | ||
| 2043 | 2087 | /* Emit the group-1 operation `ext` (0 add, 1 or, 4 and, 5 sub, 6 xor, 7 cmp) |
|
| 2044 | 2088 | * of `dst` with a sign-extended immediate. */ |
|
| 2045 | 2089 | static void emit_op_imm(struct emitter *e, bool wide, u8 ext, u8 dst, i32 imm) { |
| 2114 | 2158 | u8 d = dest_of(e, rd); |
|
| 2115 | 2159 | u8 src = host_of(e, rs1); |
|
| 2116 | 2160 | ||
| 2117 | 2161 | switch (ins.i.opcode) { |
|
| 2118 | 2162 | case OP_LUI: |
|
| 2119 | - | emit_mov_imm32_sx(e, d, (i32)(ins.u.imm_31_12 << 12)); |
|
| 2120 | - | break; |
|
| 2163 | + | emit_guest_imm32(e, rd, (i32)(ins.u.imm_31_12 << 12)); |
|
| 2164 | + | return; |
|
| 2121 | 2165 | ||
| 2122 | 2166 | case OP_AUIPC: { |
|
| 2123 | 2167 | i64 value = (i64)pc + (i64)(i32)(ins.u.imm_31_12 << 12); |
|
| 2124 | - | if (value >= INT32_MIN && value <= INT32_MAX) |
|
| 2125 | - | emit_mov_imm32_sx(e, d, (i32)value); |
|
| 2126 | - | else |
|
| 2127 | - | emit_mov_imm64(e, d, (u64)value); |
|
| 2168 | + | if (value >= INT32_MIN && value <= INT32_MAX) { |
|
| 2169 | + | emit_guest_imm32(e, rd, (i32)value); |
|
| 2170 | + | return; |
|
| 2171 | + | } |
|
| 2172 | + | emit_mov_imm64(e, d, (u64)value); |
|
| 2128 | 2173 | break; |
|
| 2129 | 2174 | } |
|
| 2130 | 2175 | ||
| 2131 | 2176 | case OP_IMM: |
|
| 2132 | 2177 | if (f3 == 0 && rs1 == 0) { |
|
| 2133 | - | emit_mov_imm32_sx(e, d, imm); /* li */ |
|
| 2134 | - | break; |
|
| 2178 | + | emit_guest_imm32(e, rd, imm); /* li */ |
|
| 2179 | + | return; |
|
| 2135 | 2180 | } |
|
| 2136 | 2181 | if (f3 == 0 && src) { |
|
| 2137 | 2182 | if (imm) |
|
| 2138 | 2183 | emit_lea(e, true, d, src, imm); |
|
| 2139 | - | else |
|
| 2140 | - | emit_mov_rr(e, d, src); /* mv */ |
|
| 2184 | + | else { |
|
| 2185 | + | emit_store_guest(e, rd, src); /* mv */ |
|
| 2186 | + | return; |
|
| 2187 | + | } |
|
| 2141 | 2188 | break; |
|
| 2142 | 2189 | } |
|
| 2143 | 2190 | if (f3 == 2 || f3 == 3) { |
|
| 2144 | 2191 | /* slti, sltiu: cmp a, imm; setl/setb al; movzx d, al */ |
|
| 2145 | 2192 | u8 a = emit_base(e, rs1); |
| 2218 | 2265 | else |
|
| 2219 | 2266 | return false; |
|
| 2220 | 2267 | if (rd == 0 || rs1 == 0 || rs2 == 0) |
|
| 2221 | 2268 | return false; |
|
| 2222 | 2269 | ||
| 2223 | - | /* Compute in d, unless it holds rs2, which the first operand would |
|
| 2224 | - | * overwrite, or the word result needs sign extension from eax. */ |
|
| 2270 | + | /* For a commutative operation, keep the destination operand in place. */ |
|
| 2271 | + | if (op != 0x2B && rd == rs2) { |
|
| 2272 | + | rs2 = rs1; |
|
| 2273 | + | rs1 = rd; |
|
| 2274 | + | } |
|
| 2275 | + | ||
| 2276 | + | /* Keep rs2 intact until the operation reads it. */ |
|
| 2225 | 2277 | u8 d = dest_of(e, rd); |
|
| 2226 | - | u8 t = wide && (rd == rs1 || rd != rs2) ? d : X_RAX; |
|
| 2278 | + | u8 t = rd == rs1 || rd != rs2 ? d : X_RAX; |
|
| 2227 | 2279 | if (t == X_RAX || rd != rs1) |
|
| 2228 | 2280 | emit_load_guest(e, t, rs1); |
|
| 2229 | 2281 | emit_op_guest(e, wide, escape, op, t, rs2); |
|
| 2230 | 2282 | if (wide) |
|
| 2231 | 2283 | emit_mov_rr(e, d, t); |
|
| 2232 | 2284 | else |
|
| 2233 | - | emit_movsxd(e, d, X_RAX); |
|
| 2285 | + | emit_movsxd(e, d, t); |
|
| 2234 | 2286 | emit_finish(e, rd, d); |
|
| 2235 | 2287 | return true; |
|
| 2236 | 2288 | } |
|
| 2237 | 2289 | ||
| 2238 | 2290 | /* Offset of the reservation in the exchange area. */ |
| 2351 | 2403 | case OP_JALR: { |
|
| 2352 | 2404 | i32 imm = get_i_imm(ins); |
|
| 2353 | 2405 | ||
| 2354 | 2406 | /* Special-case RET: jalr x0, ra, 0. */ |
|
| 2355 | 2407 | if (ins.i.rd == 0 && ins.i.rs1 == RA && imm == 0) { |
|
| 2356 | - | /* Returning to the call this block followed: check that ra |
|
| 2357 | - | * still holds its return address and continue there. */ |
|
| 2358 | - | if (e->ra_known && e->sides_len < JIT_MAX_SIDE_EXITS && |
|
| 2359 | - | e->runs <= JIT_MAX_FOLLOWS && !translated(e, e->ra_value)) { |
|
| 2360 | - | emit_load_guest(e, HTMP1, RA); |
|
| 2361 | - | emit_rexw(e); |
|
| 2362 | - | emit_u8(e, 0x3D); |
|
| 2363 | - | emit_u32(e, e->ra_value); /* cmp rax, ra_value */ |
|
| 2364 | - | emit_side_exit(e, 0x85, 0, true); |
|
| 2365 | - | follow(e, pc_next, e->ra_value); |
|
| 2366 | - | return false; |
|
| 2367 | - | } |
|
| 2368 | 2408 | emit_ret(e); |
|
| 2369 | 2409 | return true; |
|
| 2370 | 2410 | } |
|
| 2371 | 2411 | ||
| 2372 | 2412 | /* A jump through the register the previous instruction set with |
| 2430 | 2470 | /* A branch around a trap, as in a bounds check, is taken: continue |
|
| 2431 | 2471 | * at the target and leave to the trap through the side exit, |
|
| 2432 | 2472 | * with the inverse condition. */ |
|
| 2433 | 2473 | if (traps(e, pc_next) && e->runs <= JIT_MAX_FOLLOWS && |
|
| 2434 | 2474 | !translated(e, target)) { |
|
| 2435 | - | emit_side_exit(e, cc ^ 1, pc_next, false); |
|
| 2475 | + | emit_side_exit(e, cc ^ 1, pc_next); |
|
| 2436 | 2476 | follow(e, pc_next, target); |
|
| 2437 | 2477 | return false; |
|
| 2438 | 2478 | } |
|
| 2439 | - | emit_side_exit(e, cc, target, false); |
|
| 2479 | + | emit_side_exit(e, cc, target); |
|
| 2440 | 2480 | return false; |
|
| 2441 | 2481 | } |
|
| 2442 | 2482 | ||
| 2443 | 2483 | u32 jcc_pos = e->pos; |
|
| 2444 | 2484 | emit_u8(e, 0x0F); |
| 2477 | 2517 | switch (ins.r.funct3) { |
|
| 2478 | 2518 | case 0x0: |
|
| 2479 | 2519 | emit_alu_rax_rcx(e, 0x01); |
|
| 2480 | 2520 | break; /* add */ |
|
| 2481 | 2521 | case 0x1: /* sll */ |
|
| 2482 | - | emit_and_ecx_imm8(e, 0x3F); |
|
| 2483 | 2522 | emit_shift_rax_cl(e, 4); |
|
| 2484 | 2523 | break; |
|
| 2485 | 2524 | case 0x2: /* slt */ |
|
| 2486 | 2525 | emit_cmp_rax_rcx(e); |
|
| 2487 | 2526 | emit_setcc_rax(e, 0x9C); /* setl */ |
| 2492 | 2531 | break; |
|
| 2493 | 2532 | case 0x4: |
|
| 2494 | 2533 | emit_alu_rax_rcx(e, 0x31); |
|
| 2495 | 2534 | break; /* xor */ |
|
| 2496 | 2535 | case 0x5: /* srl */ |
|
| 2497 | - | emit_and_ecx_imm8(e, 0x3F); |
|
| 2498 | 2536 | emit_shift_rax_cl(e, 5); |
|
| 2499 | 2537 | break; |
|
| 2500 | 2538 | case 0x6: |
|
| 2501 | 2539 | emit_alu_rax_rcx(e, 0x09); |
|
| 2502 | 2540 | break; /* or */ |
| 2510 | 2548 | switch (ins.r.funct3) { |
|
| 2511 | 2549 | case 0x0: |
|
| 2512 | 2550 | emit_alu_rax_rcx(e, 0x29); |
|
| 2513 | 2551 | break; /* sub */ |
|
| 2514 | 2552 | case 0x5: /* sra */ |
|
| 2515 | - | emit_and_ecx_imm8(e, 0x3F); |
|
| 2516 | 2553 | emit_shift_rax_cl(e, 7); |
|
| 2517 | 2554 | break; |
|
| 2518 | 2555 | } |
|
| 2519 | 2556 | break; |
|
| 2520 | 2557 |
| 2559 | 2596 | case 0x0: /* addw */ |
|
| 2560 | 2597 | emit_u8(e, 0x01); |
|
| 2561 | 2598 | emit_u8(e, modrm(3, X_RCX, X_RAX)); |
|
| 2562 | 2599 | break; |
|
| 2563 | 2600 | case 0x1: /* sllw */ |
|
| 2564 | - | emit_and_ecx_imm8(e, 0x1F); |
|
| 2565 | 2601 | emit_u8(e, 0xD3); |
|
| 2566 | 2602 | emit_u8(e, modrm(3, 4, X_RAX)); |
|
| 2567 | 2603 | break; |
|
| 2568 | 2604 | case 0x5: /* srlw */ |
|
| 2569 | - | emit_and_ecx_imm8(e, 0x1F); |
|
| 2570 | 2605 | emit_u8(e, 0xD3); |
|
| 2571 | 2606 | emit_u8(e, modrm(3, 5, X_RAX)); |
|
| 2572 | 2607 | break; |
|
| 2573 | 2608 | } |
|
| 2574 | 2609 | break; |
| 2578 | 2613 | case 0x0: /* subw */ |
|
| 2579 | 2614 | emit_u8(e, 0x29); |
|
| 2580 | 2615 | emit_u8(e, modrm(3, X_RCX, X_RAX)); |
|
| 2581 | 2616 | break; |
|
| 2582 | 2617 | case 0x5: /* sraw */ |
|
| 2583 | - | emit_and_ecx_imm8(e, 0x1F); |
|
| 2584 | 2618 | emit_u8(e, 0xD3); |
|
| 2585 | 2619 | emit_u8(e, modrm(3, 7, X_RAX)); |
|
| 2586 | 2620 | break; |
|
| 2587 | 2621 | } |
|
| 2588 | 2622 | break; |
| 2662 | 2696 | default: |
|
| 2663 | 2697 | return false; |
|
| 2664 | 2698 | } |
|
| 2665 | 2699 | } |
|
| 2666 | 2700 | ||
| 2667 | - | /* Host registers other than rax and rcx that an instruction clobbers: the |
|
| 2668 | - | * high multiplications and divisions use rdx and r11. */ |
|
| 2701 | + | /* High multiplications and divisions use rdx. Mulhsu also uses r11. */ |
|
| 2669 | 2702 | static u32 scratch_mask(instr_t ins) { |
|
| 2670 | 2703 | u32 opcode = ins.r.opcode; |
|
| 2671 | 2704 | if ((opcode == OP_OP || opcode == OP_OP_32) && ins.r.funct7 == FUNCT7_MUL && |
|
| 2672 | 2705 | ins.r.funct3 != 0 && bitmanip_decode(ins.raw) == BM_NONE) |
|
| 2673 | - | return 1u << X_RDX | 1u << X_R11; |
|
| 2706 | + | return (1u << X_RDX) | (ins.r.funct3 == 2 ? 1u << X_R11 : 0); |
|
| 2674 | 2707 | return 0; |
|
| 2675 | 2708 | } |
|
| 2676 | 2709 | ||
| 2677 | 2710 | /* Whether a scalar instruction may be translated with every guest register |
|
| 2678 | 2711 | * in the register file: it addresses them only through host_of(), clobbers |
| 2754 | 2787 | u32 h = (block->guest_pc >> 2) & (JIT_BLOCK_HASH_SIZE - 1); |
|
| 2755 | 2788 | block->hash_next = jit->block_hash[h]; |
|
| 2756 | 2789 | jit->block_hash[h] = block; |
|
| 2757 | 2790 | } |
|
| 2758 | 2791 | ||
| 2759 | - | /* Whether an instruction may write ra. Conservative: any instruction with |
|
| 2760 | - | * ra in the destination field except stores and branches, which have none. |
|
| 2761 | - | */ |
|
| 2762 | - | static bool writes_ra(instr_t ins) { |
|
| 2763 | - | u32 opcode = ins.r.opcode; |
|
| 2764 | - | if (opcode == OP_STORE || opcode == OP_BRANCH || opcode == 39) |
|
| 2765 | - | return false; |
|
| 2766 | - | return ins.r.rd == RA; |
|
| 2767 | - | } |
|
| 2768 | - | ||
| 2769 | 2792 | /* Translate the block at `guest_pc` into `em`. The vector shape guard is |
|
| 2770 | 2793 | * emitted only when `em->guard` is set. */ |
|
| 2771 | 2794 | static void translate_block( |
|
| 2772 | 2795 | struct emitter *em, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes |
|
| 2773 | 2796 | ) { |
|
| 2774 | 2797 | /* if (budget <= 0) goto .exhausted */ |
|
| 2775 | - | emit_u8(em, 0x4D); |
|
| 2776 | - | emit_u8(em, 0x85); |
|
| 2777 | - | emit_u8(em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */ |
|
| 2778 | 2798 | u32 jle_pos = emit_jle_placeholder(em); |
|
| 2779 | 2799 | ||
| 2780 | 2800 | if (em->guard) |
|
| 2781 | 2801 | emit_vector_guard(em, guest_pc); |
|
| 2782 | 2802 | ||
| 2783 | - | /* sub r15, insn_count -- imm32 is patched once the count is known. */ |
|
| 2784 | - | emit_u8(em, 0x49); |
|
| 2785 | - | emit_u8(em, 0x81); |
|
| 2786 | - | emit_u8(em, modrm(3, 5, HBUDGET)); |
|
| 2787 | - | u32 count_pos = em->pos; |
|
| 2788 | - | emit_u32(em, 0); |
|
| 2789 | - | ||
| 2790 | 2803 | u32 pc = guest_pc; |
|
| 2791 | 2804 | u32 insn_count = 0; |
|
| 2792 | 2805 | em->run_start[0] = guest_pc; |
|
| 2793 | 2806 | em->runs = 1; |
|
| 2794 | 2807 | u32 prog_end_pc = prog_base + prog_bytes; |
| 2801 | 2814 | instr_t ins; |
|
| 2802 | 2815 | memcpy(&ins, &mem[pc], sizeof(instr_t)); |
|
| 2803 | 2816 | insn_count++; |
|
| 2804 | 2817 | em->insns = insn_count; |
|
| 2805 | 2818 | ||
| 2806 | - | if (writes_ra(ins)) |
|
| 2807 | - | em->ra_known = false; |
|
| 2808 | 2819 | em->next_pc = pc + INSTR_SIZE; |
|
| 2809 | 2820 | bool ends_block = translate_insn(em, ins, pc); |
|
| 2810 | 2821 | ||
| 2811 | 2822 | /* Remember the value of an auipc for a jalr right after it. */ |
|
| 2812 | 2823 | em->auipc_reg = 0; |
| 2833 | 2844 | emit_side_exits(em); |
|
| 2834 | 2845 | em->spilled = false; |
|
| 2835 | 2846 | ||
| 2836 | 2847 | /* .exhausted: out of budget, leave before running anything. */ |
|
| 2837 | 2848 | patch_jcc(em, jle_pos, em->pos); |
|
| 2838 | - | emit_block_exit(em, JIT_EXIT_BRANCH, guest_pc); |
|
| 2839 | - | ||
| 2840 | - | for (u32 i = 0; i < 4; i++) |
|
| 2841 | - | em->buf[count_pos + i] = (u8)(insn_count >> (8 * i)); |
|
| 2842 | - | ||
| 2843 | - | for (u32 i = 0; i < em->refunds_len; i++) { |
|
| 2844 | - | u32 refund = insn_count - em->consumed[i]; |
|
| 2845 | - | for (u32 j = 0; j < 4; j++) |
|
| 2846 | - | em->buf[em->refunds[i] + j] = (u8)(refund >> (8 * j)); |
|
| 2847 | - | } |
|
| 2849 | + | emit_store_pc_imm(em, guest_pc); |
|
| 2850 | + | emit_exit(em, JIT_EXIT_BRANCH); |
|
| 2848 | 2851 | } |
|
| 2849 | 2852 | ||
| 2850 | 2853 | struct jit_block *jit_compile_block( |
|
| 2851 | 2854 | struct jit_state *jit, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes |
|
| 2852 | 2855 | ) { |
| 2926 | 2929 | jit->exit_raw = jit->code_cache + e.pos; |
|
| 2927 | 2930 | emit_u8(&e, 0x4D); |
|
| 2928 | 2931 | emit_u8(&e, 0x89); |
|
| 2929 | 2932 | emit_u8(&e, modrm(1, HBUDGET, HIO)); |
|
| 2930 | 2933 | emit_u8(&e, IO_BUDGET); /* mov [r14 + IO_BUDGET], r15 */ |
|
| 2934 | + | ||
| 2935 | + | /* Return through all outstanding native call frames before returning to C. |
|
| 2936 | + | */ |
|
| 2937 | + | jit->exit_unwind = jit->code_cache + e.pos; |
|
| 2938 | + | emit_u8(&e, 0x49); |
|
| 2939 | + | emit_u8(&e, 0x3B); |
|
| 2940 | + | emit_u8(&e, modrm(1, X_RSP, HIO)); |
|
| 2941 | + | emit_u8(&e, IO_STACK_BASE); /* cmp rsp, [r14 + IO_STACK_BASE] */ |
|
| 2942 | + | u32 frames = emit_jnz_placeholder(&e); |
|
| 2943 | + | ||
| 2931 | 2944 | emit_u8(&e, 0x41); |
|
| 2932 | 2945 | emit_u8(&e, 0x5F); /* pop r15 */ |
|
| 2933 | 2946 | emit_u8(&e, 0x41); |
|
| 2934 | 2947 | emit_u8(&e, 0x5E); /* pop r14 */ |
|
| 2935 | 2948 | emit_u8(&e, 0x41); |
| 2938 | 2951 | emit_u8(&e, 0x5C); /* pop r12 */ |
|
| 2939 | 2952 | emit_u8(&e, 0x5D); /* pop rbp */ |
|
| 2940 | 2953 | emit_u8(&e, 0x5B); /* pop rbx */ |
|
| 2941 | 2954 | emit_u8(&e, 0xC3); /* ret */ |
|
| 2942 | 2955 | ||
| 2943 | - | /* Entry: (rdi=regs, rsi=memory, rdx=io, rcx=code, r8=budget). Blocks |
|
| 2944 | - | * make no calls, so the stack alignment is irrelevant. */ |
|
| 2956 | + | patch_jcc(&e, frames, e.pos); |
|
| 2957 | + | /* Set OF for the return continuation. */ |
|
| 2958 | + | emit_u8(&e, 0xB1); |
|
| 2959 | + | emit_u8(&e, INT8_MAX); /* mov cl, 127 */ |
|
| 2960 | + | emit_u8(&e, 0x80); |
|
| 2961 | + | emit_u8(&e, modrm(3, 0, X_RCX)); |
|
| 2962 | + | emit_u8(&e, 1); /* add cl, 1 */ |
|
| 2963 | + | emit_native_ret(&e); |
|
| 2964 | + | ||
| 2965 | + | /* Entry: (rdi=memory, rsi=io, rdx=code, rcx=budget). */ |
|
| 2945 | 2966 | jit->enter = (jit_enter_fn)(void *)(jit->code_cache + e.pos); |
|
| 2946 | 2967 | emit_u8(&e, 0x53); /* push rbx */ |
|
| 2947 | 2968 | emit_u8(&e, 0x55); /* push rbp */ |
|
| 2948 | 2969 | emit_u8(&e, 0x41); |
|
| 2949 | 2970 | emit_u8(&e, 0x54); /* push r12 */ |
| 2951 | 2972 | emit_u8(&e, 0x55); /* push r13 */ |
|
| 2952 | 2973 | emit_u8(&e, 0x41); |
|
| 2953 | 2974 | emit_u8(&e, 0x56); /* push r14 */ |
|
| 2954 | 2975 | emit_u8(&e, 0x41); |
|
| 2955 | 2976 | emit_u8(&e, 0x57); /* push r15 */ |
|
| 2977 | + | emit_mov_rr(&e, HMEM, X_RDI); |
|
| 2978 | + | emit_mov_rr(&e, HIO, X_RSI); |
|
| 2979 | + | emit_mov_rr(&e, HBUDGET, X_RCX); |
|
| 2980 | + | emit_mov_rr(&e, X_RAX, X_RDX); |
|
| 2981 | + | ||
| 2982 | + | /* Set the bounds of the native call stack. */ |
|
| 2956 | 2983 | emit_u8(&e, 0x49); |
|
| 2957 | 2984 | emit_u8(&e, 0x89); |
|
| 2958 | - | emit_u8(&e, 0xFC); /* mov r12, rdi */ |
|
| 2959 | - | emit_u8(&e, 0x49); |
|
| 2960 | - | emit_u8(&e, 0x89); |
|
| 2961 | - | emit_u8(&e, 0xF5); /* mov r13, rsi */ |
|
| 2962 | - | emit_u8(&e, 0x49); |
|
| 2963 | - | emit_u8(&e, 0x89); |
|
| 2964 | - | emit_u8(&e, 0xD6); /* mov r14, rdx */ |
|
| 2985 | + | emit_u8(&e, modrm(1, X_RSP, HIO)); |
|
| 2986 | + | emit_u8(&e, IO_STACK_BASE); /* mov [r14 + IO_STACK_BASE], rsp */ |
|
| 2987 | + | emit_u8(&e, 0x4C); |
|
| 2988 | + | emit_u8(&e, 0x8D); |
|
| 2989 | + | emit_u8(&e, modrm(2, X_R11, X_RSP)); |
|
| 2990 | + | emit_u8(&e, sib(0, X_RSP, X_RSP)); |
|
| 2991 | + | emit_u32(&e, (u32)-JIT_CALL_STACK_SIZE); /* lea r11, [rsp - stack size] */ |
|
| 2965 | 2992 | emit_u8(&e, 0x4D); |
|
| 2966 | 2993 | emit_u8(&e, 0x89); |
|
| 2967 | - | emit_u8(&e, 0xC7); /* mov r15, r8 */ |
|
| 2968 | - | emit_u8(&e, 0x48); |
|
| 2969 | - | emit_u8(&e, 0x89); |
|
| 2970 | - | emit_u8(&e, 0xC8); /* mov rax, rcx */ |
|
| 2994 | + | emit_u8(&e, modrm(1, X_R11, HIO)); |
|
| 2995 | + | emit_u8(&e, IO_STACK_FLOOR); /* mov [r14 + IO_STACK_FLOOR], r11 */ |
|
| 2996 | + | ||
| 2971 | 2997 | /* The argument registers are free now: load the held guest registers. */ |
|
| 2972 | 2998 | emit_transfer(&e, ALL_HOSTS, false); |
|
| 2999 | + | ||
| 3000 | + | /* The entry frame returns to C when the guest returns to PC zero. */ |
|
| 3001 | + | emit_u8(&e, 0x6A); |
|
| 3002 | + | emit_u8(&e, 0); /* push guest link */ |
|
| 3003 | + | emit_u8(&e, 0x4D); |
|
| 3004 | + | emit_u8(&e, 0x85); |
|
| 3005 | + | emit_u8(&e, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */ |
|
| 2973 | 3006 | emit_u8(&e, 0xFF); |
|
| 2974 | - | emit_u8(&e, 0xE0); /* jmp rax */ |
|
| 3007 | + | emit_u8(&e, modrm(3, 2, X_RAX)); /* call rax */ |
|
| 3008 | + | emit_unwind_check(&e); |
|
| 3009 | + | emit_store_pc_imm(&e, 0); |
|
| 3010 | + | emit_exit(&e, JIT_EXIT_RET); |
|
| 2975 | 3011 | ||
| 2976 | 3012 | memcpy(jit->code_cache, buf, e.pos); |
|
| 2977 | 3013 | jit->code_cache_used = e.pos; |
|
| 2978 | 3014 | } |
|
| 2979 | 3015 |
jit.h
+22 -13
| 9 | 9 | * result, and execute the native code on subsequent visits. |
|
| 10 | 10 | */ |
|
| 11 | 11 | #ifndef JIT_H |
|
| 12 | 12 | #define JIT_H |
|
| 13 | 13 | ||
| 14 | + | #include <string.h> |
|
| 15 | + | ||
| 14 | 16 | #include "riscv.h" |
|
| 15 | 17 | #include "types.h" |
|
| 16 | 18 | ||
| 17 | 19 | /* Forward declarations. */ |
|
| 18 | 20 | struct cpu; |
| 56 | 58 | enum jit_exit { |
|
| 57 | 59 | JIT_EXIT_BRANCH = 0, /* Return to C, which continues at io.pc. */ |
|
| 58 | 60 | JIT_EXIT_ECALL = 1, /* ECALL -- needs interpreter handling. */ |
|
| 59 | 61 | JIT_EXIT_EBREAK = 2, /* EBREAK -- needs interpreter handling. */ |
|
| 60 | 62 | JIT_EXIT_FAULT = 3, /* Memory fault -- needs interpreter handling. */ |
|
| 61 | - | JIT_EXIT_CHAIN = 4, /* Direct jump to a known PC; io.patch is its site. */ |
|
| 62 | - | JIT_EXIT_RET = 5, /* Program returned (jalr x0, ra, 0 with ra==0). */ |
|
| 63 | + | JIT_EXIT_CHAIN = 4, /* Direct branch to a known PC; io.patch is its site. */ |
|
| 64 | + | JIT_EXIT_RET = 5, /* Program returned (jalr x0, ra, 0 with ra==0). */ |
|
| 63 | 65 | }; |
|
| 64 | 66 | ||
| 65 | 67 | /* A translated block of native code. */ |
|
| 66 | 68 | struct jit_block { |
|
| 67 | 69 | u32 guest_pc; /* Guest PC this block starts at. */ |
| 78 | 80 | struct vector_state *vector; /* Architectural vector registers. */ |
|
| 79 | 81 | u64 memory_size; /* Accessible hosted memory bytes. */ |
|
| 80 | 82 | /* LR reservation: the address in the low word and the access width in |
|
| 81 | 83 | * the high word, or zero when none is held. */ |
|
| 82 | 84 | u64 reservation; |
|
| 85 | + | /* Bounds of the host stack frames for translated calls. */ |
|
| 86 | + | u64 stack_base; |
|
| 87 | + | u64 stack_floor; |
|
| 83 | 88 | }; |
|
| 84 | 89 | ||
| 85 | 90 | /* Reservation value of an LR of `size` bytes at `address`. */ |
|
| 86 | 91 | static inline u64 jit_reservation(u32 address, u32 size) { |
|
| 87 | 92 | return (u64)size << 32 | address; |
| 92 | 97 | struct jit_jump_entry { |
|
| 93 | 98 | u32 guest_pc; /* Guest PC, or JIT_JUMP_CACHE_EMPTY. */ |
|
| 94 | 99 | u8 *code; /* Entry point of the block at that PC. */ |
|
| 95 | 100 | }; |
|
| 96 | 101 | ||
| 97 | - | /* Signature of the shared entry trampoline. It saves host registers, pins |
|
| 98 | - | * regs/memory/io/budget and jumps to `code`; the shared exit stub returns the |
|
| 99 | - | * exit reason. */ |
|
| 102 | + | /* Signature of the shared entry trampoline. It loads guest registers from |
|
| 103 | + | * the register file adjacent to io and calls the first translated block. */ |
|
| 100 | 104 | typedef int (*jit_enter_fn)( |
|
| 101 | - | u64 *regs, u8 *memory, struct jit_io *io, u8 *code, i64 budget |
|
| 105 | + | u8 *memory, struct jit_io *io, u8 *code, i64 budget |
|
| 102 | 106 | ); |
|
| 103 | 107 | ||
| 104 | 108 | /* Complete JIT state. */ |
|
| 105 | 109 | struct jit_state { |
|
| 106 | 110 | /* Executable code cache (mmap'd RWX). */ |
| 115 | 119 | u32 block_count; |
|
| 116 | 120 | ||
| 117 | 121 | /* Shared entry trampoline and exit stubs, at the start of the cache. |
|
| 118 | 122 | * `exit_stub` writes the guest registers held in host registers back to |
|
| 119 | 123 | * the register file and falls into `exit_raw`, which returns without |
|
| 120 | - | * writing them. */ |
|
| 124 | + | * writing them. `exit_unwind` returns through pending native calls. */ |
|
| 121 | 125 | jit_enter_fn enter; |
|
| 122 | 126 | u8 *exit_stub; |
|
| 123 | 127 | u8 *exit_raw; |
|
| 128 | + | u8 *exit_unwind; |
|
| 124 | 129 | ||
| 125 | - | /* Exit information for the current run. */ |
|
| 130 | + | /* The register file and exit information share one host base register. */ |
|
| 131 | + | u64 regs[32]; |
|
| 126 | 132 | struct jit_io io; |
|
| 127 | 133 | ||
| 128 | 134 | /* Direct-mapped cache of indirect jump targets, read by generated code |
|
| 129 | 135 | * and filled by C. */ |
|
| 130 | 136 | struct jit_jump_entry jump_cache[JIT_JUMP_CACHE_SIZE]; |
| 194 | 200 | &jit->jump_cache[(block->guest_pc >> 2) & (JIT_JUMP_CACHE_SIZE - 1)]; |
|
| 195 | 201 | ent->guest_pc = block->guest_pc; |
|
| 196 | 202 | ent->code = block->code; |
|
| 197 | 203 | } |
|
| 198 | 204 | ||
| 199 | - | /* Point the direct jump whose rel32 field is at `site` to `target`. */ |
|
| 205 | + | /* Point the direct branch whose rel32 field is at `site` to `target`. */ |
|
| 200 | 206 | void jit_patch_chain(u8 *site, struct jit_block *target); |
|
| 201 | 207 | ||
| 202 | 208 | /* Run translated code starting at `block`, with room for `budget` more guest |
|
| 203 | - | * instructions. Generated code follows direct and cached indirect jumps |
|
| 204 | - | * without returning. Returns the exit reason; the next PC, the remaining |
|
| 205 | - | * budget (which may be negative) and any patch site are left in jit->io. */ |
|
| 209 | + | * instructions. Copy guest register state to the JIT at entry and back at |
|
| 210 | + | * exit. Return the exit reason and leave the next PC, remaining budget and |
|
| 211 | + | * any patch site in jit->io. */ |
|
| 206 | 212 | static inline int jit_run( |
|
| 207 | 213 | struct jit_state *jit, |
|
| 208 | 214 | struct jit_block *block, |
|
| 209 | 215 | u64 *regs, |
|
| 210 | 216 | u8 *memory, |
|
| 211 | 217 | i64 budget |
|
| 212 | 218 | ) { |
|
| 213 | - | return jit->enter(regs, memory, &jit->io, block->code, budget); |
|
| 219 | + | memcpy(jit->regs, regs, sizeof(jit->regs)); |
|
| 220 | + | int reason = jit->enter(memory, &jit->io, block->code, budget); |
|
| 221 | + | memcpy(regs, jit->regs, sizeof(jit->regs)); |
|
| 222 | + | return reason; |
|
| 214 | 223 | } |
|
| 215 | 224 | ||
| 216 | 225 | #endif /* JIT_H */ |
test/jit_calls.c
+20 -4
| 1 | 1 | /* Writes a raw RV64 image for comparing translated and interpreted runs. |
|
| 2 | 2 | * |
|
| 3 | 3 | * The program makes calls through jal and through auipc and jalr, also with |
|
| 4 | 4 | * a jalr that reads the register it writes. It returns through ra, saves ra |
|
| 5 | 5 | * in a non-leaf function, branches around ebreak as a bounds check does, and |
|
| 6 | - | * uses LR and SC. Translated code follows these |
|
| 7 | - | * calls and checks the returns in place, so both runs must end with the |
|
| 8 | - | * same status after the same number of instructions. */ |
|
| 6 | + | * uses LR and SC. Recursive calls exceed the native call stack limit. |
|
| 7 | + | * Both runs must end with the same status and instruction count. */ |
|
| 9 | 8 | #include <stdint.h> |
|
| 10 | 9 | #include <stdio.h> |
|
| 11 | 10 | ||
| 12 | 11 | typedef uint32_t u32; |
|
| 13 | 12 | typedef int32_t i32; |
| 17 | 16 | ||
| 18 | 17 | static u32 code[256]; |
|
| 19 | 18 | static u32 len; |
|
| 20 | 19 | ||
| 21 | 20 | /* Labels and the instructions that refer to them. */ |
|
| 22 | - | enum label { LOOP, LEAF, LEAF_OK, NEST, LABELS }; |
|
| 21 | + | enum label { LOOP, LEAF, LEAF_OK, NEST, RECURSE, RECURSE_DONE, LABELS }; |
|
| 23 | 22 | static u32 labels[LABELS]; |
|
| 24 | 23 | static struct { |
|
| 25 | 24 | u32 at; |
|
| 26 | 25 | enum label to; |
|
| 27 | 26 | u32 kind; /* 0 jal, 1 branch, 2 jalr after auipc */ |
| 87 | 86 | fprintf(stderr, "usage: %s <output>\n", argv[0]); |
|
| 88 | 87 | return 1; |
|
| 89 | 88 | } |
|
| 90 | 89 | /* main: sum calls over 200 iterations into a0. */ |
|
| 91 | 90 | addi(A0, ZERO, 0); |
|
| 91 | + | addi(A1, ZERO, 600); |
|
| 92 | + | ref(0, RA << 7 | 0x6f, RECURSE); |
|
| 93 | + | addi(A1, ZERO, 600); |
|
| 94 | + | ref(0, RA << 7 | 0x6f, RECURSE); |
|
| 92 | 95 | addi(S1, ZERO, 200); |
|
| 93 | 96 | addi(T6, SP, -64); |
|
| 94 | 97 | mark(LOOP); |
|
| 95 | 98 | addi(A1, S1, 0); |
|
| 96 | 99 | ref(0, RA << 7 | 0x6f, LEAF); /* jal ra, leaf */ |
| 139 | 142 | ref(0, RA << 7 | 0x6f, LEAF); |
|
| 140 | 143 | emit(i_type(0x03, RA, 3, SP, 8)); |
|
| 141 | 144 | addi(SP, SP, 16); |
|
| 142 | 145 | emit(i_type(0x67, ZERO, 0, RA, 0)); |
|
| 143 | 146 | ||
| 147 | + | /* recurse(a1): add one to the result for each level after returning. */ |
|
| 148 | + | mark(RECURSE); |
|
| 149 | + | ref(1, r_type(0x63, 0, 0, A1, ZERO, 0), RECURSE_DONE); |
|
| 150 | + | addi(SP, SP, -16); |
|
| 151 | + | emit(s_type(RA, SP, 3, 8)); |
|
| 152 | + | addi(A1, A1, -1); |
|
| 153 | + | ref(0, RA << 7 | 0x6f, RECURSE); |
|
| 154 | + | emit(i_type(0x03, RA, 3, SP, 8)); |
|
| 155 | + | addi(SP, SP, 16); |
|
| 156 | + | addi(A0, A0, 1); |
|
| 157 | + | mark(RECURSE_DONE); |
|
| 158 | + | emit(i_type(0x67, ZERO, 0, RA, 0)); |
|
| 159 | + | ||
| 144 | 160 | resolve(); |
|
| 145 | 161 | FILE *f = fopen(argv[1], "wb"); |
|
| 146 | 162 | if (!f || fwrite(code, 4, len, f) != len || fclose(f)) { |
|
| 147 | 163 | perror(argv[1]); |
|
| 148 | 164 | return 1; |
test/jit_vector.c
+44 -0
| 271 | 271 | require_native = false; |
|
| 272 | 272 | u32 invalid = alu(25, 0, 9, 8, 16); |
|
| 273 | 273 | check(&invalid, 1, 0xd3, 33, 0, 128); |
|
| 274 | 274 | } |
|
| 275 | 275 | ||
| 276 | + | /* Check the budget at entry and across linked and indirect block exits. */ |
|
| 277 | + | static void check_budget(void) { |
|
| 278 | + | static struct jit_state jit; |
|
| 279 | + | u8 mem[CODE + 24] = { 0 }; |
|
| 280 | + | u64 x[32] = { 0 }; |
|
| 281 | + | const u32 code[] = { |
|
| 282 | + | 0x00000063, /* beq zero, zero, 0 */ |
|
| 283 | + | 0x00100513, /* addi a0, zero, 1 */ |
|
| 284 | + | 0x00008067, /* ret */ |
|
| 285 | + | 0x00028067, /* jalr zero, t0, 0 */ |
|
| 286 | + | 0x00200513, /* addi a0, zero, 2 */ |
|
| 287 | + | 0x00008067, /* ret */ |
|
| 288 | + | }; |
|
| 289 | + | memcpy(mem + CODE, code, sizeof(code)); |
|
| 290 | + | assert(jit_init(&jit)); |
|
| 291 | + | ||
| 292 | + | struct jit_block *loop = jit_get_block(&jit, CODE, mem, CODE, sizeof(code)); |
|
| 293 | + | assert(loop); |
|
| 294 | + | assert(jit_run(&jit, loop, x, mem, -1) == JIT_EXIT_BRANCH); |
|
| 295 | + | assert(jit.io.pc == CODE && jit.io.budget == -1 && x[10] == 0); |
|
| 296 | + | assert(jit_run(&jit, loop, x, mem, 5) == JIT_EXIT_CHAIN); |
|
| 297 | + | assert(jit.io.pc == CODE && jit.io.budget == 4 && x[10] == 0); |
|
| 298 | + | jit_patch_chain(jit.io.patch, loop); |
|
| 299 | + | assert(jit_run(&jit, loop, x, mem, 5) == JIT_EXIT_BRANCH); |
|
| 300 | + | assert(jit.io.pc == CODE && jit.io.budget == 0 && x[10] == 0); |
|
| 301 | + | ||
| 302 | + | u32 target = CODE + 16; |
|
| 303 | + | struct jit_block *jump = |
|
| 304 | + | jit_get_block(&jit, CODE + 12, mem, CODE, sizeof(code)); |
|
| 305 | + | struct jit_block *dest = |
|
| 306 | + | jit_get_block(&jit, target, mem, CODE, sizeof(code)); |
|
| 307 | + | assert(jump && dest); |
|
| 308 | + | x[5] = target; |
|
| 309 | + | assert(jit_run(&jit, jump, x, mem, 2) == JIT_EXIT_BRANCH); |
|
| 310 | + | assert(jit.io.pc == target && jit.io.budget == 1 && x[10] == 0); |
|
| 311 | + | jit_cache_jump(&jit, dest); |
|
| 312 | + | assert(jit_run(&jit, jump, x, mem, 1) == JIT_EXIT_BRANCH); |
|
| 313 | + | assert(jit.io.pc == target && jit.io.budget == 0 && x[10] == 0); |
|
| 314 | + | assert(jit_run(&jit, jump, x, mem, 2) == JIT_EXIT_RET); |
|
| 315 | + | assert(jit.io.budget == -1 && x[10] == 2); |
|
| 316 | + | jit_destroy(&jit); |
|
| 317 | + | } |
|
| 318 | + | ||
| 276 | 319 | int main(void) { |
|
| 320 | + | check_budget(); |
|
| 277 | 321 | check_bulk(); |
|
| 278 | 322 | check_word_groups(); |
|
| 279 | 323 | u32 ops[] = { |
|
| 280 | 324 | alu(0, 0, 8, 10, 12), alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0), |
|
| 281 | 325 | alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31), alu(9, 3, 8, 8, 3), |