jit: Hold hot registers and translate superblocks
df02cea04c38773d5b79a1e05f510986a745fb619f62c97a637b371ba81257d8
Halves the time of Radiance self-compilation (1.50s to 0.75s, 51% fewer cycles) with identical output. Nine guest registers live in host registers while translated code runs: the entry trampoline loads them and the exit stub stores them. Instructions whose emitters address the register file directly run spilled, sharing one spill and reload across neighbours. Loads and stores fold their address into one instruction, and common integer operations and branch compares work on held registers. Blocks continue past conditional branches, leaving through side exits that refund the budget, and past branches around ebreak on the taken path. They follow jal and auipc+jalr calls, and check a return to a followed call in place. LR and SC are translated, with the reservation shared with the interpreter. The jump cache grows to 64K entries and the code cache to 64 MB. Assisted-By: Claude:opus-5.5 Change-Id: I700003506f45b2291e95c646580b4ba2be7734de
1 parent
edb99b53
Makefile
+18 -2
| 62 | 62 | # Compare hosted vector translation with architectural interpreter state. |
|
| 63 | 63 | bin/jit-vector-test: test/jit_vector.c jit.c jit.h vector.c vector.h riscv.c riscv.h riscv/debug.c riscv/debug.h bitmanip.h color.h types.h |
|
| 64 | 64 | @mkdir -p bin |
|
| 65 | 65 | @$(CC) $(CPPFLAGS) -std=c99 -O2 -g -Wall -Wextra -fsanitize=address,undefined test/jit_vector.c jit.c vector.c riscv.c riscv/debug.c -o $@ |
|
| 66 | 66 | ||
| 67 | - | test-jit: bin/jit-vector-test |
|
| 67 | + | test-jit: bin/jit-vector-test test-jit-calls |
|
| 68 | 68 | @ASAN_OPTIONS=detect_leaks=0 timeout 60 $< |
|
| 69 | 69 | ||
| 70 | - | .PHONY: test-jit |
|
| 70 | + | # Compare translated and interpreted runs of calls, returns and LR/SC. |
|
| 71 | + | bin/jit-calls-gen: test/jit_calls.c |
|
| 72 | + | @mkdir -p bin |
|
| 73 | + | @$(CC) $(CPPFLAGS) -std=c99 -Wall -Wextra -g $< -o $@ |
|
| 74 | + | ||
| 75 | + | test-jit-calls: bin/emulator bin/jit-calls-gen |
|
| 76 | + | @bin/jit-calls-gen bin/jit-calls.bin |
|
| 77 | + | @jit=$$(bin/emulator -run -count-instructions bin/jit-calls.bin 2>&1; \ |
|
| 78 | + | echo "status $$?"); \ |
|
| 79 | + | int=$$(bin/emulator -run -count-instructions -no-jit bin/jit-calls.bin 2>&1; \ |
|
| 80 | + | echo "status $$?"); \ |
|
| 81 | + | if [ "$$jit" != "$$int" ]; then \ |
|
| 82 | + | echo "JIT call checks failed: '$$jit' != '$$int'"; exit 1; \ |
|
| 83 | + | fi; \ |
|
| 84 | + | echo "JIT call checks passed" |
|
| 85 | + | ||
| 86 | + | .PHONY: test-jit test-jit-calls |
emulator.c
+19 -1
| 1556 | 1556 | else |
|
| 1557 | 1557 | abi_linux_call(cpu, pc); |
|
| 1558 | 1558 | } |
|
| 1559 | 1559 | ||
| 1560 | 1560 | /* Execute a single instruction. */ |
|
| 1561 | + | /* Take the LR reservation held by translated code. */ |
|
| 1562 | + | static void reservation_from_jit(struct cpu *cpu, const struct jit_io *io) { |
|
| 1563 | + | cpu->reserved = io->reservation != 0; |
|
| 1564 | + | cpu->reservation = (u32)io->reservation; |
|
| 1565 | + | cpu->reservation_size = (u32)(io->reservation >> 32); |
|
| 1566 | + | } |
|
| 1567 | + | ||
| 1568 | + | /* Hand the LR reservation back to translated code. */ |
|
| 1569 | + | static void reservation_to_jit(const struct cpu *cpu, struct jit_io *io) { |
|
| 1570 | + | io->reservation = |
|
| 1571 | + | cpu->reserved ? jit_reservation(cpu->reservation, cpu->reservation_size) |
|
| 1572 | + | : 0; |
|
| 1573 | + | } |
|
| 1574 | + | ||
| 1561 | 1575 | static void cpu_execute(struct cpu *cpu, enum display display, bool headless) { |
|
| 1562 | 1576 | if (cpu_out_of_bounds(cpu)) { |
|
| 1563 | 1577 | cpu->running = false; |
|
| 1564 | 1578 | emit_fault_diagnostics(cpu, cpu->pc); |
|
| 1565 | 1579 | if (headless) { |
| 2581 | 2595 | &jit, cpu.pc, memory, program_base, program_bytes |
|
| 2582 | 2596 | ); |
|
| 2583 | 2597 | if (!block) { |
|
| 2584 | 2598 | /* Cache full or compilation error -- fall back to |
|
| 2585 | 2599 | * interpreter for remainder. */ |
|
| 2600 | + | reservation_from_jit(&cpu, &jit.io); |
|
| 2586 | 2601 | while (cpu.running && steps++ < max_steps) { |
|
| 2587 | 2602 | cpu_execute(&cpu, display, true); |
|
| 2588 | 2603 | } |
|
| 2589 | 2604 | break; |
|
| 2590 | 2605 | } |
| 2615 | 2630 | break; |
|
| 2616 | 2631 | ||
| 2617 | 2632 | default: |
|
| 2618 | 2633 | /* ECALL, EBREAK, FAULT -- interpreter handles it. |
|
| 2619 | 2634 | * The instruction is already counted above, |
|
| 2620 | - | * so don't increment steps again. */ |
|
| 2635 | + | * so don't increment steps again. It sees the |
|
| 2636 | + | * reservation of translated LR and SC. */ |
|
| 2637 | + | reservation_from_jit(&cpu, &jit.io); |
|
| 2621 | 2638 | cpu_execute(&cpu, display, true); |
|
| 2639 | + | reservation_to_jit(&cpu, &jit.io); |
|
| 2622 | 2640 | break; |
|
| 2623 | 2641 | } |
|
| 2624 | 2642 | } |
|
| 2625 | 2643 | jit_destroy(&jit); |
|
| 2626 | 2644 | } else { |
jit.c
+937 -348
| 45 | 45 | u8 *buf; /* Start of buffer. */ |
|
| 46 | 46 | u32 pos; /* Current write position. */ |
|
| 47 | 47 | u32 capacity; /* Maximum bytes. */ |
|
| 48 | 48 | bool overflow; /* Set if we ran out of space. */ |
|
| 49 | 49 | bool vector; /* Block uses four-lane 32-bit SIMD. */ |
|
| 50 | - | u32 insns; /* Instructions through the current operation. */ |
|
| 50 | + | bool guard; /* Emit the vector shape guard at block entry. */ |
|
| 51 | + | /* The register file holds every guest register and host registers may be |
|
| 52 | + | * clobbered: set from an instruction that addresses the register file |
|
| 53 | + | * directly until the held registers are reloaded. */ |
|
| 54 | + | bool spilled; |
|
| 55 | + | u32 insns; /* Instructions through the current operation. */ |
|
| 51 | 56 | /* Each indexed load can fail three checks for each of four lanes. */ |
|
| 52 | 57 | u32 refunds[JIT_MAX_BLOCK_INSNS * 12]; /* Budget patch offsets. */ |
|
| 53 | 58 | u32 consumed[JIT_MAX_BLOCK_INSNS * 12]; /* Instructions at each patch. */ |
|
| 54 | 59 | u32 refunds_len; /* Number of guarded exits. */ |
|
| 55 | 60 | ||
| 61 | + | /* Exits from inside the block, on a branch or a checked return, emitted |
|
| 62 | + | * after its main path. */ |
|
| 63 | + | struct { |
|
| 64 | + | u32 jcc; /* Offset of the jcc rel32 instruction. */ |
|
| 65 | + | u32 target; /* Guest PC of the branch target. */ |
|
| 66 | + | u32 insns; /* Instructions executed when it is taken. */ |
|
| 67 | + | bool ret; /* Return to ra instead of jumping to `target`. */ |
|
| 68 | + | } sides[JIT_MAX_SIDE_EXITS]; |
|
| 69 | + | u32 sides_len; |
|
| 70 | + | ||
| 71 | + | /* Guest PC after the current instruction. */ |
|
| 72 | + | u32 next_pc; |
|
| 73 | + | /* Runs of consecutive guest instructions translated so far: the block |
|
| 74 | + | * starts one, and each direct jump it follows starts another. The last |
|
| 75 | + | * run ends at next_pc. */ |
|
| 76 | + | u32 run_start[JIT_MAX_FOLLOWS + 1]; |
|
| 77 | + | u32 run_end[JIT_MAX_FOLLOWS + 1]; |
|
| 78 | + | u32 runs; |
|
| 79 | + | /* The previous instruction set `auipc_reg` to `auipc_value` with auipc, |
|
| 80 | + | * or `auipc_reg` is zero. */ |
|
| 81 | + | u32 auipc_reg; |
|
| 82 | + | u64 auipc_value; |
|
| 83 | + | /* ra holds `ra_value`, set by a call the block followed. */ |
|
| 84 | + | bool ra_known; |
|
| 85 | + | u32 ra_value; |
|
| 86 | + | ||
| 87 | + | u8 *mem; /* Guest memory, for looking ahead. */ |
|
| 88 | + | u32 prog_base; /* Guest code bounds. */ |
|
| 89 | + | u32 prog_end; |
|
| 56 | 90 | struct jit_state *jit; /* For the exit stub and jump cache addresses. */ |
|
| 57 | 91 | u8 *dest; /* Address the buffer will be copied to. */ |
|
| 58 | 92 | }; |
|
| 59 | 93 | ||
| 60 | 94 | static inline void emit_u8(struct emitter *e, u8 b) { |
| 119 | 153 | ||
| 120 | 154 | /* Scratch registers for codegen (caller-saved, not pinned). */ |
|
| 121 | 155 | #define HTMP1 X_RAX |
|
| 122 | 156 | #define HTMP2 X_RCX |
|
| 123 | 157 | ||
| 158 | + | /* Guest registers held in host registers for the whole time generated code |
|
| 159 | + | * runs, chosen by how often a compiler workload uses them. The entry |
|
| 160 | + | * trampoline loads them from the register file and the exit stub stores them |
|
| 161 | + | * back, so C always sees the register file. Zero entries are not held. */ |
|
| 162 | + | static const u8 guest_host[32] = { |
|
| 163 | + | [5] = X_RBX, /* t0 */ |
|
| 164 | + | [29] = X_RBP, /* t4 */ |
|
| 165 | + | [6] = X_RSI, /* t1 */ |
|
| 166 | + | [2] = X_RDI, /* sp */ |
|
| 167 | + | [30] = X_R8, /* t5 */ |
|
| 168 | + | [7] = X_R9, /* t2 */ |
|
| 169 | + | [10] = X_R10, /* a0 */ |
|
| 170 | + | [28] = X_R11, /* t3 */ |
|
| 171 | + | [11] = X_RDX, /* a1 */ |
|
| 172 | + | }; |
|
| 173 | + | ||
| 174 | + | /* Host register holding `guest`, or zero if it lives in the register file. */ |
|
| 175 | + | static inline u8 host_of(struct emitter *e, u32 guest) { |
|
| 176 | + | return e->spilled ? 0 : guest_host[guest]; |
|
| 177 | + | } |
|
| 178 | + | ||
| 124 | 179 | /* ---------- Common encoding helpers ------------------------------------ */ |
|
| 125 | 180 | ||
| 126 | 181 | /* Emit REX.W prefix (64-bit operand size, no extended regs). */ |
|
| 127 | 182 | static inline void emit_rexw(struct emitter *e) { |
|
| 128 | 183 | emit_u8(e, 0x48); |
| 166 | 221 | emit_u8(e, rex(true, reg >= 8, false, true)); /* B=1 for r12 */ |
|
| 167 | 222 | emit_u8(e, opcode); |
|
| 168 | 223 | emit_r12_address(e, reg, off); |
|
| 169 | 224 | } |
|
| 170 | 225 | ||
| 171 | - | /* Load guest register `guest` into host `dst` from [r12 + guest*8]. */ |
|
| 226 | + | /* mov dst, src (64-bit), omitted when they are the same register. */ |
|
| 227 | + | static void emit_mov_rr(struct emitter *e, u8 dst, u8 src) { |
|
| 228 | + | if (dst == src) |
|
| 229 | + | return; |
|
| 230 | + | emit_u8(e, rex(true, src >= 8, false, dst >= 8)); |
|
| 231 | + | emit_u8(e, 0x89); |
|
| 232 | + | emit_u8(e, modrm(3, src, dst)); |
|
| 233 | + | } |
|
| 234 | + | ||
| 235 | + | /* Load guest register `guest` into host `dst`. */ |
|
| 172 | 236 | static void emit_load_guest(struct emitter *e, enum x86reg dst, u32 guest) { |
|
| 173 | 237 | if (guest == 0) { |
|
| 174 | 238 | /* x0 is always zero -- xor dst, dst. */ |
|
| 175 | 239 | emit_u8(e, rex(true, dst >= 8, false, dst >= 8)); |
|
| 176 | 240 | emit_u8(e, 0x31); |
|
| 177 | 241 | emit_u8(e, modrm(3, dst, dst)); |
|
| 178 | 242 | return; |
|
| 179 | 243 | } |
|
| 244 | + | u8 host = host_of(e, guest); |
|
| 245 | + | if (host) { |
|
| 246 | + | emit_mov_rr(e, dst, host); |
|
| 247 | + | return; |
|
| 248 | + | } |
|
| 180 | 249 | emit_r12_disp(e, 0x8B, dst, guest * 8); |
|
| 181 | 250 | } |
|
| 182 | 251 | ||
| 183 | 252 | /* Load and zero-extend the low word of a guest register. */ |
|
| 184 | 253 | static void emit_load_guest32(struct emitter *e, enum x86reg dst, u32 guest) { |
|
| 185 | 254 | if (guest == 0) { |
|
| 186 | 255 | emit_load_guest(e, dst, guest); |
|
| 187 | 256 | return; |
|
| 188 | 257 | } |
|
| 258 | + | u8 host = host_of(e, guest); |
|
| 259 | + | if (host) { |
|
| 260 | + | /* mov dst32, host32 */ |
|
| 261 | + | if (dst >= 8 || host >= 8) |
|
| 262 | + | emit_u8(e, rex(false, host >= 8, false, dst >= 8)); |
|
| 263 | + | emit_u8(e, 0x89); |
|
| 264 | + | emit_u8(e, modrm(3, host, dst)); |
|
| 265 | + | return; |
|
| 266 | + | } |
|
| 189 | 267 | emit_u8(e, rex(false, dst >= 8, false, true)); |
|
| 190 | 268 | emit_u8(e, 0x8b); |
|
| 191 | 269 | emit_r12_address(e, dst, guest * 8); |
|
| 192 | 270 | } |
|
| 193 | 271 | ||
| 194 | - | /* Store host `src` to guest register [r12 + guest*8]. */ |
|
| 272 | + | /* Store host `src` to guest register `guest`. */ |
|
| 195 | 273 | static void emit_store_guest(struct emitter *e, u32 guest, enum x86reg src) { |
|
| 196 | 274 | if (guest == 0) |
|
| 197 | 275 | return; |
|
| 276 | + | u8 host = host_of(e, guest); |
|
| 277 | + | if (host) { |
|
| 278 | + | emit_mov_rr(e, host, src); |
|
| 279 | + | return; |
|
| 280 | + | } |
|
| 198 | 281 | emit_r12_disp(e, 0x89, src, guest * 8); |
|
| 199 | 282 | } |
|
| 200 | 283 | ||
| 284 | + | /* Every host register, as a mask for emit_transfer(). */ |
|
| 285 | + | #define ALL_HOSTS 0xFFFFu |
|
| 286 | + | ||
| 287 | + | /* Store (`store`) or load the held guest registers whose host registers are |
|
| 288 | + | * in `mask` to or from the register file. */ |
|
| 289 | + | static void emit_transfer(struct emitter *e, u32 mask, bool store) { |
|
| 290 | + | for (u32 g = 1; g < 32; g++) { |
|
| 291 | + | if (guest_host[g] && (mask >> guest_host[g]) & 1) |
|
| 292 | + | emit_r12_disp(e, store ? 0x89 : 0x8B, guest_host[g], g * 8); |
|
| 293 | + | } |
|
| 294 | + | } |
|
| 295 | + | ||
| 296 | + | /* Make the register file hold every guest register before an instruction |
|
| 297 | + | * that addresses it directly or clobbers host registers. */ |
|
| 298 | + | static void emit_spill(struct emitter *e) { |
|
| 299 | + | emit_transfer(e, ALL_HOSTS, true); |
|
| 300 | + | e->spilled = true; |
|
| 301 | + | } |
|
| 302 | + | ||
| 303 | + | /* Reload the held guest registers after such an instruction. */ |
|
| 304 | + | static void emit_reload(struct emitter *e) { |
|
| 305 | + | emit_transfer(e, ALL_HOSTS, false); |
|
| 306 | + | e->spilled = false; |
|
| 307 | + | } |
|
| 308 | + | ||
| 201 | 309 | /* mov reg, imm64 (REX.W + B8+rd) */ |
|
| 202 | 310 | static void emit_mov_imm64(struct emitter *e, enum x86reg dst, u64 imm) { |
|
| 203 | 311 | emit_u8(e, rex(true, false, false, dst >= 8)); |
|
| 204 | 312 | emit_u8(e, (u8)(0xB8 + (dst & 7))); |
|
| 205 | 313 | emit_u64(e, imm); |
| 278 | 386 | emit_u8(e, 0xC7); |
|
| 279 | 387 | emit_u8(e, modrm(0, 0, HIO)); |
|
| 280 | 388 | emit_u32(e, pc); |
|
| 281 | 389 | } |
|
| 282 | 390 | ||
| 283 | - | /* mov eax, reason; jmp exit_stub. */ |
|
| 391 | + | /* mov eax, reason; jmp exit_stub. While spilled, the register |
|
| 392 | + | * file is already current and the host registers may be stale. */ |
|
| 284 | 393 | static void emit_exit(struct emitter *e, u32 exit_reason) { |
|
| 285 | 394 | emit_u8(e, 0xB8); |
|
| 286 | 395 | emit_u32(e, exit_reason); /* mov eax, reason */ |
|
| 287 | - | emit_jmp_abs(e, e->jit->exit_stub); |
|
| 396 | + | emit_jmp_abs(e, e->spilled ? e->jit->exit_raw : e->jit->exit_stub); |
|
| 288 | 397 | } |
|
| 289 | 398 | ||
| 290 | 399 | /* Store next-PC to *r14 and leave the cache with `reason`. */ |
|
| 291 | 400 | static void emit_block_exit( |
|
| 292 | 401 | struct emitter *e, enum jit_exit reason, u32 next_pc |
| 297 | 406 | ||
| 298 | 407 | /* Jump to the guest block at the constant PC `target`. Emits a jmp rel32 |
|
| 299 | 408 | * that initially falls into a stub returning JIT_EXIT_CHAIN with the address |
|
| 300 | 409 | * of its rel32 field in io.patch; see jit_patch_chain. */ |
|
| 301 | 410 | static void emit_chain_exit(struct emitter *e, u32 target) { |
|
| 411 | + | /* A linked jump enters the target with the guest registers held. */ |
|
| 412 | + | bool spilled = e->spilled; |
|
| 413 | + | if (spilled) |
|
| 414 | + | emit_reload(e); |
|
| 415 | + | ||
| 302 | 416 | u32 site = e->pos; |
|
| 303 | 417 | emit_u8(e, 0xE9); |
|
| 304 | 418 | emit_u32(e, 0); /* jmp .stub (next instruction) */ |
|
| 305 | 419 | ||
| 306 | 420 | emit_store_pc_imm(e, target); |
| 316 | 430 | emit_u8(e, 0x89); |
|
| 317 | 431 | emit_u8(e, modrm(1, X_RAX, HIO)); |
|
| 318 | 432 | emit_u8(e, IO_PATCH); |
|
| 319 | 433 | ||
| 320 | 434 | emit_exit(e, JIT_EXIT_CHAIN); |
|
| 435 | + | e->spilled = spilled; |
|
| 321 | 436 | } |
|
| 322 | 437 | ||
| 438 | + | /* Offset of the jump cache from the exchange area in r14. */ |
|
| 439 | + | #define IO_JUMP_CACHE \ |
|
| 440 | + | ((u32)(offsetof(struct jit_state, jump_cache) - \ |
|
| 441 | + | offsetof(struct jit_state, io))) |
|
| 442 | + | ||
| 323 | 443 | /* Jump to the guest PC in rax (already even). Stores it to io.pc, probes the |
|
| 324 | - | * jump cache and jumps to the cached block on a hit, else exits to C. */ |
|
| 444 | + | * jump cache and jumps to the cached block on a hit, else exits to C. |
|
| 445 | + | * Clobbers only rax and rcx, since the target expects held registers. */ |
|
| 325 | 446 | static void emit_indirect_jump(struct emitter *e) { |
|
| 447 | + | if (e->spilled) |
|
| 448 | + | emit_reload(e); |
|
| 449 | + | ||
| 326 | 450 | /* mov dword [r14], eax */ |
|
| 327 | 451 | emit_u8(e, 0x41); |
|
| 328 | 452 | emit_u8(e, 0x89); |
|
| 329 | 453 | emit_u8(e, modrm(0, X_RAX, HIO)); |
|
| 330 | 454 |
| 337 | 461 | emit_u32(e, (JIT_JUMP_CACHE_SIZE - 1) << 2); |
|
| 338 | 462 | emit_u8(e, 0xC1); |
|
| 339 | 463 | emit_u8(e, modrm(3, 4, X_RCX)); |
|
| 340 | 464 | emit_u8(e, 2); |
|
| 341 | 465 | ||
| 342 | - | emit_mov_imm64(e, X_RDX, (u64)(uintptr_t)e->jit->jump_cache); |
|
| 343 | - | ||
| 344 | - | /* cmp eax, [rdx + rcx] */ |
|
| 466 | + | /* cmp eax, [r14 + rcx + IO_JUMP_CACHE] */ |
|
| 467 | + | emit_u8(e, 0x41); |
|
| 345 | 468 | emit_u8(e, 0x3B); |
|
| 346 | - | emit_u8(e, modrm(0, X_RAX, 4)); |
|
| 347 | - | emit_u8(e, sib(0, X_RCX, X_RDX)); |
|
| 469 | + | emit_u8(e, modrm(2, X_RAX, 4)); |
|
| 470 | + | emit_u8(e, sib(0, X_RCX, HIO)); |
|
| 471 | + | emit_u32(e, IO_JUMP_CACHE); |
|
| 348 | 472 | ||
| 349 | 473 | u32 jne_pos = emit_jnz_placeholder(e); |
|
| 350 | 474 | ||
| 351 | - | /* jmp [rdx + rcx + offsetof(code)] */ |
|
| 475 | + | /* jmp [r14 + rcx + IO_JUMP_CACHE + offsetof(code)] */ |
|
| 476 | + | emit_u8(e, 0x41); |
|
| 352 | 477 | emit_u8(e, 0xFF); |
|
| 353 | - | emit_u8(e, modrm(1, 4, 4)); |
|
| 354 | - | emit_u8(e, sib(0, X_RCX, X_RDX)); |
|
| 355 | - | emit_u8(e, offsetof(struct jit_jump_entry, code)); |
|
| 478 | + | emit_u8(e, modrm(2, 4, 4)); |
|
| 479 | + | emit_u8(e, sib(0, X_RCX, HIO)); |
|
| 480 | + | emit_u32(e, IO_JUMP_CACHE + offsetof(struct jit_jump_entry, code)); |
|
| 356 | 481 | ||
| 357 | 482 | /* Miss: io.pc is set, let C dispatch. */ |
|
| 358 | 483 | patch_jcc(e, jne_pos, e->pos); |
|
| 359 | 484 | emit_exit(e, JIT_EXIT_BRANCH); |
|
| 360 | 485 | } |
| 680 | 805 | } |
|
| 681 | 806 | } |
|
| 682 | 807 | ||
| 683 | 808 | /* ---------- Memory access helpers -------------------------------------- */ |
|
| 684 | 809 | ||
| 810 | + | /* Emit `opcode` with `reg` and the memory operand [r13 + index + disp], where |
|
| 811 | + | * `opcode` is one byte, or two when `escape` prefixes it with 0x0F. `word` |
|
| 812 | + | * selects a 16-bit operand. */ |
|
| 813 | + | static void emit_guest_memory( |
|
| 814 | + | struct emitter *e, |
|
| 815 | + | bool word, |
|
| 816 | + | bool wide, |
|
| 817 | + | bool escape, |
|
| 818 | + | u8 opcode, |
|
| 819 | + | u8 reg, |
|
| 820 | + | u8 index, |
|
| 821 | + | i32 disp |
|
| 822 | + | ) { |
|
| 823 | + | if (word) |
|
| 824 | + | emit_u8(e, 0x66); |
|
| 825 | + | /* Always present: r13 needs REX.B, and it selects sil/dil for bytes. */ |
|
| 826 | + | emit_u8(e, rex(wide, reg >= 8, index >= 8, true)); |
|
| 827 | + | if (escape) |
|
| 828 | + | emit_u8(e, 0x0F); |
|
| 829 | + | emit_u8(e, opcode); |
|
| 830 | + | bool short_disp = disp >= -128 && disp <= 127; |
|
| 831 | + | emit_u8(e, modrm(short_disp ? 1 : 2, reg, 4)); |
|
| 832 | + | emit_u8(e, sib(0, index, HMEM)); |
|
| 833 | + | if (short_disp) |
|
| 834 | + | emit_u8(e, (u8)(i8)disp); |
|
| 835 | + | else |
|
| 836 | + | emit_u32(e, (u32)disp); |
|
| 837 | + | } |
|
| 838 | + | ||
| 839 | + | /* Host register holding the base address register `guest`, loading it into |
|
| 840 | + | * rax unless it is held. */ |
|
| 841 | + | static u8 emit_base(struct emitter *e, u32 guest) { |
|
| 842 | + | u8 host = host_of(e, guest); |
|
| 843 | + | if (host) |
|
| 844 | + | return host; |
|
| 845 | + | emit_load_guest(e, X_RAX, guest); |
|
| 846 | + | return X_RAX; |
|
| 847 | + | } |
|
| 848 | + | ||
| 849 | + | /* Translate a scalar load. Returns false for an invalid width. */ |
|
| 850 | + | static bool emit_scalar_load(struct emitter *e, instr_t ins) { |
|
| 851 | + | static const struct { |
|
| 852 | + | bool valid, wide, escape; |
|
| 853 | + | u8 opcode; |
|
| 854 | + | } forms[8] = { |
|
| 855 | + | { true, true, true, 0xBE }, /* lb: movsx r64, byte */ |
|
| 856 | + | { true, true, true, 0xBF }, /* lh: movsx r64, word */ |
|
| 857 | + | { true, true, false, 0x63 }, /* lw: movsxd r64, dword */ |
|
| 858 | + | { true, true, false, 0x8B }, /* ld: mov r64, qword */ |
|
| 859 | + | { true, false, true, 0xB6 }, /* lbu: movzx r32, byte */ |
|
| 860 | + | { true, false, true, 0xB7 }, /* lhu: movzx r32, word */ |
|
| 861 | + | { true, false, false, 0x8B }, /* lwu: mov r32, dword */ |
|
| 862 | + | { false, false, false, 0 }, |
|
| 863 | + | }; |
|
| 864 | + | u32 f = ins.i.funct3; |
|
| 865 | + | if (!forms[f].valid) |
|
| 866 | + | return false; |
|
| 867 | + | if (ins.i.rd == 0) |
|
| 868 | + | return true; |
|
| 869 | + | u8 base = emit_base(e, ins.i.rs1); |
|
| 870 | + | u8 dst = host_of(e, ins.i.rd); |
|
| 871 | + | if (!dst) |
|
| 872 | + | dst = X_RAX; |
|
| 873 | + | emit_guest_memory( |
|
| 874 | + | e, |
|
| 875 | + | false, |
|
| 876 | + | forms[f].wide, |
|
| 877 | + | forms[f].escape, |
|
| 878 | + | forms[f].opcode, |
|
| 879 | + | dst, |
|
| 880 | + | base, |
|
| 881 | + | get_i_imm(ins) |
|
| 882 | + | ); |
|
| 883 | + | if (dst == X_RAX) |
|
| 884 | + | emit_store_guest(e, ins.i.rd, X_RAX); |
|
| 885 | + | return true; |
|
| 886 | + | } |
|
| 887 | + | ||
| 888 | + | /* Translate a scalar store. Returns false for an invalid width. */ |
|
| 889 | + | static bool emit_scalar_store(struct emitter *e, instr_t ins) { |
|
| 890 | + | u32 f = ins.s.funct3; |
|
| 891 | + | if (f > 3) |
|
| 892 | + | return false; |
|
| 893 | + | u8 base = emit_base(e, ins.s.rs1); |
|
| 894 | + | u8 src = host_of(e, ins.s.rs2); |
|
| 895 | + | if (!src) { |
|
| 896 | + | emit_load_guest(e, X_RCX, ins.s.rs2); |
|
| 897 | + | src = X_RCX; |
|
| 898 | + | } |
|
| 899 | + | emit_guest_memory( |
|
| 900 | + | e, |
|
| 901 | + | f == 1, |
|
| 902 | + | f == 3, |
|
| 903 | + | false, |
|
| 904 | + | f == 0 ? 0x88 : 0x89, |
|
| 905 | + | src, |
|
| 906 | + | base, |
|
| 907 | + | get_s_imm(ins) |
|
| 908 | + | ); |
|
| 909 | + | return true; |
|
| 910 | + | } |
|
| 911 | + | ||
| 685 | 912 | /* add rax, r13 (compute host address from guest address). */ |
|
| 686 | 913 | static void emit_add_rax_r13(struct emitter *e) { |
|
| 687 | 914 | emit_u8(e, 0x4C); |
|
| 688 | 915 | emit_u8(e, 0x01); |
|
| 689 | 916 | emit_u8(e, 0xE8); |
|
| 690 | 917 | } |
|
| 691 | 918 | ||
| 692 | - | /* Load from [r13+rax] into rax, with sign/zero extension. */ |
|
| 693 | - | static void emit_load_mem_i8(struct emitter *e) { |
|
| 694 | - | emit_add_rax_r13(e); |
|
| 695 | - | emit_rexw(e); |
|
| 696 | - | emit_u8(e, 0x0F); |
|
| 697 | - | emit_u8(e, 0xBE); |
|
| 698 | - | emit_u8(e, modrm(0, X_RAX, X_RAX)); |
|
| 699 | - | } |
|
| 700 | - | ||
| 701 | - | static void emit_load_mem_u8(struct emitter *e) { |
|
| 702 | - | emit_add_rax_r13(e); |
|
| 703 | - | emit_u8(e, 0x0F); |
|
| 704 | - | emit_u8(e, 0xB6); |
|
| 705 | - | emit_u8(e, modrm(0, X_RAX, X_RAX)); |
|
| 706 | - | } |
|
| 707 | - | ||
| 708 | - | static void emit_load_mem_i16(struct emitter *e) { |
|
| 709 | - | emit_add_rax_r13(e); |
|
| 710 | - | emit_rexw(e); |
|
| 711 | - | emit_u8(e, 0x0F); |
|
| 712 | - | emit_u8(e, 0xBF); |
|
| 713 | - | emit_u8(e, modrm(0, X_RAX, X_RAX)); |
|
| 714 | - | } |
|
| 715 | - | ||
| 716 | - | static void emit_load_mem_u16(struct emitter *e) { |
|
| 717 | - | emit_add_rax_r13(e); |
|
| 718 | - | emit_u8(e, 0x0F); |
|
| 719 | - | emit_u8(e, 0xB7); |
|
| 720 | - | emit_u8(e, modrm(0, X_RAX, X_RAX)); |
|
| 721 | - | } |
|
| 722 | - | ||
| 723 | - | static void emit_load_mem_i32(struct emitter *e) { |
|
| 724 | - | emit_add_rax_r13(e); |
|
| 725 | - | emit_rexw(e); |
|
| 726 | - | emit_u8(e, 0x63); |
|
| 727 | - | emit_u8(e, modrm(0, X_RAX, X_RAX)); |
|
| 728 | - | } |
|
| 729 | - | ||
| 730 | - | static void emit_load_mem_u32(struct emitter *e) { |
|
| 731 | - | emit_add_rax_r13(e); |
|
| 732 | - | emit_u8(e, 0x8B); |
|
| 733 | - | emit_u8(e, modrm(0, X_RAX, X_RAX)); |
|
| 734 | - | } |
|
| 735 | - | ||
| 736 | - | static void emit_load_mem_u64(struct emitter *e) { |
|
| 737 | - | emit_add_rax_r13(e); |
|
| 738 | - | emit_rexw(e); |
|
| 739 | - | emit_u8(e, 0x8B); |
|
| 740 | - | emit_u8(e, modrm(0, X_RAX, X_RAX)); |
|
| 741 | - | } |
|
| 742 | - | ||
| 743 | - | /* Store from rcx to [r13+rax]. */ |
|
| 744 | - | static void emit_store_mem_u8(struct emitter *e) { |
|
| 745 | - | emit_add_rax_r13(e); |
|
| 746 | - | emit_u8(e, 0x88); |
|
| 747 | - | emit_u8(e, modrm(0, X_RCX, X_RAX)); |
|
| 748 | - | } |
|
| 749 | - | ||
| 750 | - | static void emit_store_mem_u16(struct emitter *e) { |
|
| 751 | - | emit_add_rax_r13(e); |
|
| 752 | - | emit_u8(e, 0x66); |
|
| 753 | - | emit_u8(e, 0x89); |
|
| 754 | - | emit_u8(e, modrm(0, X_RCX, X_RAX)); |
|
| 755 | - | } |
|
| 756 | - | ||
| 757 | - | static void emit_store_mem_u32(struct emitter *e) { |
|
| 758 | - | emit_add_rax_r13(e); |
|
| 759 | - | emit_u8(e, 0x89); |
|
| 760 | - | emit_u8(e, modrm(0, X_RCX, X_RAX)); |
|
| 761 | - | } |
|
| 762 | - | ||
| 763 | - | static void emit_store_mem_u64(struct emitter *e) { |
|
| 764 | - | emit_add_rax_r13(e); |
|
| 765 | - | emit_rexw(e); |
|
| 766 | - | emit_u8(e, 0x89); |
|
| 767 | - | emit_u8(e, modrm(0, X_RCX, X_RAX)); |
|
| 768 | - | } |
|
| 769 | - | ||
| 770 | 919 | /* Count the low 32 or 64 bits in rax and return the result in rax. */ |
|
| 771 | 920 | static void emit_popcount(struct emitter *e, bool wide) { |
|
| 772 | 921 | if (__builtin_cpu_supports("popcnt")) { |
|
| 773 | 922 | emit_u8(e, 0xf3); |
|
| 774 | 923 | if (wide) |
| 1089 | 1238 | break; |
|
| 1090 | 1239 | } |
|
| 1091 | 1240 | emit_store_rd(e, ins.r.rd); |
|
| 1092 | 1241 | } |
|
| 1093 | 1242 | ||
| 1243 | + | /* Give back the budget of the instructions after the first `insns` of the |
|
| 1244 | + | * block, on a path that leaves after them: add r15, refund. The refund is |
|
| 1245 | + | * patched in once the length of the block is known. */ |
|
| 1246 | + | static void emit_refund(struct emitter *e, u32 insns) { |
|
| 1247 | + | emit_u8(e, 0x49); |
|
| 1248 | + | emit_u8(e, 0x81); |
|
| 1249 | + | emit_u8(e, modrm(3, 0, HBUDGET)); |
|
| 1250 | + | e->refunds[e->refunds_len] = e->pos; |
|
| 1251 | + | e->consumed[e->refunds_len++] = insns; |
|
| 1252 | + | emit_u32(e, 0); |
|
| 1253 | + | } |
|
| 1254 | + | ||
| 1094 | 1255 | /* ---------- Vector translation ----------------------------------------- */ |
|
| 1095 | 1256 | ||
| 1096 | - | /* Return a failed memory operation to the interpreter before its side effects. |
|
| 1097 | - | */ |
|
| 1098 | - | static void emit_vector_fault(struct emitter *e, u32 pc, u8 condition) { |
|
| 1257 | + | /* Return a failed memory operation to the interpreter before its side effects |
|
| 1258 | + | * when the jcc `condition` (the second opcode byte) holds. */ |
|
| 1259 | + | static void emit_fault_if(struct emitter *e, u32 pc, u8 condition) { |
|
| 1099 | 1260 | u32 bad = e->pos; |
|
| 1100 | 1261 | emit_u8(e, 0x0f); |
|
| 1101 | 1262 | emit_u8(e, condition); |
|
| 1102 | 1263 | emit_u32(e, 0); |
|
| 1103 | 1264 | u32 ok = emit_jmp_placeholder(e); |
|
| 1104 | 1265 | patch_jcc(e, bad, e->pos); |
|
| 1105 | - | emit_u8(e, 0x49); |
|
| 1106 | - | emit_u8(e, 0x81); |
|
| 1107 | - | emit_u8(e, modrm(3, 0, HBUDGET)); |
|
| 1108 | - | e->refunds[e->refunds_len] = e->pos; |
|
| 1109 | - | e->consumed[e->refunds_len++] = e->insns; |
|
| 1110 | - | emit_u32(e, 0); |
|
| 1266 | + | emit_refund(e, e->insns); |
|
| 1111 | 1267 | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 1112 | 1268 | patch_jmp(e, ok, e->pos); |
|
| 1113 | 1269 | } |
|
| 1114 | 1270 | ||
| 1115 | 1271 | /* Load the architectural vector state into rax. */ |
| 1140 | 1296 | emit_u32(e, ~0xc7u); |
|
| 1141 | 1297 | emit_rexw(e); |
|
| 1142 | 1298 | emit_u8(e, 0x83); |
|
| 1143 | 1299 | emit_u8(e, 0xfa); |
|
| 1144 | 1300 | emit_u8(e, width << 3); |
|
| 1145 | - | emit_vector_fault(e, pc, 0x85); |
|
| 1301 | + | emit_fault_if(e, pc, 0x85); |
|
| 1146 | 1302 | emit_rexw(e); |
|
| 1147 | 1303 | emit_u8(e, 0x83); |
|
| 1148 | 1304 | emit_u8(e, 0xe1); |
|
| 1149 | 1305 | emit_u8(e, 7); |
|
| 1150 | 1306 | emit_rexw(e); |
|
| 1151 | 1307 | emit_u8(e, 0x83); |
|
| 1152 | 1308 | emit_u8(e, 0xf9); |
|
| 1153 | 1309 | emit_u8(e, lm); |
|
| 1154 | - | emit_vector_fault(e, pc, 0x87); |
|
| 1310 | + | emit_fault_if(e, pc, 0x87); |
|
| 1155 | 1311 | emit_mov_imm32_sx(e, X_RDX, VECTOR_BYTES >> width); |
|
| 1156 | 1312 | emit_rexw(e); |
|
| 1157 | 1313 | emit_u8(e, 0xd3); |
|
| 1158 | 1314 | emit_u8(e, 0xe2); /* shl rdx, cl */ |
|
| 1159 | 1315 | emit_rexw(e); |
|
| 1160 | 1316 | emit_u8(e, 0x83); |
|
| 1161 | 1317 | emit_u8(e, modrm(2, 7, X_RAX)); |
|
| 1162 | 1318 | emit_u32(e, offsetof(struct vector_state, vstart)); |
|
| 1163 | 1319 | emit_u8(e, 0); |
|
| 1164 | - | emit_vector_fault(e, pc, 0x85); |
|
| 1320 | + | emit_fault_if(e, pc, 0x85); |
|
| 1165 | 1321 | emit_rexw(e); |
|
| 1166 | 1322 | emit_u8(e, 0x8b); |
|
| 1167 | 1323 | emit_u8(e, modrm(2, X_RCX, X_RAX)); |
|
| 1168 | 1324 | emit_u32(e, offsetof(struct vector_state, vl)); |
|
| 1169 | 1325 | emit_rexw(e); |
|
| 1170 | 1326 | emit_u8(e, 0x39); |
|
| 1171 | 1327 | emit_u8(e, 0xd1); /* cmp rcx, rdx */ |
|
| 1172 | - | emit_vector_fault(e, pc, 0x87); |
|
| 1328 | + | emit_fault_if(e, pc, 0x87); |
|
| 1173 | 1329 | } |
|
| 1174 | 1330 | ||
| 1175 | 1331 | /* Form a pointer into one architectural register group. */ |
|
| 1176 | 1332 | static void emit_group_pointer(struct emitter *e, enum x86reg dst, u32 reg) { |
|
| 1177 | 1333 | emit_rexw(e); |
| 1256 | 1412 | emit_u32(e, index * VECTOR_BYTES + lane * 4); |
|
| 1257 | 1413 | emit_load_guest(e, X_RAX, base); |
|
| 1258 | 1414 | emit_u8(e, 0x48); |
|
| 1259 | 1415 | emit_u8(e, 0x01); |
|
| 1260 | 1416 | emit_u8(e, 0xd0); |
|
| 1261 | - | emit_vector_fault(e, pc, 0x82); /* jc */ |
|
| 1417 | + | emit_fault_if(e, pc, 0x82); /* jc */ |
|
| 1262 | 1418 | } |
|
| 1263 | 1419 | ||
| 1264 | 1420 | /* Check a complete transfer against the hosted memory bound. */ |
|
| 1265 | 1421 | static void emit_vector_bound(struct emitter *e, u32 size, u32 pc) { |
|
| 1266 | 1422 | emit_u8(e, 0x49); |
| 1269 | 1425 | emit_u8(e, offsetof(struct jit_io, memory_size)); |
|
| 1270 | 1426 | emit_u8(e, 0x48); |
|
| 1271 | 1427 | emit_u8(e, 0x83); |
|
| 1272 | 1428 | emit_u8(e, 0xe9); |
|
| 1273 | 1429 | emit_u8(e, size); |
|
| 1274 | - | emit_vector_fault(e, pc, 0x82); |
|
| 1430 | + | emit_fault_if(e, pc, 0x82); |
|
| 1275 | 1431 | emit_u8(e, 0x48); |
|
| 1276 | 1432 | emit_u8(e, 0x39); |
|
| 1277 | 1433 | emit_u8(e, 0xc8); |
|
| 1278 | - | emit_vector_fault(e, pc, 0x87); /* ja */ |
|
| 1434 | + | emit_fault_if(e, pc, 0x87); /* ja */ |
|
| 1279 | 1435 | } |
|
| 1280 | 1436 | ||
| 1281 | 1437 | /* Transfer active bytes with complete bounds checks before any guest write. */ |
|
| 1282 | 1438 | static bool emit_group_memory(struct emitter *e, u32 ins, u32 pc) { |
|
| 1283 | 1439 | u32 op = ins & 127, reg = (ins >> 7) & 31; |
| 1310 | 1466 | emit_u8(e, modrm(1, X_RDX, HIO)); |
|
| 1311 | 1467 | emit_u8(e, offsetof(struct jit_io, memory_size)); |
|
| 1312 | 1468 | emit_rexw(e); |
|
| 1313 | 1469 | emit_u8(e, 0x29); |
|
| 1314 | 1470 | emit_u8(e, 0xca); /* sub rdx, rcx */ |
|
| 1315 | - | emit_vector_fault(e, pc, 0x82); |
|
| 1471 | + | emit_fault_if(e, pc, 0x82); |
|
| 1316 | 1472 | emit_rexw(e); |
|
| 1317 | 1473 | emit_u8(e, 0x39); |
|
| 1318 | 1474 | emit_u8(e, 0xd0); /* cmp rax, rdx */ |
|
| 1319 | - | emit_vector_fault(e, pc, 0x87); |
|
| 1475 | + | emit_fault_if(e, pc, 0x87); |
|
| 1320 | 1476 | emit_add_rax_r13(e); |
|
| 1321 | 1477 | emit_rexw(e); |
|
| 1322 | 1478 | emit_u8(e, 0x89); |
|
| 1323 | 1479 | emit_u8(e, store ? 0xc7 : 0xc6); |
|
| 1324 | 1480 | emit_u8(e, 0xf3); |
| 1336 | 1492 | emit_u8(e, 0xf7); |
|
| 1337 | 1493 | emit_u8(e, 0xe6); /* mul rsi */ |
|
| 1338 | 1494 | emit_rexw(e); |
|
| 1339 | 1495 | emit_u8(e, 0x85); |
|
| 1340 | 1496 | emit_u8(e, 0xd2); |
|
| 1341 | - | emit_vector_fault(e, pc, 0x85); |
|
| 1497 | + | emit_fault_if(e, pc, 0x85); |
|
| 1342 | 1498 | emit_rexw(e); |
|
| 1343 | 1499 | emit_u8(e, 0x01); |
|
| 1344 | 1500 | emit_u8(e, 0xf8); /* add rax, rdi */ |
|
| 1345 | - | emit_vector_fault(e, pc, 0x82); |
|
| 1501 | + | emit_fault_if(e, pc, 0x82); |
|
| 1346 | 1502 | emit_vector_bound(e, 1, pc); |
|
| 1347 | 1503 | emit_rexw(e); |
|
| 1348 | 1504 | emit_u8(e, 0x89); |
|
| 1349 | 1505 | emit_u8(e, 0xf2); /* mov rdx, rsi */ |
|
| 1350 | 1506 | emit_vector_base(e); |
| 1741 | 1897 | emit_u8(e, 1); |
|
| 1742 | 1898 | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 1743 | 1899 | patch_jmp(e, ok, e->pos); |
|
| 1744 | 1900 | } |
|
| 1745 | 1901 | ||
| 1902 | + | /* Set the flags as cmp rs1, rs2 would, using held registers in place. */ |
|
| 1903 | + | static void emit_compare(struct emitter *e, u32 rs1, u32 rs2) { |
|
| 1904 | + | u8 a = emit_base(e, rs1); |
|
| 1905 | + | if (rs2 == 0) { |
|
| 1906 | + | /* test a, a */ |
|
| 1907 | + | emit_u8(e, rex(true, a >= 8, false, a >= 8)); |
|
| 1908 | + | emit_u8(e, 0x85); |
|
| 1909 | + | emit_u8(e, modrm(3, a, a)); |
|
| 1910 | + | return; |
|
| 1911 | + | } |
|
| 1912 | + | u8 b = host_of(e, rs2); |
|
| 1913 | + | if (b) { |
|
| 1914 | + | /* cmp a, b */ |
|
| 1915 | + | emit_u8(e, rex(true, b >= 8, false, a >= 8)); |
|
| 1916 | + | emit_u8(e, 0x39); |
|
| 1917 | + | emit_u8(e, modrm(3, b, a)); |
|
| 1918 | + | return; |
|
| 1919 | + | } |
|
| 1920 | + | /* cmp a, [r12 + rs2 * 8] */ |
|
| 1921 | + | emit_r12_disp(e, 0x3B, a, rs2 * 8); |
|
| 1922 | + | } |
|
| 1923 | + | ||
| 1924 | + | /* Whether the block already holds the instruction at `pc`. */ |
|
| 1925 | + | static bool translated(struct emitter *e, u32 pc) { |
|
| 1926 | + | for (u32 i = 0; i < e->runs; i++) { |
|
| 1927 | + | u32 end = i + 1 == e->runs ? e->next_pc : e->run_end[i]; |
|
| 1928 | + | if (pc >= e->run_start[i] && pc < end) |
|
| 1929 | + | return true; |
|
| 1930 | + | } |
|
| 1931 | + | return false; |
|
| 1932 | + | } |
|
| 1933 | + | ||
| 1934 | + | /* Continue translating the block at `target` after a jump whose next |
|
| 1935 | + | * instruction is at `next`. */ |
|
| 1936 | + | static void follow(struct emitter *e, u32 next, u32 target) { |
|
| 1937 | + | e->run_end[e->runs - 1] = next; |
|
| 1938 | + | e->run_start[e->runs] = target; |
|
| 1939 | + | e->runs++; |
|
| 1940 | + | e->next_pc = target; |
|
| 1941 | + | } |
|
| 1942 | + | ||
| 1943 | + | /* Jump to the constant `target`, linking `next` in `rd`. Continues the |
|
| 1944 | + | * block at the target unless that would loop, and returns whether the block |
|
| 1945 | + | * ends. */ |
|
| 1946 | + | static bool emit_direct_jump(struct emitter *e, u32 rd, u32 next, u32 target) { |
|
| 1947 | + | if (rd != 0) { |
|
| 1948 | + | emit_mov_imm64(e, HTMP1, (u64)next); |
|
| 1949 | + | emit_store_guest(e, rd, HTMP1); |
|
| 1950 | + | } |
|
| 1951 | + | if (e->runs > JIT_MAX_FOLLOWS || translated(e, target)) { |
|
| 1952 | + | emit_chain_exit(e, target); |
|
| 1953 | + | return true; |
|
| 1954 | + | } |
|
| 1955 | + | /* A return to `next` can then be checked in place. */ |
|
| 1956 | + | if (rd == RA && next <= INT32_MAX) { |
|
| 1957 | + | e->ra_known = true; |
|
| 1958 | + | e->ra_value = next; |
|
| 1959 | + | } |
|
| 1960 | + | follow(e, next, target); |
|
| 1961 | + | return false; |
|
| 1962 | + | } |
|
| 1963 | + | ||
| 1964 | + | /* Return to ra: leave the program when it is zero, else jump to it. */ |
|
| 1965 | + | static void emit_ret(struct emitter *e) { |
|
| 1966 | + | emit_load_guest(e, HTMP1, RA); |
|
| 1967 | + | ||
| 1968 | + | /* test rax, rax */ |
|
| 1969 | + | emit_rexw(e); |
|
| 1970 | + | emit_u8(e, 0x85); |
|
| 1971 | + | emit_u8(e, modrm(3, X_RAX, X_RAX)); |
|
| 1972 | + | ||
| 1973 | + | u32 jnz_pos = emit_jnz_placeholder(e); |
|
| 1974 | + | ||
| 1975 | + | /* RA == 0: program exit. */ |
|
| 1976 | + | emit_block_exit(e, JIT_EXIT_RET, 0); |
|
| 1977 | + | ||
| 1978 | + | /* RA != 0: compute target = RA & ~1. */ |
|
| 1979 | + | patch_jcc(e, jnz_pos, e->pos); |
|
| 1980 | + | emit_rexw(e); |
|
| 1981 | + | emit_u8(e, 0x83); |
|
| 1982 | + | emit_u8(e, modrm(3, 4, X_RAX)); |
|
| 1983 | + | emit_u8(e, 0xFE); /* and rax, ~1 */ |
|
| 1984 | + | ||
| 1985 | + | emit_indirect_jump(e); |
|
| 1986 | + | } |
|
| 1987 | + | ||
| 1988 | + | /* Emit the side exits recorded for taken branches and checked returns. */ |
|
| 1989 | + | static void emit_side_exits(struct emitter *e) { |
|
| 1990 | + | /* Branches are never translated spilled. */ |
|
| 1991 | + | e->spilled = false; |
|
| 1992 | + | for (u32 i = 0; i < e->sides_len; i++) { |
|
| 1993 | + | patch_jcc(e, e->sides[i].jcc, e->pos); |
|
| 1994 | + | emit_refund(e, e->sides[i].insns); |
|
| 1995 | + | if (e->sides[i].ret) |
|
| 1996 | + | emit_ret(e); |
|
| 1997 | + | else |
|
| 1998 | + | emit_chain_exit(e, e->sides[i].target); |
|
| 1999 | + | } |
|
| 2000 | + | } |
|
| 2001 | + | ||
| 2002 | + | /* Leave the block through a side exit when the jcc `cc` (its second opcode |
|
| 2003 | + | * byte) holds: to `target`, or by returning to ra when `ret` is set. */ |
|
| 2004 | + | static void emit_side_exit(struct emitter *e, u8 cc, u32 target, bool ret) { |
|
| 2005 | + | e->sides[e->sides_len].jcc = e->pos; |
|
| 2006 | + | e->sides[e->sides_len].target = target; |
|
| 2007 | + | e->sides[e->sides_len].insns = e->insns; |
|
| 2008 | + | e->sides[e->sides_len].ret = ret; |
|
| 2009 | + | e->sides_len++; |
|
| 2010 | + | emit_u8(e, 0x0F); |
|
| 2011 | + | emit_u8(e, cc); |
|
| 2012 | + | emit_u32(e, 0); |
|
| 2013 | + | } |
|
| 2014 | + | ||
| 2015 | + | /* Whether the instruction at `pc` is ebreak. */ |
|
| 2016 | + | static bool traps(struct emitter *e, u32 pc) { |
|
| 2017 | + | if (pc < e->prog_base || pc >= e->prog_end) |
|
| 2018 | + | return false; |
|
| 2019 | + | u32 raw; |
|
| 2020 | + | memcpy(&raw, &e->mem[pc], sizeof(raw)); |
|
| 2021 | + | return raw == 0x00100073; |
|
| 2022 | + | } |
|
| 2023 | + | ||
| 2024 | + | /* ---------- Register-direct integer operations ------------------------- */ |
|
| 2025 | + | ||
| 2026 | + | /* Emit `opcode` (an "op r, r/m" form) with `dst` and guest register `src` as |
|
| 2027 | + | * the r/m operand, held or in the register file. `src` is not x0. */ |
|
| 2028 | + | static void emit_op_guest( |
|
| 2029 | + | struct emitter *e, bool wide, bool escape, u8 opcode, u8 dst, u32 src |
|
| 2030 | + | ) { |
|
| 2031 | + | u8 host = host_of(e, src); |
|
| 2032 | + | /* The register file is addressed through r12, which needs REX.B. */ |
|
| 2033 | + | emit_u8(e, rex(wide, dst >= 8, false, host ? host >= 8 : true)); |
|
| 2034 | + | if (escape) |
|
| 2035 | + | emit_u8(e, 0x0F); |
|
| 2036 | + | emit_u8(e, opcode); |
|
| 2037 | + | if (host) |
|
| 2038 | + | emit_u8(e, modrm(3, dst, host)); |
|
| 2039 | + | else |
|
| 2040 | + | emit_r12_address(e, dst, src * 8); |
|
| 2041 | + | } |
|
| 2042 | + | ||
| 2043 | + | /* Emit the group-1 operation `ext` (0 add, 1 or, 4 and, 5 sub, 6 xor, 7 cmp) |
|
| 2044 | + | * of `dst` with a sign-extended immediate. */ |
|
| 2045 | + | static void emit_op_imm(struct emitter *e, bool wide, u8 ext, u8 dst, i32 imm) { |
|
| 2046 | + | emit_u8(e, rex(wide, false, false, dst >= 8)); |
|
| 2047 | + | if (imm >= -128 && imm <= 127) { |
|
| 2048 | + | emit_u8(e, 0x83); |
|
| 2049 | + | emit_u8(e, modrm(3, ext, dst)); |
|
| 2050 | + | emit_u8(e, (u8)(i8)imm); |
|
| 2051 | + | } else { |
|
| 2052 | + | emit_u8(e, 0x81); |
|
| 2053 | + | emit_u8(e, modrm(3, ext, dst)); |
|
| 2054 | + | emit_u32(e, (u32)imm); |
|
| 2055 | + | } |
|
| 2056 | + | } |
|
| 2057 | + | ||
| 2058 | + | /* Emit the shift `ext` (4 shl, 5 shr, 7 sar) of `dst` by a constant. */ |
|
| 2059 | + | static void emit_shift_imm( |
|
| 2060 | + | struct emitter *e, bool wide, u8 ext, u8 dst, u8 amount |
|
| 2061 | + | ) { |
|
| 2062 | + | emit_u8(e, rex(wide, false, false, dst >= 8)); |
|
| 2063 | + | emit_u8(e, 0xC1); |
|
| 2064 | + | emit_u8(e, modrm(3, ext, dst)); |
|
| 2065 | + | emit_u8(e, amount); |
|
| 2066 | + | } |
|
| 2067 | + | ||
| 2068 | + | /* lea dst, [base + disp] (64-bit, or 32-bit when `wide` is clear). */ |
|
| 2069 | + | static void emit_lea(struct emitter *e, bool wide, u8 dst, u8 base, i32 disp) { |
|
| 2070 | + | if (wide || dst >= 8 || base >= 8) |
|
| 2071 | + | emit_u8(e, rex(wide, dst >= 8, false, base >= 8)); |
|
| 2072 | + | emit_u8(e, 0x8D); |
|
| 2073 | + | bool short_disp = disp >= -128 && disp <= 127; |
|
| 2074 | + | emit_u8(e, modrm(short_disp ? 1 : 2, dst, base)); |
|
| 2075 | + | if ((base & 7) == 4) |
|
| 2076 | + | emit_u8(e, sib(0, 4, base)); |
|
| 2077 | + | if (short_disp) |
|
| 2078 | + | emit_u8(e, (u8)(i8)disp); |
|
| 2079 | + | else |
|
| 2080 | + | emit_u32(e, (u32)disp); |
|
| 2081 | + | } |
|
| 2082 | + | ||
| 2083 | + | /* movsxd dst, src32 */ |
|
| 2084 | + | static void emit_movsxd(struct emitter *e, u8 dst, u8 src) { |
|
| 2085 | + | emit_u8(e, rex(true, dst >= 8, false, src >= 8)); |
|
| 2086 | + | emit_u8(e, 0x63); |
|
| 2087 | + | emit_u8(e, modrm(3, dst, src)); |
|
| 2088 | + | } |
|
| 2089 | + | ||
| 2090 | + | /* The shift extension of an immediate shift: funct3 1 shifts left, 5 right, |
|
| 2091 | + | * arithmetically when bit 10 of the immediate is set. */ |
|
| 2092 | + | static u8 shift_ext(u32 funct3, i32 imm) { |
|
| 2093 | + | return funct3 == 1 ? 4 : (imm & 0x400) ? 7 : 5; |
|
| 2094 | + | } |
|
| 2095 | + | ||
| 2096 | + | /* Host register to compute `rd` in: the one holding it, or rax. */ |
|
| 2097 | + | static u8 dest_of(struct emitter *e, u32 rd) { |
|
| 2098 | + | u8 d = host_of(e, rd); |
|
| 2099 | + | return d ? d : X_RAX; |
|
| 2100 | + | } |
|
| 2101 | + | ||
| 2102 | + | /* Write a result computed in `d`, chosen by dest_of(), to `rd`. */ |
|
| 2103 | + | static void emit_finish(struct emitter *e, u32 rd, u8 d) { |
|
| 2104 | + | if (d == X_RAX) |
|
| 2105 | + | emit_store_guest(e, rd, X_RAX); |
|
| 2106 | + | } |
|
| 2107 | + | ||
| 2108 | + | /* Translate lui, auipc and the integer operations with an immediate. */ |
|
| 2109 | + | static void emit_immediate(struct emitter *e, instr_t ins, u32 pc) { |
|
| 2110 | + | u32 rd = ins.i.rd, rs1 = ins.i.rs1, f3 = ins.i.funct3; |
|
| 2111 | + | i32 imm = get_i_imm(ins); |
|
| 2112 | + | if (rd == 0) |
|
| 2113 | + | return; |
|
| 2114 | + | u8 d = dest_of(e, rd); |
|
| 2115 | + | u8 src = host_of(e, rs1); |
|
| 2116 | + | ||
| 2117 | + | switch (ins.i.opcode) { |
|
| 2118 | + | case OP_LUI: |
|
| 2119 | + | emit_mov_imm32_sx(e, d, (i32)(ins.u.imm_31_12 << 12)); |
|
| 2120 | + | break; |
|
| 2121 | + | ||
| 2122 | + | case OP_AUIPC: { |
|
| 2123 | + | i64 value = (i64)pc + (i64)(i32)(ins.u.imm_31_12 << 12); |
|
| 2124 | + | if (value >= INT32_MIN && value <= INT32_MAX) |
|
| 2125 | + | emit_mov_imm32_sx(e, d, (i32)value); |
|
| 2126 | + | else |
|
| 2127 | + | emit_mov_imm64(e, d, (u64)value); |
|
| 2128 | + | break; |
|
| 2129 | + | } |
|
| 2130 | + | ||
| 2131 | + | case OP_IMM: |
|
| 2132 | + | if (f3 == 0 && rs1 == 0) { |
|
| 2133 | + | emit_mov_imm32_sx(e, d, imm); /* li */ |
|
| 2134 | + | break; |
|
| 2135 | + | } |
|
| 2136 | + | if (f3 == 0 && src) { |
|
| 2137 | + | if (imm) |
|
| 2138 | + | emit_lea(e, true, d, src, imm); |
|
| 2139 | + | else |
|
| 2140 | + | emit_mov_rr(e, d, src); /* mv */ |
|
| 2141 | + | break; |
|
| 2142 | + | } |
|
| 2143 | + | if (f3 == 2 || f3 == 3) { |
|
| 2144 | + | /* slti, sltiu: cmp a, imm; setl/setb al; movzx d, al */ |
|
| 2145 | + | u8 a = emit_base(e, rs1); |
|
| 2146 | + | emit_op_imm(e, true, 7, a, imm); |
|
| 2147 | + | emit_u8(e, 0x0F); |
|
| 2148 | + | emit_u8(e, f3 == 2 ? 0x9C : 0x92); |
|
| 2149 | + | emit_u8(e, modrm(3, 0, X_RAX)); |
|
| 2150 | + | if (d >= 8) |
|
| 2151 | + | emit_u8(e, rex(false, true, false, false)); |
|
| 2152 | + | emit_u8(e, 0x0F); |
|
| 2153 | + | emit_u8(e, 0xB6); |
|
| 2154 | + | emit_u8(e, modrm(3, d, X_RAX)); |
|
| 2155 | + | break; |
|
| 2156 | + | } |
|
| 2157 | + | emit_load_guest(e, d, rs1); |
|
| 2158 | + | switch (f3) { |
|
| 2159 | + | case 0x0: /* addi */ |
|
| 2160 | + | if (imm) |
|
| 2161 | + | emit_op_imm(e, true, 0, d, imm); |
|
| 2162 | + | break; |
|
| 2163 | + | case 0x1: /* slli */ |
|
| 2164 | + | case 0x5: /* srli, srai */ |
|
| 2165 | + | emit_shift_imm(e, true, shift_ext(f3, imm), d, (u8)(imm & 0x3F)); |
|
| 2166 | + | break; |
|
| 2167 | + | case 0x4: /* xori */ |
|
| 2168 | + | emit_op_imm(e, true, 6, d, imm); |
|
| 2169 | + | break; |
|
| 2170 | + | case 0x6: /* ori */ |
|
| 2171 | + | emit_op_imm(e, true, 1, d, imm); |
|
| 2172 | + | break; |
|
| 2173 | + | case 0x7: /* andi */ |
|
| 2174 | + | emit_op_imm(e, true, 4, d, imm); |
|
| 2175 | + | break; |
|
| 2176 | + | } |
|
| 2177 | + | break; |
|
| 2178 | + | ||
| 2179 | + | case OP_IMM_32: |
|
| 2180 | + | /* Compute in eax and sign-extend into d. */ |
|
| 2181 | + | if (f3 == 0 && src) { |
|
| 2182 | + | emit_lea(e, false, X_RAX, src, imm); /* addiw */ |
|
| 2183 | + | } else { |
|
| 2184 | + | emit_load_guest(e, X_RAX, rs1); |
|
| 2185 | + | if (f3 == 0) |
|
| 2186 | + | emit_op_imm(e, false, 0, X_RAX, imm); /* addiw */ |
|
| 2187 | + | else if (f3 == 1 || f3 == 5) |
|
| 2188 | + | emit_shift_imm( |
|
| 2189 | + | e, false, shift_ext(f3, imm), X_RAX, (u8)(imm & 0x1F) |
|
| 2190 | + | ); /* slliw, srliw, sraiw */ |
|
| 2191 | + | } |
|
| 2192 | + | emit_movsxd(e, d, X_RAX); |
|
| 2193 | + | break; |
|
| 2194 | + | } |
|
| 2195 | + | emit_finish(e, rd, d); |
|
| 2196 | + | } |
|
| 2197 | + | ||
| 2198 | + | /* Translate add, sub, xor, or, and, mul and the word forms of add, sub and |
|
| 2199 | + | * mul of two registers other than x0, on held registers in place. Returns |
|
| 2200 | + | * false for the other register operations. */ |
|
| 2201 | + | static bool emit_register_op(struct emitter *e, instr_t ins) { |
|
| 2202 | + | u32 rd = ins.r.rd, rs1 = ins.r.rs1, rs2 = ins.r.rs2; |
|
| 2203 | + | bool wide = ins.r.opcode == OP_OP; |
|
| 2204 | + | bool escape = false; |
|
| 2205 | + | u8 op; |
|
| 2206 | + | if (ins.r.funct7 == FUNCT7_NORMAL && ins.r.funct3 == 0) |
|
| 2207 | + | op = 0x03; /* add */ |
|
| 2208 | + | else if (ins.r.funct7 == FUNCT7_SUB && ins.r.funct3 == 0) |
|
| 2209 | + | op = 0x2B; /* sub */ |
|
| 2210 | + | else if (ins.r.funct7 == FUNCT7_MUL && ins.r.funct3 == 0) |
|
| 2211 | + | op = 0xAF, escape = true; /* imul */ |
|
| 2212 | + | else if (wide && ins.r.funct7 == FUNCT7_NORMAL && ins.r.funct3 == 4) |
|
| 2213 | + | op = 0x33; /* xor */ |
|
| 2214 | + | else if (wide && ins.r.funct7 == FUNCT7_NORMAL && ins.r.funct3 == 6) |
|
| 2215 | + | op = 0x0B; /* or */ |
|
| 2216 | + | else if (wide && ins.r.funct7 == FUNCT7_NORMAL && ins.r.funct3 == 7) |
|
| 2217 | + | op = 0x23; /* and */ |
|
| 2218 | + | else |
|
| 2219 | + | return false; |
|
| 2220 | + | if (rd == 0 || rs1 == 0 || rs2 == 0) |
|
| 2221 | + | return false; |
|
| 2222 | + | ||
| 2223 | + | /* Compute in d, unless it holds rs2, which the first operand would |
|
| 2224 | + | * overwrite, or the word result needs sign extension from eax. */ |
|
| 2225 | + | u8 d = dest_of(e, rd); |
|
| 2226 | + | u8 t = wide && (rd == rs1 || rd != rs2) ? d : X_RAX; |
|
| 2227 | + | if (t == X_RAX || rd != rs1) |
|
| 2228 | + | emit_load_guest(e, t, rs1); |
|
| 2229 | + | emit_op_guest(e, wide, escape, op, t, rs2); |
|
| 2230 | + | if (wide) |
|
| 2231 | + | emit_mov_rr(e, d, t); |
|
| 2232 | + | else |
|
| 2233 | + | emit_movsxd(e, d, X_RAX); |
|
| 2234 | + | emit_finish(e, rd, d); |
|
| 2235 | + | return true; |
|
| 2236 | + | } |
|
| 2237 | + | ||
| 2238 | + | /* Offset of the reservation in the exchange area. */ |
|
| 2239 | + | #define IO_RESERVATION ((u8)offsetof(struct jit_io, reservation)) |
|
| 2240 | + | ||
| 2241 | + | /* Translate LR and SC of a word or doubleword for a single hart. Only LR, |
|
| 2242 | + | * SC and AMOs touch the reservation, as in the interpreter. A misaligned |
|
| 2243 | + | * address leaves for the interpreter, which reports it. Returns false for |
|
| 2244 | + | * the operations left to the interpreter. */ |
|
| 2245 | + | static bool emit_reserved(struct emitter *e, instr_t ins, u32 pc) { |
|
| 2246 | + | u32 op = ins.raw >> 27; |
|
| 2247 | + | if ((ins.r.funct3 != 2 && ins.r.funct3 != 3) || (op != 2 && op != 3)) |
|
| 2248 | + | return false; |
|
| 2249 | + | u32 size = ins.r.funct3 == 2 ? 4 : 8; |
|
| 2250 | + | bool lr = op == 2; |
|
| 2251 | + | ||
| 2252 | + | emit_load_guest(e, X_RAX, ins.r.rs1); |
|
| 2253 | + | emit_u8(e, 0xA8); |
|
| 2254 | + | emit_u8(e, (u8)(size - 1)); /* test al, size - 1 */ |
|
| 2255 | + | emit_fault_if(e, pc, 0x85); |
|
| 2256 | + | emit_u8(e, 0x89); |
|
| 2257 | + | emit_u8(e, 0xC0); /* mov eax, eax */ |
|
| 2258 | + | ||
| 2259 | + | /* mov rcx, size << 32; or rcx, rax -- the reservation value. */ |
|
| 2260 | + | emit_mov_imm64(e, X_RCX, jit_reservation(0, size)); |
|
| 2261 | + | emit_u8(e, 0x48); |
|
| 2262 | + | emit_u8(e, 0x09); |
|
| 2263 | + | emit_u8(e, modrm(3, X_RAX, X_RCX)); |
|
| 2264 | + | ||
| 2265 | + | if (lr) { |
|
| 2266 | + | /* mov [r14 + IO_RESERVATION], rcx */ |
|
| 2267 | + | emit_u8(e, 0x49); |
|
| 2268 | + | emit_u8(e, 0x89); |
|
| 2269 | + | emit_u8(e, modrm(1, X_RCX, HIO)); |
|
| 2270 | + | emit_u8(e, IO_RESERVATION); |
|
| 2271 | + | if (ins.r.rd != 0) { |
|
| 2272 | + | u8 d = host_of(e, ins.r.rd); |
|
| 2273 | + | if (!d) |
|
| 2274 | + | d = X_RAX; |
|
| 2275 | + | /* movsxd d, [r13 + rax] or mov d, [r13 + rax] */ |
|
| 2276 | + | emit_guest_memory( |
|
| 2277 | + | e, false, true, false, size == 4 ? 0x63 : 0x8B, d, X_RAX, 0 |
|
| 2278 | + | ); |
|
| 2279 | + | if (d == X_RAX) |
|
| 2280 | + | emit_store_guest(e, ins.r.rd, X_RAX); |
|
| 2281 | + | } |
|
| 2282 | + | return true; |
|
| 2283 | + | } |
|
| 2284 | + | /* cmp [r14 + IO_RESERVATION], rcx */ |
|
| 2285 | + | emit_u8(e, 0x49); |
|
| 2286 | + | emit_u8(e, 0x39); |
|
| 2287 | + | emit_u8(e, modrm(1, X_RCX, HIO)); |
|
| 2288 | + | emit_u8(e, IO_RESERVATION); |
|
| 2289 | + | /* mov qword [r14 + IO_RESERVATION], 0 -- keeps the flags. */ |
|
| 2290 | + | emit_u8(e, 0x49); |
|
| 2291 | + | emit_u8(e, 0xC7); |
|
| 2292 | + | emit_u8(e, modrm(1, 0, HIO)); |
|
| 2293 | + | emit_u8(e, IO_RESERVATION); |
|
| 2294 | + | emit_u32(e, 0); |
|
| 2295 | + | u32 fail = emit_jne_placeholder(e); |
|
| 2296 | + | ||
| 2297 | + | emit_load_guest(e, X_RCX, ins.r.rs2); |
|
| 2298 | + | emit_guest_memory(e, false, size == 8, false, 0x89, X_RCX, X_RAX, 0); |
|
| 2299 | + | emit_mov_imm32_sx(e, X_RAX, 0); |
|
| 2300 | + | u32 done = emit_jmp_placeholder(e); |
|
| 2301 | + | ||
| 2302 | + | patch_jcc(e, fail, e->pos); |
|
| 2303 | + | emit_mov_imm32_sx(e, X_RAX, 1); |
|
| 2304 | + | patch_jmp(e, done, e->pos); |
|
| 2305 | + | emit_store_guest(e, ins.r.rd, X_RAX); |
|
| 2306 | + | return true; |
|
| 2307 | + | } |
|
| 2308 | + | ||
| 1746 | 2309 | /* ---------- Translate one RV64I instruction ----------------------------- */ |
|
| 1747 | 2310 | ||
| 1748 | 2311 | /* |
|
| 1749 | 2312 | * Returns: true if the instruction ends the basic block (branch/jump/ecall), |
|
| 1750 | 2313 | * false if execution should continue to the next instruction. |
|
| 1751 | 2314 | */ |
|
| 1752 | - | static bool translate_insn(struct emitter *e, instr_t ins, u32 pc) { |
|
| 2315 | + | static bool translate_body(struct emitter *e, instr_t ins, u32 pc) { |
|
| 1753 | 2316 | u32 opcode = ins.r.opcode; |
|
| 1754 | 2317 | u32 pc_next = pc + INSTR_SIZE; |
|
| 1755 | 2318 | ||
| 1756 | 2319 | enum bitmanip_op bm = bitmanip_decode(ins.raw); |
|
| 1757 | 2320 | if (bm != BM_NONE) { |
| 1773 | 2336 | return false; |
|
| 1774 | 2337 | ||
| 1775 | 2338 | switch (opcode) { |
|
| 1776 | 2339 | ||
| 1777 | 2340 | case OP_LUI: |
|
| 1778 | - | if (ins.u.rd != 0) { |
|
| 1779 | - | emit_mov_imm32_sx(e, HTMP1, (i32)(ins.u.imm_31_12 << 12)); |
|
| 1780 | - | emit_store_rd(e, ins.u.rd); |
|
| 1781 | - | } |
|
| 1782 | - | return false; |
|
| 1783 | - | ||
| 1784 | 2341 | case OP_AUIPC: |
|
| 1785 | - | if (ins.u.rd != 0) { |
|
| 1786 | - | i64 result = (i64)pc + (i64)(i32)(ins.u.imm_31_12 << 12); |
|
| 1787 | - | emit_mov_imm64(e, HTMP1, (u64)result); |
|
| 1788 | - | emit_store_rd(e, ins.u.rd); |
|
| 1789 | - | } |
|
| 2342 | + | case OP_IMM: |
|
| 2343 | + | case OP_IMM_32: |
|
| 2344 | + | emit_immediate(e, ins, pc); |
|
| 1790 | 2345 | return false; |
|
| 1791 | 2346 | ||
| 1792 | 2347 | case OP_JAL: { |
|
| 1793 | - | u32 target = pc + (u32)get_j_imm(ins); |
|
| 1794 | - | if (ins.j.rd != 0) { |
|
| 1795 | - | emit_mov_imm64(e, HTMP1, (u64)pc_next); |
|
| 1796 | - | emit_store_guest(e, ins.j.rd, HTMP1); |
|
| 1797 | - | } |
|
| 1798 | - | emit_chain_exit(e, target); |
|
| 1799 | - | return true; |
|
| 2348 | + | return emit_direct_jump(e, ins.j.rd, pc_next, pc + get_j_imm(ins)); |
|
| 1800 | 2349 | } |
|
| 1801 | 2350 | ||
| 1802 | 2351 | case OP_JALR: { |
|
| 1803 | 2352 | i32 imm = get_i_imm(ins); |
|
| 1804 | 2353 | ||
| 1805 | 2354 | /* Special-case RET: jalr x0, ra, 0. */ |
|
| 1806 | 2355 | if (ins.i.rd == 0 && ins.i.rs1 == RA && imm == 0) { |
|
| 1807 | - | emit_load_guest(e, HTMP1, RA); |
|
| 1808 | - | ||
| 1809 | - | /* test rax, rax */ |
|
| 1810 | - | emit_rexw(e); |
|
| 1811 | - | emit_u8(e, 0x85); |
|
| 1812 | - | emit_u8(e, modrm(3, X_RAX, X_RAX)); |
|
| 1813 | - | ||
| 1814 | - | u32 jnz_pos = emit_jnz_placeholder(e); |
|
| 1815 | - | ||
| 1816 | - | /* RA == 0: program exit. */ |
|
| 1817 | - | emit_block_exit(e, JIT_EXIT_RET, 0); |
|
| 1818 | - | ||
| 1819 | - | /* RA != 0: compute target = RA & ~1. */ |
|
| 1820 | - | patch_jcc(e, jnz_pos, e->pos); |
|
| 1821 | - | emit_rexw(e); |
|
| 1822 | - | emit_u8(e, 0x83); |
|
| 1823 | - | emit_u8(e, modrm(3, 4, X_RAX)); |
|
| 1824 | - | emit_u8(e, 0xFE); /* and rax, ~1 */ |
|
| 1825 | - | ||
| 1826 | - | emit_indirect_jump(e); |
|
| 2356 | + | /* Returning to the call this block followed: check that ra |
|
| 2357 | + | * still holds its return address and continue there. */ |
|
| 2358 | + | if (e->ra_known && e->sides_len < JIT_MAX_SIDE_EXITS && |
|
| 2359 | + | e->runs <= JIT_MAX_FOLLOWS && !translated(e, e->ra_value)) { |
|
| 2360 | + | emit_load_guest(e, HTMP1, RA); |
|
| 2361 | + | emit_rexw(e); |
|
| 2362 | + | emit_u8(e, 0x3D); |
|
| 2363 | + | emit_u32(e, e->ra_value); /* cmp rax, ra_value */ |
|
| 2364 | + | emit_side_exit(e, 0x85, 0, true); |
|
| 2365 | + | follow(e, pc_next, e->ra_value); |
|
| 2366 | + | return false; |
|
| 2367 | + | } |
|
| 2368 | + | emit_ret(e); |
|
| 1827 | 2369 | return true; |
|
| 1828 | 2370 | } |
|
| 1829 | 2371 | ||
| 2372 | + | /* A jump through the register the previous instruction set with |
|
| 2373 | + | * auipc has a constant target: jump there directly. */ |
|
| 2374 | + | if (e->auipc_reg && ins.i.rs1 == e->auipc_reg) { |
|
| 2375 | + | u32 target = (u32)(e->auipc_value + (u64)(i64)imm) & ~1u; |
|
| 2376 | + | return emit_direct_jump(e, ins.i.rd, pc_next, target); |
|
| 2377 | + | } |
|
| 2378 | + | ||
| 1830 | 2379 | /* General JALR: target = (rs1 + imm) & ~1. */ |
|
| 1831 | 2380 | emit_load_rs1(e, ins.i.rs1); |
|
| 1832 | 2381 | emit_add_rax_imm32(e, imm); |
|
| 1833 | 2382 | emit_rexw(e); |
|
| 1834 | 2383 | emit_u8(e, 0x83); |
| 1845 | 2394 | } |
|
| 1846 | 2395 | ||
| 1847 | 2396 | case OP_BRANCH: { |
|
| 1848 | 2397 | u32 target = pc + (u32)get_b_imm(ins); |
|
| 1849 | 2398 | ||
| 1850 | - | emit_load_rs1(e, ins.b.rs1); |
|
| 1851 | - | emit_load_rs2(e, ins.b.rs2); |
|
| 1852 | - | emit_cmp_rax_rcx(e); |
|
| 2399 | + | emit_compare(e, ins.b.rs1, ins.b.rs2); |
|
| 1853 | 2400 | ||
| 1854 | 2401 | /* jCC to .taken */ |
|
| 1855 | 2402 | u8 cc; |
|
| 1856 | 2403 | switch (ins.b.funct3) { |
|
| 1857 | 2404 | case 0x0: |
| 1875 | 2422 | default: |
|
| 1876 | 2423 | cc = 0x84; |
|
| 1877 | 2424 | break; |
|
| 1878 | 2425 | } |
|
| 1879 | 2426 | ||
| 2427 | + | /* Continue the block on the fall-through path; the taken path |
|
| 2428 | + | * leaves through a side exit after the block's main path. */ |
|
| 2429 | + | if (e->sides_len < JIT_MAX_SIDE_EXITS) { |
|
| 2430 | + | /* A branch around a trap, as in a bounds check, is taken: continue |
|
| 2431 | + | * at the target and leave to the trap through the side exit, |
|
| 2432 | + | * with the inverse condition. */ |
|
| 2433 | + | if (traps(e, pc_next) && e->runs <= JIT_MAX_FOLLOWS && |
|
| 2434 | + | !translated(e, target)) { |
|
| 2435 | + | emit_side_exit(e, cc ^ 1, pc_next, false); |
|
| 2436 | + | follow(e, pc_next, target); |
|
| 2437 | + | return false; |
|
| 2438 | + | } |
|
| 2439 | + | emit_side_exit(e, cc, target, false); |
|
| 2440 | + | return false; |
|
| 2441 | + | } |
|
| 2442 | + | ||
| 1880 | 2443 | u32 jcc_pos = e->pos; |
|
| 1881 | 2444 | emit_u8(e, 0x0F); |
|
| 1882 | 2445 | emit_u8(e, cc); |
|
| 1883 | 2446 | emit_u32(e, 0); |
|
| 1884 | 2447 |
| 1889 | 2452 | patch_jcc(e, jcc_pos, e->pos); |
|
| 1890 | 2453 | emit_chain_exit(e, target); |
|
| 1891 | 2454 | return true; |
|
| 1892 | 2455 | } |
|
| 1893 | 2456 | ||
| 1894 | - | case OP_LOAD: { |
|
| 1895 | - | i32 imm = get_i_imm(ins); |
|
| 1896 | - | if (ins.i.rd == 0) |
|
| 2457 | + | case OP_LOAD: |
|
| 2458 | + | if (emit_scalar_load(e, ins)) |
|
| 1897 | 2459 | return false; |
|
| 2460 | + | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 2461 | + | return true; |
|
| 1898 | 2462 | ||
| 1899 | - | emit_load_rs1(e, ins.i.rs1); |
|
| 1900 | - | emit_add_rax_imm32(e, imm); |
|
| 1901 | - | ||
| 1902 | - | switch (ins.i.funct3) { |
|
| 1903 | - | case 0x0: |
|
| 1904 | - | emit_load_mem_i8(e); |
|
| 1905 | - | break; /* lb */ |
|
| 1906 | - | case 0x1: |
|
| 1907 | - | emit_load_mem_i16(e); |
|
| 1908 | - | break; /* lh */ |
|
| 1909 | - | case 0x2: |
|
| 1910 | - | emit_load_mem_i32(e); |
|
| 1911 | - | break; /* lw */ |
|
| 1912 | - | case 0x3: |
|
| 1913 | - | emit_load_mem_u64(e); |
|
| 1914 | - | break; /* ld */ |
|
| 1915 | - | case 0x4: |
|
| 1916 | - | emit_load_mem_u8(e); |
|
| 1917 | - | break; /* lbu */ |
|
| 1918 | - | case 0x5: |
|
| 1919 | - | emit_load_mem_u16(e); |
|
| 1920 | - | break; /* lhu */ |
|
| 1921 | - | case 0x6: |
|
| 1922 | - | emit_load_mem_u32(e); |
|
| 1923 | - | break; /* lwu */ |
|
| 1924 | - | default: |
|
| 1925 | - | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 1926 | - | return true; |
|
| 1927 | - | } |
|
| 1928 | - | emit_store_rd(e, ins.i.rd); |
|
| 1929 | - | return false; |
|
| 1930 | - | } |
|
| 1931 | - | ||
| 1932 | - | case OP_STORE: { |
|
| 1933 | - | i32 imm = get_s_imm(ins); |
|
| 1934 | - | emit_load_guest(e, HTMP1, ins.s.rs1); |
|
| 1935 | - | emit_add_rax_imm32(e, imm); |
|
| 1936 | - | emit_load_guest(e, HTMP2, ins.s.rs2); |
|
| 1937 | - | ||
| 1938 | - | switch (ins.s.funct3) { |
|
| 1939 | - | case 0x0: |
|
| 1940 | - | emit_store_mem_u8(e); |
|
| 1941 | - | break; /* sb */ |
|
| 1942 | - | case 0x1: |
|
| 1943 | - | emit_store_mem_u16(e); |
|
| 1944 | - | break; /* sh */ |
|
| 1945 | - | case 0x2: |
|
| 1946 | - | emit_store_mem_u32(e); |
|
| 1947 | - | break; /* sw */ |
|
| 1948 | - | case 0x3: |
|
| 1949 | - | emit_store_mem_u64(e); |
|
| 1950 | - | break; /* sd */ |
|
| 1951 | - | default: |
|
| 1952 | - | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 1953 | - | return true; |
|
| 1954 | - | } |
|
| 1955 | - | return false; |
|
| 1956 | - | } |
|
| 1957 | - | ||
| 1958 | - | case OP_IMM: { |
|
| 1959 | - | i32 imm = get_i_imm(ins); |
|
| 1960 | - | if (ins.i.rd == 0) |
|
| 1961 | - | return false; |
|
| 1962 | - | ||
| 1963 | - | emit_load_rs1(e, ins.i.rs1); |
|
| 1964 | - | ||
| 1965 | - | switch (ins.i.funct3) { |
|
| 1966 | - | case 0x0: /* addi */ |
|
| 1967 | - | emit_add_rax_imm32(e, imm); |
|
| 1968 | - | break; |
|
| 1969 | - | case 0x1: /* slli */ |
|
| 1970 | - | emit_rexw(e); |
|
| 1971 | - | emit_u8(e, 0xC1); |
|
| 1972 | - | emit_u8(e, modrm(3, 4, X_RAX)); |
|
| 1973 | - | emit_u8(e, (u8)(imm & 0x3F)); |
|
| 1974 | - | break; |
|
| 1975 | - | case 0x2: /* slti */ |
|
| 1976 | - | emit_mov_imm32_sx(e, HTMP2, imm); |
|
| 1977 | - | emit_cmp_rax_rcx(e); |
|
| 1978 | - | emit_setcc_rax(e, 0x9C); /* setl */ |
|
| 1979 | - | break; |
|
| 1980 | - | case 0x3: /* sltiu */ |
|
| 1981 | - | emit_mov_imm32_sx(e, HTMP2, imm); |
|
| 1982 | - | emit_cmp_rax_rcx(e); |
|
| 1983 | - | emit_setcc_rax(e, 0x92); /* setb */ |
|
| 1984 | - | break; |
|
| 1985 | - | case 0x4: /* xori */ |
|
| 1986 | - | emit_mov_imm32_sx(e, HTMP2, imm); |
|
| 1987 | - | emit_alu_rax_rcx(e, 0x31); /* xor */ |
|
| 1988 | - | break; |
|
| 1989 | - | case 0x5: /* srli/srai */ |
|
| 1990 | - | emit_rexw(e); |
|
| 1991 | - | emit_u8(e, 0xC1); |
|
| 1992 | - | emit_u8(e, modrm(3, (imm & 0x400) ? 7 : 5, X_RAX)); |
|
| 1993 | - | emit_u8(e, (u8)(imm & 0x3F)); |
|
| 1994 | - | break; |
|
| 1995 | - | case 0x6: /* ori */ |
|
| 1996 | - | emit_mov_imm32_sx(e, HTMP2, imm); |
|
| 1997 | - | emit_alu_rax_rcx(e, 0x09); /* or */ |
|
| 1998 | - | break; |
|
| 1999 | - | case 0x7: /* andi */ |
|
| 2000 | - | emit_mov_imm32_sx(e, HTMP2, imm); |
|
| 2001 | - | emit_alu_rax_rcx(e, 0x21); /* and */ |
|
| 2002 | - | break; |
|
| 2003 | - | } |
|
| 2004 | - | emit_store_rd(e, ins.i.rd); |
|
| 2005 | - | return false; |
|
| 2006 | - | } |
|
| 2007 | - | ||
| 2008 | - | case OP_IMM_32: { |
|
| 2009 | - | i32 imm = get_i_imm(ins); |
|
| 2010 | - | if (ins.i.rd == 0) |
|
| 2463 | + | case OP_STORE: |
|
| 2464 | + | if (emit_scalar_store(e, ins)) |
|
| 2011 | 2465 | return false; |
|
| 2012 | - | ||
| 2013 | - | emit_load_rs1(e, ins.i.rs1); |
|
| 2014 | - | ||
| 2015 | - | switch (ins.i.funct3) { |
|
| 2016 | - | case 0x0: /* addiw */ |
|
| 2017 | - | emit_add_rax_imm32(e, imm); |
|
| 2018 | - | break; |
|
| 2019 | - | case 0x1: /* slliw */ |
|
| 2020 | - | emit_u8(e, 0xC1); |
|
| 2021 | - | emit_u8(e, modrm(3, 4, X_RAX)); |
|
| 2022 | - | emit_u8(e, (u8)(imm & 0x1F)); |
|
| 2023 | - | break; |
|
| 2024 | - | case 0x5: /* srliw/sraiw */ |
|
| 2025 | - | emit_u8(e, 0xC1); |
|
| 2026 | - | emit_u8(e, modrm(3, (imm & 0x400) ? 7 : 5, X_RAX)); |
|
| 2027 | - | emit_u8(e, (u8)(imm & 0x1F)); |
|
| 2028 | - | break; |
|
| 2029 | - | } |
|
| 2030 | - | emit_movsxd_rax_eax(e); |
|
| 2031 | - | emit_store_rd(e, ins.i.rd); |
|
| 2032 | - | return false; |
|
| 2033 | - | } |
|
| 2466 | + | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 2467 | + | return true; |
|
| 2034 | 2468 | ||
| 2035 | 2469 | case OP_OP: { |
|
| 2036 | - | if (ins.r.rd == 0) |
|
| 2470 | + | if (emit_register_op(e, ins) || ins.r.rd == 0) |
|
| 2037 | 2471 | return false; |
|
| 2038 | 2472 | emit_load_rs1(e, ins.r.rs1); |
|
| 2039 | 2473 | emit_load_rs2(e, ins.r.rs2); |
|
| 2040 | 2474 | ||
| 2041 | 2475 | switch (ins.r.funct7) { |
| 2112 | 2546 | emit_store_rd(e, ins.r.rd); |
|
| 2113 | 2547 | return false; |
|
| 2114 | 2548 | } |
|
| 2115 | 2549 | ||
| 2116 | 2550 | case OP_OP_32: { |
|
| 2117 | - | if (ins.r.rd == 0) |
|
| 2551 | + | if (emit_register_op(e, ins) || ins.r.rd == 0) |
|
| 2118 | 2552 | return false; |
|
| 2119 | 2553 | emit_load_rs1(e, ins.r.rs1); |
|
| 2120 | 2554 | emit_load_rs2(e, ins.r.rs2); |
|
| 2121 | 2555 | ||
| 2122 | 2556 | switch (ins.r.funct7) { |
| 2194 | 2628 | ||
| 2195 | 2629 | case OP_FENCE: |
|
| 2196 | 2630 | return false; |
|
| 2197 | 2631 | ||
| 2198 | 2632 | case OP_AMO: |
|
| 2199 | - | /* Atomics are rare; the interpreter runs them. */ |
|
| 2633 | + | if (emit_reserved(e, ins, pc)) |
|
| 2634 | + | return false; |
|
| 2635 | + | /* Other atomics are rare; the interpreter runs them. */ |
|
| 2200 | 2636 | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 2201 | 2637 | return true; |
|
| 2202 | 2638 | ||
| 2203 | 2639 | default: |
|
| 2204 | 2640 | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 2205 | 2641 | return true; |
|
| 2206 | 2642 | } |
|
| 2207 | 2643 | } |
|
| 2208 | 2644 | ||
| 2645 | + | /* Whether an instruction is translated by an emitter that addresses the |
|
| 2646 | + | * register file directly or clobbers host registers other than rax, rcx and |
|
| 2647 | + | * those in `scratch_mask`. */ |
|
| 2648 | + | static bool needs_spill(instr_t ins) { |
|
| 2649 | + | u32 opcode = ins.r.opcode; |
|
| 2650 | + | if (opcode == 0x57 || opcode == 7 || opcode == 39) |
|
| 2651 | + | return true; |
|
| 2652 | + | switch (bitmanip_decode(ins.raw)) { |
|
| 2653 | + | case BM_NONE: |
|
| 2654 | + | return false; |
|
| 2655 | + | case BM_SEXTB: |
|
| 2656 | + | case BM_SEXTH: |
|
| 2657 | + | case BM_ZEXTH: |
|
| 2658 | + | case BM_ORCB: |
|
| 2659 | + | case BM_CPOP: |
|
| 2660 | + | case BM_CPOPW: |
|
| 2661 | + | return true; |
|
| 2662 | + | default: |
|
| 2663 | + | return false; |
|
| 2664 | + | } |
|
| 2665 | + | } |
|
| 2666 | + | ||
| 2667 | + | /* Host registers other than rax and rcx that an instruction clobbers: the |
|
| 2668 | + | * high multiplications and divisions use rdx and r11. */ |
|
| 2669 | + | static u32 scratch_mask(instr_t ins) { |
|
| 2670 | + | u32 opcode = ins.r.opcode; |
|
| 2671 | + | if ((opcode == OP_OP || opcode == OP_OP_32) && ins.r.funct7 == FUNCT7_MUL && |
|
| 2672 | + | ins.r.funct3 != 0 && bitmanip_decode(ins.raw) == BM_NONE) |
|
| 2673 | + | return 1u << X_RDX | 1u << X_R11; |
|
| 2674 | + | return 0; |
|
| 2675 | + | } |
|
| 2676 | + | ||
| 2677 | + | /* Whether a scalar instruction may be translated with every guest register |
|
| 2678 | + | * in the register file: it addresses them only through host_of(), clobbers |
|
| 2679 | + | * nothing beyond rax and rcx, and does not end the block. */ |
|
| 2680 | + | static bool spill_neutral(instr_t ins) { |
|
| 2681 | + | switch (ins.r.opcode) { |
|
| 2682 | + | case OP_LUI: |
|
| 2683 | + | case OP_AUIPC: |
|
| 2684 | + | case OP_IMM: |
|
| 2685 | + | case OP_IMM_32: |
|
| 2686 | + | case OP_LOAD: |
|
| 2687 | + | case OP_STORE: |
|
| 2688 | + | return true; |
|
| 2689 | + | case OP_OP: |
|
| 2690 | + | case OP_OP_32: |
|
| 2691 | + | return scratch_mask(ins) == 0; |
|
| 2692 | + | default: |
|
| 2693 | + | return false; |
|
| 2694 | + | } |
|
| 2695 | + | } |
|
| 2696 | + | ||
| 2697 | + | /* Instructions a spill-neutral run may span before the next instruction that |
|
| 2698 | + | * needs the register file, so that it is cheaper to stay spilled. */ |
|
| 2699 | + | #define SPILL_LOOKAHEAD 4 |
|
| 2700 | + | ||
| 2701 | + | /* Whether an instruction that needs the register file follows within a few |
|
| 2702 | + | * spill-neutral instructions of `pc`. */ |
|
| 2703 | + | static bool spill_follows(struct emitter *e, u32 pc) { |
|
| 2704 | + | for (u32 i = 1; i <= SPILL_LOOKAHEAD; i++) { |
|
| 2705 | + | u32 next = pc + i * INSTR_SIZE; |
|
| 2706 | + | if (next < e->prog_base || next >= e->prog_end) |
|
| 2707 | + | return false; |
|
| 2708 | + | instr_t ins; |
|
| 2709 | + | memcpy(&ins, &e->mem[next], sizeof(ins)); |
|
| 2710 | + | if (needs_spill(ins)) |
|
| 2711 | + | return true; |
|
| 2712 | + | if (!spill_neutral(ins)) |
|
| 2713 | + | return false; |
|
| 2714 | + | } |
|
| 2715 | + | return false; |
|
| 2716 | + | } |
|
| 2717 | + | ||
| 2718 | + | /* Translate one instruction, keeping the held guest registers coherent |
|
| 2719 | + | * around those whose emitters address the register file or clobber them. */ |
|
| 2720 | + | static bool translate_insn(struct emitter *e, instr_t ins, u32 pc) { |
|
| 2721 | + | /* Consecutive such instructions share one spill and reload. */ |
|
| 2722 | + | if (needs_spill(ins)) { |
|
| 2723 | + | if (!e->spilled) |
|
| 2724 | + | emit_spill(e); |
|
| 2725 | + | return translate_body(e, ins, pc); |
|
| 2726 | + | } |
|
| 2727 | + | if (e->spilled && spill_neutral(ins) && spill_follows(e, pc)) |
|
| 2728 | + | return translate_body(e, ins, pc); |
|
| 2729 | + | if (e->spilled) |
|
| 2730 | + | emit_reload(e); |
|
| 2731 | + | u32 mask = scratch_mask(ins); |
|
| 2732 | + | u32 rd = ins.r.rd; |
|
| 2733 | + | if (!mask || rd == 0) |
|
| 2734 | + | return translate_body(e, ins, pc); |
|
| 2735 | + | ||
| 2736 | + | /* These clobber the scratch registers after reading their operands and |
|
| 2737 | + | * write the destination last: save the scratch registers and restore |
|
| 2738 | + | * all but the destination. */ |
|
| 2739 | + | emit_transfer(e, mask, true); |
|
| 2740 | + | bool ends = translate_body(e, ins, pc); |
|
| 2741 | + | emit_transfer(e, mask & ~(1u << host_of(e, rd)), false); |
|
| 2742 | + | return ends; |
|
| 2743 | + | } |
|
| 2744 | + | ||
| 2209 | 2745 | /* ---------- Block compiler --------------------------------------------- */ |
|
| 2210 | 2746 | ||
| 2211 | 2747 | static struct jit_block *jit_alloc_block(struct jit_state *jit) { |
|
| 2212 | 2748 | if (jit->block_count >= JIT_MAX_BLOCKS) |
|
| 2213 | 2749 | return NULL; |
| 2218 | 2754 | u32 h = (block->guest_pc >> 2) & (JIT_BLOCK_HASH_SIZE - 1); |
|
| 2219 | 2755 | block->hash_next = jit->block_hash[h]; |
|
| 2220 | 2756 | jit->block_hash[h] = block; |
|
| 2221 | 2757 | } |
|
| 2222 | 2758 | ||
| 2223 | - | struct jit_block *jit_compile_block( |
|
| 2224 | - | struct jit_state *jit, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes |
|
| 2225 | - | ) { |
|
| 2226 | - | ||
| 2227 | - | #define JIT_EMIT_BUF_SIZE (JIT_MAX_BLOCK_INSNS * JIT_MAX_INSN_BYTES + 256) |
|
| 2228 | - | static u8 emit_buf[JIT_EMIT_BUF_SIZE]; |
|
| 2229 | - | ||
| 2230 | - | struct emitter em = { |
|
| 2231 | - | .buf = emit_buf, |
|
| 2232 | - | .pos = 0, |
|
| 2233 | - | .capacity = JIT_EMIT_BUF_SIZE, |
|
| 2234 | - | .overflow = false, |
|
| 2235 | - | .jit = jit, |
|
| 2236 | - | .dest = jit->code_cache + jit->code_cache_used, |
|
| 2237 | - | }; |
|
| 2759 | + | /* Whether an instruction may write ra. Conservative: any instruction with |
|
| 2760 | + | * ra in the destination field except stores and branches, which have none. |
|
| 2761 | + | */ |
|
| 2762 | + | static bool writes_ra(instr_t ins) { |
|
| 2763 | + | u32 opcode = ins.r.opcode; |
|
| 2764 | + | if (opcode == OP_STORE || opcode == OP_BRANCH || opcode == 39) |
|
| 2765 | + | return false; |
|
| 2766 | + | return ins.r.rd == RA; |
|
| 2767 | + | } |
|
| 2238 | 2768 | ||
| 2769 | + | /* Translate the block at `guest_pc` into `em`. The vector shape guard is |
|
| 2770 | + | * emitted only when `em->guard` is set. */ |
|
| 2771 | + | static void translate_block( |
|
| 2772 | + | struct emitter *em, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes |
|
| 2773 | + | ) { |
|
| 2239 | 2774 | /* if (budget <= 0) goto .exhausted */ |
|
| 2240 | - | emit_u8(&em, 0x4D); |
|
| 2241 | - | emit_u8(&em, 0x85); |
|
| 2242 | - | emit_u8(&em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */ |
|
| 2243 | - | u32 jle_pos = emit_jle_placeholder(&em); |
|
| 2775 | + | emit_u8(em, 0x4D); |
|
| 2776 | + | emit_u8(em, 0x85); |
|
| 2777 | + | emit_u8(em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */ |
|
| 2778 | + | u32 jle_pos = emit_jle_placeholder(em); |
|
| 2244 | 2779 | ||
| 2245 | - | u32 skip_vector = emit_jmp_placeholder(&em); |
|
| 2246 | - | u32 vector_guard = em.pos; |
|
| 2247 | - | emit_vector_guard(&em, guest_pc); |
|
| 2248 | - | u32 vector_body = em.pos; |
|
| 2780 | + | if (em->guard) |
|
| 2781 | + | emit_vector_guard(em, guest_pc); |
|
| 2249 | 2782 | ||
| 2250 | 2783 | /* sub r15, insn_count -- imm32 is patched once the count is known. */ |
|
| 2251 | - | emit_u8(&em, 0x49); |
|
| 2252 | - | emit_u8(&em, 0x81); |
|
| 2253 | - | emit_u8(&em, modrm(3, 5, HBUDGET)); |
|
| 2254 | - | u32 count_pos = em.pos; |
|
| 2255 | - | emit_u32(&em, 0); |
|
| 2256 | - | ||
| 2257 | - | u32 pc = guest_pc; |
|
| 2258 | - | u32 insn_count = 0; |
|
| 2259 | - | u32 prog_end_pc = prog_base + prog_bytes; |
|
| 2784 | + | emit_u8(em, 0x49); |
|
| 2785 | + | emit_u8(em, 0x81); |
|
| 2786 | + | emit_u8(em, modrm(3, 5, HBUDGET)); |
|
| 2787 | + | u32 count_pos = em->pos; |
|
| 2788 | + | emit_u32(em, 0); |
|
| 2789 | + | ||
| 2790 | + | u32 pc = guest_pc; |
|
| 2791 | + | u32 insn_count = 0; |
|
| 2792 | + | em->run_start[0] = guest_pc; |
|
| 2793 | + | em->runs = 1; |
|
| 2794 | + | u32 prog_end_pc = prog_base + prog_bytes; |
|
| 2260 | 2795 | ||
| 2261 | 2796 | while (insn_count < JIT_MAX_BLOCK_INSNS) { |
|
| 2262 | 2797 | if (pc < prog_base || pc >= prog_end_pc) { |
|
| 2263 | - | emit_block_exit(&em, JIT_EXIT_FAULT, pc); |
|
| 2798 | + | emit_block_exit(em, JIT_EXIT_FAULT, pc); |
|
| 2264 | 2799 | break; |
|
| 2265 | 2800 | } |
|
| 2266 | 2801 | instr_t ins; |
|
| 2267 | 2802 | memcpy(&ins, &mem[pc], sizeof(instr_t)); |
|
| 2268 | 2803 | insn_count++; |
|
| 2269 | - | em.insns = insn_count; |
|
| 2270 | - | ||
| 2271 | - | bool ends_block = translate_insn(&em, ins, pc); |
|
| 2272 | - | pc += INSTR_SIZE; |
|
| 2804 | + | em->insns = insn_count; |
|
| 2805 | + | ||
| 2806 | + | if (writes_ra(ins)) |
|
| 2807 | + | em->ra_known = false; |
|
| 2808 | + | em->next_pc = pc + INSTR_SIZE; |
|
| 2809 | + | bool ends_block = translate_insn(em, ins, pc); |
|
| 2810 | + | ||
| 2811 | + | /* Remember the value of an auipc for a jalr right after it. */ |
|
| 2812 | + | em->auipc_reg = 0; |
|
| 2813 | + | if (ins.u.opcode == OP_AUIPC && ins.u.rd != 0) { |
|
| 2814 | + | em->auipc_reg = ins.u.rd; |
|
| 2815 | + | em->auipc_value = pc + (u64)(i64)(i32)(ins.u.imm_31_12 << 12); |
|
| 2816 | + | } |
|
| 2817 | + | pc = em->next_pc; |
|
| 2273 | 2818 | ||
| 2274 | 2819 | if (ends_block) |
|
| 2275 | 2820 | break; |
|
| 2276 | 2821 | ||
| 2277 | - | if (em.pos + JIT_MAX_INSN_BYTES + 64 > em.capacity) { |
|
| 2278 | - | emit_chain_exit(&em, pc); |
|
| 2822 | + | if (em->pos + JIT_MAX_INSN_BYTES + 64 > em->capacity) { |
|
| 2823 | + | emit_chain_exit(em, pc); |
|
| 2279 | 2824 | break; |
|
| 2280 | 2825 | } |
|
| 2281 | 2826 | } |
|
| 2282 | 2827 | ||
| 2283 | 2828 | if (insn_count >= JIT_MAX_BLOCK_INSNS) { |
|
| 2284 | - | emit_chain_exit(&em, pc); |
|
| 2829 | + | emit_chain_exit(em, pc); |
|
| 2285 | 2830 | } |
|
| 2286 | 2831 | ||
| 2832 | + | /* The side exits and the budget exit start with held registers. */ |
|
| 2833 | + | emit_side_exits(em); |
|
| 2834 | + | em->spilled = false; |
|
| 2835 | + | ||
| 2287 | 2836 | /* .exhausted: out of budget, leave before running anything. */ |
|
| 2288 | - | patch_jcc(&em, jle_pos, em.pos); |
|
| 2289 | - | emit_block_exit(&em, JIT_EXIT_BRANCH, guest_pc); |
|
| 2837 | + | patch_jcc(em, jle_pos, em->pos); |
|
| 2838 | + | emit_block_exit(em, JIT_EXIT_BRANCH, guest_pc); |
|
| 2290 | 2839 | ||
| 2291 | - | patch_jmp(&em, skip_vector, em.vector ? vector_guard : vector_body); |
|
| 2292 | - | if (em.overflow) |
|
| 2293 | - | return NULL; |
|
| 2294 | 2840 | for (u32 i = 0; i < 4; i++) |
|
| 2295 | - | em.buf[count_pos + i] = (u8)(insn_count >> (8 * i)); |
|
| 2841 | + | em->buf[count_pos + i] = (u8)(insn_count >> (8 * i)); |
|
| 2296 | 2842 | ||
| 2297 | - | for (u32 i = 0; i < em.refunds_len; i++) { |
|
| 2298 | - | u32 refund = insn_count - em.consumed[i]; |
|
| 2843 | + | for (u32 i = 0; i < em->refunds_len; i++) { |
|
| 2844 | + | u32 refund = insn_count - em->consumed[i]; |
|
| 2299 | 2845 | for (u32 j = 0; j < 4; j++) |
|
| 2300 | - | em.buf[em.refunds[i] + j] = (u8)(refund >> (8 * j)); |
|
| 2846 | + | em->buf[em->refunds[i] + j] = (u8)(refund >> (8 * j)); |
|
| 2847 | + | } |
|
| 2848 | + | } |
|
| 2849 | + | ||
| 2850 | + | struct jit_block *jit_compile_block( |
|
| 2851 | + | struct jit_state *jit, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes |
|
| 2852 | + | ) { |
|
| 2853 | + | ||
| 2854 | + | #define JIT_EMIT_BUF_SIZE (JIT_MAX_BLOCK_INSNS * JIT_MAX_INSN_BYTES + 256) |
|
| 2855 | + | static u8 emit_buf[JIT_EMIT_BUF_SIZE]; |
|
| 2856 | + | static struct emitter em; |
|
| 2857 | + | ||
| 2858 | + | /* A block that turns out to use four-lane SIMD is translated again with |
|
| 2859 | + | * the guard that checks the vector shape on entry. */ |
|
| 2860 | + | for (u32 pass = 0; pass < 2; pass++) { |
|
| 2861 | + | memset(&em, 0, sizeof(em)); |
|
| 2862 | + | em.buf = emit_buf; |
|
| 2863 | + | em.capacity = JIT_EMIT_BUF_SIZE; |
|
| 2864 | + | em.jit = jit; |
|
| 2865 | + | em.dest = jit->code_cache + jit->code_cache_used; |
|
| 2866 | + | em.guard = pass == 1; |
|
| 2867 | + | em.mem = mem; |
|
| 2868 | + | em.prog_base = prog_base; |
|
| 2869 | + | em.prog_end = prog_base + prog_bytes; |
|
| 2870 | + | ||
| 2871 | + | translate_block(&em, guest_pc, mem, prog_base, prog_bytes); |
|
| 2872 | + | if (!em.vector) |
|
| 2873 | + | break; |
|
| 2301 | 2874 | } |
|
| 2875 | + | if (em.overflow) |
|
| 2876 | + | return NULL; |
|
| 2877 | + | ||
| 2302 | 2878 | u32 code_size = em.pos; |
|
| 2303 | 2879 | if (jit->code_cache_used + code_size > JIT_CODE_CACHE_SIZE) |
|
| 2304 | 2880 | return NULL; |
|
| 2305 | 2881 | ||
| 2306 | 2882 | u8 *dest = jit->code_cache + jit->code_cache_used; |
| 2328 | 2904 | site[i] = (u8)(rel >> (8 * i)); |
|
| 2329 | 2905 | } |
|
| 2330 | 2906 | ||
| 2331 | 2907 | /* ---------- Init / Destroy -------------------------------------------- */ |
|
| 2332 | 2908 | ||
| 2333 | - | /* Emit the shared exit stub and entry trampoline at the cache start. */ |
|
| 2909 | + | /* Emit the shared exit stubs and entry trampoline at the cache start. */ |
|
| 2334 | 2910 | static void jit_emit_trampolines(struct jit_state *jit) { |
|
| 2335 | - | u8 buf[128]; |
|
| 2911 | + | u8 buf[512]; |
|
| 2336 | 2912 | struct emitter e = { |
|
| 2337 | 2913 | .buf = buf, |
|
| 2338 | 2914 | .pos = 0, |
|
| 2339 | 2915 | .capacity = sizeof(buf), |
|
| 2340 | 2916 | .overflow = false, |
|
| 2341 | 2917 | .jit = jit, |
|
| 2342 | 2918 | .dest = jit->code_cache, |
|
| 2343 | 2919 | }; |
|
| 2344 | 2920 | ||
| 2345 | - | /* Exit stub: reason is in eax. */ |
|
| 2921 | + | /* Exit stub: reason is in eax. Write back the held guest registers. */ |
|
| 2346 | 2922 | jit->exit_stub = jit->code_cache; |
|
| 2923 | + | emit_transfer(&e, ALL_HOSTS, true); |
|
| 2924 | + | ||
| 2925 | + | /* Raw exit: the register file is current. */ |
|
| 2926 | + | jit->exit_raw = jit->code_cache + e.pos; |
|
| 2347 | 2927 | emit_u8(&e, 0x4D); |
|
| 2348 | 2928 | emit_u8(&e, 0x89); |
|
| 2349 | 2929 | emit_u8(&e, modrm(1, HBUDGET, HIO)); |
|
| 2350 | 2930 | emit_u8(&e, IO_BUDGET); /* mov [r14 + IO_BUDGET], r15 */ |
|
| 2351 | 2931 | emit_u8(&e, 0x41); |
| 2354 | 2934 | emit_u8(&e, 0x5E); /* pop r14 */ |
|
| 2355 | 2935 | emit_u8(&e, 0x41); |
|
| 2356 | 2936 | emit_u8(&e, 0x5D); /* pop r13 */ |
|
| 2357 | 2937 | emit_u8(&e, 0x41); |
|
| 2358 | 2938 | emit_u8(&e, 0x5C); /* pop r12 */ |
|
| 2939 | + | emit_u8(&e, 0x5D); /* pop rbp */ |
|
| 2940 | + | emit_u8(&e, 0x5B); /* pop rbx */ |
|
| 2359 | 2941 | emit_u8(&e, 0xC3); /* ret */ |
|
| 2360 | 2942 | ||
| 2361 | 2943 | /* Entry: (rdi=regs, rsi=memory, rdx=io, rcx=code, r8=budget). Blocks |
|
| 2362 | 2944 | * make no calls, so the stack alignment is irrelevant. */ |
|
| 2363 | 2945 | jit->enter = (jit_enter_fn)(void *)(jit->code_cache + e.pos); |
|
| 2946 | + | emit_u8(&e, 0x53); /* push rbx */ |
|
| 2947 | + | emit_u8(&e, 0x55); /* push rbp */ |
|
| 2364 | 2948 | emit_u8(&e, 0x41); |
|
| 2365 | 2949 | emit_u8(&e, 0x54); /* push r12 */ |
|
| 2366 | 2950 | emit_u8(&e, 0x41); |
|
| 2367 | 2951 | emit_u8(&e, 0x55); /* push r13 */ |
|
| 2368 | 2952 | emit_u8(&e, 0x41); |
| 2379 | 2963 | emit_u8(&e, 0x89); |
|
| 2380 | 2964 | emit_u8(&e, 0xD6); /* mov r14, rdx */ |
|
| 2381 | 2965 | emit_u8(&e, 0x4D); |
|
| 2382 | 2966 | emit_u8(&e, 0x89); |
|
| 2383 | 2967 | emit_u8(&e, 0xC7); /* mov r15, r8 */ |
|
| 2968 | + | emit_u8(&e, 0x48); |
|
| 2969 | + | emit_u8(&e, 0x89); |
|
| 2970 | + | emit_u8(&e, 0xC8); /* mov rax, rcx */ |
|
| 2971 | + | /* The argument registers are free now: load the held guest registers. */ |
|
| 2972 | + | emit_transfer(&e, ALL_HOSTS, false); |
|
| 2384 | 2973 | emit_u8(&e, 0xFF); |
|
| 2385 | - | emit_u8(&e, 0xE1); /* jmp rcx */ |
|
| 2974 | + | emit_u8(&e, 0xE0); /* jmp rax */ |
|
| 2386 | 2975 | ||
| 2387 | 2976 | memcpy(jit->code_cache, buf, e.pos); |
|
| 2388 | 2977 | jit->code_cache_used = e.pos; |
|
| 2389 | 2978 | } |
|
| 2390 | 2979 |
jit.h
+23 -4
| 20 | 20 | ||
| 21 | 21 | /* --------------------------------------------------------------------------- |
|
| 22 | 22 | * Tuning constants |
|
| 23 | 23 | * ------------------------------------------------------------------------- */ |
|
| 24 | 24 | ||
| 25 | - | /* Size of the JIT code cache (16 MB). */ |
|
| 26 | - | #define JIT_CODE_CACHE_SIZE (16 * 1024 * 1024) |
|
| 25 | + | /* Size of the JIT code cache (64 MB). Only the pages used are committed. */ |
|
| 26 | + | #define JIT_CODE_CACHE_SIZE (64 * 1024 * 1024) |
|
| 27 | 27 | ||
| 28 | 28 | /* Maximum number of x86-64 bytes emitted per guest instruction. |
|
| 29 | 29 | * Conservative upper bound -- most instructions compile to <30 bytes. */ |
|
| 30 | 30 | #define JIT_MAX_INSN_BYTES 1024 |
|
| 31 | 31 | ||
| 32 | 32 | /* Maximum guest instructions in a single translated block. */ |
|
| 33 | 33 | #define JIT_MAX_BLOCK_INSNS 256 |
|
| 34 | 34 | ||
| 35 | + | /* Maximum conditional branches a translated block continues past. Each |
|
| 36 | + | * taken branch leaves the block through a side exit. */ |
|
| 37 | + | #define JIT_MAX_SIDE_EXITS 8 |
|
| 38 | + | ||
| 39 | + | /* Maximum direct jumps a translated block follows to their targets. */ |
|
| 40 | + | #define JIT_MAX_FOLLOWS 4 |
|
| 41 | + | ||
| 35 | 42 | /* Number of hash-table buckets for block lookup (must be power of 2). */ |
|
| 36 | 43 | #define JIT_BLOCK_HASH_SIZE (64 * 1024) |
|
| 37 | 44 | ||
| 38 | 45 | /* Maximum number of translated blocks. */ |
|
| 39 | 46 | #define JIT_MAX_BLOCKS (256 * 1024) |
|
| 40 | 47 | ||
| 41 | 48 | /* Number of jump cache entries (must be power of 2). */ |
|
| 42 | - | #define JIT_JUMP_CACHE_SIZE 16384 |
|
| 49 | + | #define JIT_JUMP_CACHE_SIZE 65536 |
|
| 43 | 50 | ||
| 44 | 51 | /* Guest PC that marks an empty jump cache entry. Indirect jump targets are |
|
| 45 | 52 | * always even, so this never matches. */ |
|
| 46 | 53 | #define JIT_JUMP_CACHE_EMPTY 0xFFFFFFFFu |
|
| 47 | 54 |
| 68 | 75 | u32 pc; /* Next guest PC on exit. */ |
|
| 69 | 76 | i64 budget; /* Remaining guest instructions. */ |
|
| 70 | 77 | u8 *patch; /* Chain jump rel32 field. */ |
|
| 71 | 78 | struct vector_state *vector; /* Architectural vector registers. */ |
|
| 72 | 79 | u64 memory_size; /* Accessible hosted memory bytes. */ |
|
| 80 | + | /* LR reservation: the address in the low word and the access width in |
|
| 81 | + | * the high word, or zero when none is held. */ |
|
| 82 | + | u64 reservation; |
|
| 73 | 83 | }; |
|
| 74 | 84 | ||
| 85 | + | /* Reservation value of an LR of `size` bytes at `address`. */ |
|
| 86 | + | static inline u64 jit_reservation(u32 address, u32 size) { |
|
| 87 | + | return (u64)size << 32 | address; |
|
| 88 | + | } |
|
| 89 | + | ||
| 75 | 90 | /* One entry of the indirect jump cache. Generated code indexes the cache |
|
| 76 | 91 | * assuming 16-byte entries. */ |
|
| 77 | 92 | struct jit_jump_entry { |
|
| 78 | 93 | u32 guest_pc; /* Guest PC, or JIT_JUMP_CACHE_EMPTY. */ |
|
| 79 | 94 | u8 *code; /* Entry point of the block at that PC. */ |
| 97 | 112 | ||
| 98 | 113 | /* Block storage (static pool). */ |
|
| 99 | 114 | struct jit_block blocks[JIT_MAX_BLOCKS]; |
|
| 100 | 115 | u32 block_count; |
|
| 101 | 116 | ||
| 102 | - | /* Shared entry trampoline and exit stub, at the start of the cache. */ |
|
| 117 | + | /* Shared entry trampoline and exit stubs, at the start of the cache. |
|
| 118 | + | * `exit_stub` writes the guest registers held in host registers back to |
|
| 119 | + | * the register file and falls into `exit_raw`, which returns without |
|
| 120 | + | * writing them. */ |
|
| 103 | 121 | jit_enter_fn enter; |
|
| 104 | 122 | u8 *exit_stub; |
|
| 123 | + | u8 *exit_raw; |
|
| 105 | 124 | ||
| 106 | 125 | /* Exit information for the current run. */ |
|
| 107 | 126 | struct jit_io io; |
|
| 108 | 127 | ||
| 109 | 128 | /* Direct-mapped cache of indirect jump targets, read by generated code |
test/jit_calls.c
added
+148 -0
| 1 | + | /* Writes a raw RV64 image for comparing translated and interpreted runs. |
|
| 2 | + | * |
|
| 3 | + | * The program makes calls through jal and through auipc and jalr, returns |
|
| 4 | + | * through ra, saves ra in a non-leaf function, branches around ebreak as a |
|
| 5 | + | * bounds check does, and uses LR and SC. Translated code follows these |
|
| 6 | + | * calls and checks the returns in place, so both runs must end with the |
|
| 7 | + | * same status after the same number of instructions. */ |
|
| 8 | + | #include <stdint.h> |
|
| 9 | + | #include <stdio.h> |
|
| 10 | + | ||
| 11 | + | typedef uint32_t u32; |
|
| 12 | + | typedef int32_t i32; |
|
| 13 | + | ||
| 14 | + | enum { ZERO = 0, RA = 1, SP = 2, T0 = 5, T1 = 6, T2 = 7, S1 = 9 }; |
|
| 15 | + | enum { A0 = 10, A1, A2, A3, A4, A5, A7 = 17, T5 = 30, T6 = 31 }; |
|
| 16 | + | ||
| 17 | + | static u32 code[256]; |
|
| 18 | + | static u32 len; |
|
| 19 | + | ||
| 20 | + | /* Labels and the instructions that refer to them. */ |
|
| 21 | + | enum label { LOOP, LEAF, LEAF_OK, NEST, LABELS }; |
|
| 22 | + | static u32 labels[LABELS]; |
|
| 23 | + | static struct { |
|
| 24 | + | u32 at; |
|
| 25 | + | enum label to; |
|
| 26 | + | u32 kind; /* 0 jal, 1 branch, 2 jalr after auipc */ |
|
| 27 | + | u32 ins; /* Instruction without its offset. */ |
|
| 28 | + | } fixups[32]; |
|
| 29 | + | static u32 nfixups; |
|
| 30 | + | ||
| 31 | + | static u32 r_type(u32 op, u32 rd, u32 f3, u32 rs1, u32 rs2, u32 f7) { |
|
| 32 | + | return f7 << 25 | rs2 << 20 | rs1 << 15 | f3 << 12 | rd << 7 | op; |
|
| 33 | + | } |
|
| 34 | + | ||
| 35 | + | static u32 i_type(u32 op, u32 rd, u32 f3, u32 rs1, i32 imm) { |
|
| 36 | + | return ((u32)imm & 0xfff) << 20 | rs1 << 15 | f3 << 12 | rd << 7 | op; |
|
| 37 | + | } |
|
| 38 | + | ||
| 39 | + | static u32 s_type(u32 rs2, u32 rs1, u32 f3, i32 imm) { |
|
| 40 | + | u32 v = (u32)imm; |
|
| 41 | + | return (v >> 5 & 0x7f) << 25 | rs2 << 20 | rs1 << 15 | f3 << 12 | |
|
| 42 | + | (v & 0x1f) << 7 | 0x23; |
|
| 43 | + | } |
|
| 44 | + | ||
| 45 | + | static void emit(u32 ins) { |
|
| 46 | + | code[len++] = ins; |
|
| 47 | + | } |
|
| 48 | + | ||
| 49 | + | static void addi(u32 rd, u32 rs1, i32 imm) { |
|
| 50 | + | emit(i_type(0x13, rd, 0, rs1, imm)); |
|
| 51 | + | } |
|
| 52 | + | ||
| 53 | + | static void ref(u32 kind, u32 ins, enum label to) { |
|
| 54 | + | fixups[nfixups].at = len; |
|
| 55 | + | fixups[nfixups].to = to; |
|
| 56 | + | fixups[nfixups].kind = kind; |
|
| 57 | + | fixups[nfixups].ins = ins; |
|
| 58 | + | nfixups++; |
|
| 59 | + | emit(0); |
|
| 60 | + | } |
|
| 61 | + | ||
| 62 | + | static void mark(enum label l) { |
|
| 63 | + | labels[l] = len * 4; |
|
| 64 | + | } |
|
| 65 | + | ||
| 66 | + | static void resolve(void) { |
|
| 67 | + | for (u32 i = 0; i < nfixups; i++) { |
|
| 68 | + | u32 pc = fixups[i].at * 4; |
|
| 69 | + | u32 off = labels[fixups[i].to] - pc; |
|
| 70 | + | u32 ins = fixups[i].ins; |
|
| 71 | + | if (fixups[i].kind == 0) { |
|
| 72 | + | ins |= (off >> 20 & 1) << 31 | (off >> 1 & 0x3ff) << 21 | |
|
| 73 | + | (off >> 11 & 1) << 20 | (off >> 12 & 0xff) << 12; |
|
| 74 | + | } else if (fixups[i].kind == 1) { |
|
| 75 | + | ins |= (off >> 12 & 1) << 31 | (off >> 5 & 0x3f) << 25 | |
|
| 76 | + | (off >> 1 & 0xf) << 8 | (off >> 11 & 1) << 7; |
|
| 77 | + | } else { |
|
| 78 | + | ins |= ((off + 4) & 0xfff) << 20; /* Relative to the auipc. */ |
|
| 79 | + | } |
|
| 80 | + | code[fixups[i].at] = ins; |
|
| 81 | + | } |
|
| 82 | + | } |
|
| 83 | + | ||
| 84 | + | int main(int argc, char **argv) { |
|
| 85 | + | if (argc != 2) { |
|
| 86 | + | fprintf(stderr, "usage: %s <output>\n", argv[0]); |
|
| 87 | + | return 1; |
|
| 88 | + | } |
|
| 89 | + | /* main: sum calls over 200 iterations into a0. */ |
|
| 90 | + | addi(A0, ZERO, 0); |
|
| 91 | + | addi(S1, ZERO, 200); |
|
| 92 | + | addi(T6, SP, -64); |
|
| 93 | + | mark(LOOP); |
|
| 94 | + | addi(A1, S1, 0); |
|
| 95 | + | ref(0, RA << 7 | 0x6f, LEAF); /* jal ra, leaf */ |
|
| 96 | + | ref(0, RA << 7 | 0x6f, NEST); /* jal ra, nest */ |
|
| 97 | + | emit(T5 << 7 | 0x17); /* auipc t5, 0 */ |
|
| 98 | + | ref(2, i_type(0x67, RA, 0, T5, 0), LEAF); /* jalr ra, leaf(t5) */ |
|
| 99 | + | addi(S1, S1, -1); |
|
| 100 | + | ref(1, r_type(0x63, 0, 1, S1, ZERO, 0), LOOP); /* bnez s1, loop */ |
|
| 101 | + | ||
| 102 | + | /* Add 7 to a doubleword with LR and SC, then fold it into a0. */ |
|
| 103 | + | emit(0x1000302f | T6 << 15 | T0 << 7); /* lr.d t0, (t6) */ |
|
| 104 | + | addi(T0, T0, 7); |
|
| 105 | + | emit(0x1800302f | T6 << 15 | T0 << 20 | T1 << 7); /* sc.d t1, t0, (t6) */ |
|
| 106 | + | emit(r_type(0x33, A0, 0, A0, T1, 0)); |
|
| 107 | + | emit(i_type(0x03, T2, 3, T6, 0)); /* ld t2, 0(t6) */ |
|
| 108 | + | emit(r_type(0x33, A0, 0, A0, T2, 0)); |
|
| 109 | + | emit(i_type(0x13, A0, 7, A0, 0xff)); |
|
| 110 | + | addi(A7, ZERO, 0x400); /* exit(a0) */ |
|
| 111 | + | emit(0x73); |
|
| 112 | + | ||
| 113 | + | /* leaf(a1): bounds check, then mix a1 into a0. */ |
|
| 114 | + | mark(LEAF); |
|
| 115 | + | addi(A2, ZERO, 1000); |
|
| 116 | + | ref(1, r_type(0x63, 0, 6, A1, A2, 0), LEAF_OK); /* bltu a1, a2, ok */ |
|
| 117 | + | emit(0x00100073); /* ebreak */ |
|
| 118 | + | mark(LEAF_OK); |
|
| 119 | + | addi(A3, A1, 0); |
|
| 120 | + | emit(r_type(0x33, A3, 1, A3, A3, 0)); /* sll a3, a3, a3 */ |
|
| 121 | + | addi(A4, ZERO, 3); |
|
| 122 | + | emit(r_type(0x33, A4, 0, A1, A4, 1)); /* mul a4, a1, a4 */ |
|
| 123 | + | emit(i_type(0x13, A5, 5, A1, 1)); /* srli a5, a1, 1 */ |
|
| 124 | + | emit(r_type(0x33, A4, 4, A4, A5, 0)); /* xor a4, a4, a5 */ |
|
| 125 | + | emit(r_type(0x33, A0, 0, A0, A4, 0)); |
|
| 126 | + | emit(r_type(0x3b, A0, 0, A0, A3, 0x20)); /* subw a0, a0, a3 */ |
|
| 127 | + | emit(i_type(0x67, ZERO, 0, RA, 0)); /* ret */ |
|
| 128 | + | ||
| 129 | + | /* nest: save ra, call leaf twice, restore ra and return. */ |
|
| 130 | + | mark(NEST); |
|
| 131 | + | addi(SP, SP, -16); |
|
| 132 | + | emit(s_type(RA, SP, 3, 8)); |
|
| 133 | + | addi(A1, A1, 5); |
|
| 134 | + | ref(0, RA << 7 | 0x6f, LEAF); |
|
| 135 | + | addi(A1, A1, -3); |
|
| 136 | + | ref(0, RA << 7 | 0x6f, LEAF); |
|
| 137 | + | emit(i_type(0x03, RA, 3, SP, 8)); |
|
| 138 | + | addi(SP, SP, 16); |
|
| 139 | + | emit(i_type(0x67, ZERO, 0, RA, 0)); |
|
| 140 | + | ||
| 141 | + | resolve(); |
|
| 142 | + | FILE *f = fopen(argv[1], "wb"); |
|
| 143 | + | if (!f || fwrite(code, 4, len, f) != len || fclose(f)) { |
|
| 144 | + | perror(argv[1]); |
|
| 145 | + | return 1; |
|
| 146 | + | } |
|
| 147 | + | return 0; |
|
| 148 | + | } |