jit: Speed up translated calls and register access

a503d772dc87f89f24f3f56c9fcf45f7e10edd3ce1d05dbe6fe0148d49b2b118
Use native calls and returns for direct guest calls, and charge the
instruction budget at block exits. Share a base register for the guest
register file and JIT exchange area to hold another guest register.

Radiance self-compilation runs 20% faster with identical output and
guest instruction counts.

Assisted-By: Codex:gpt-6.2-sol
Change-Id: I31c39b37ad5d63319e55069d6d9974ab93a23e89
Alexis Sellier committed ago 1 parent 4799a8d7
jit.c +278 -242
4 4
 * Strategy:
5 5
 *   - One entry trampoline and one exit stub are generated at init.  The
6 6
 *     trampoline saves the callee-saved registers, pins the registers below
7 7
 *     and jumps into a block.  The exit stub stores the remaining budget,
8 8
 *     restores the registers and returns the exit reason in eax.  Blocks
9 -
 *     have no prologue or epilogue and never call.
9 +
 *     can call other blocks directly.
10 10
 *   - During block execution we pin:
11 -
 *       r12 = &regs[0]   (guest register file)
12 11
 *       r13 = memory base
13 -
 *       r14 = &jit->io   (exit information; pc is its first field)
12 +
 *       r14 = &jit->io   (exit information and register file base)
14 13
 *       r15 = budget     (guest instructions left, signed)
15 14
 *   - Guest registers are loaded/stored on demand from/to the array.
16 15
 *   - Vector operations check their element width, group size, active length
17 16
 *     and restart index before access.
18 -
 *   - Each block starts with "if budget <= 0 exit at this PC; budget -=
19 -
 *     insn_count".  Every exit is the block's last instruction, so a block
20 -
 *     always accounts exactly the instructions it executes.
17 +
 *   - Each block starts with a budget check.  Every exit charges the
18 +
 *     instructions on its path.  A jump to another block carries the flags
19 +
 *     from the subtraction, so the target needs only a conditional jump.
21 20
 *   - Direct jumps (jal, both sides of a branch, block split) are a jmp
22 21
 *     rel32 that first falls into a stub returning JIT_EXIT_CHAIN.  C then
23 22
 *     patches the jmp to the target block, so later runs stay in the cache.
24 23
 *   - Indirect jumps (jalr) probe a direct-mapped jump cache filled by C and
25 24
 *     jump to the hit; a miss returns JIT_EXIT_BRANCH.
25 +
 *   - Direct calls save the guest return address on the host stack.  A return
26 +
 *     to that address uses a native ret.  The overflow flag at a return
27 +
 *     continuation marks an unwind to C.
26 28
 *
27 29
 *  We emit raw x86-64 bytes into the code cache.
28 30
 */
29 31
30 32
#include <stddef.h>
51 53
    /* The register file holds every guest register and host registers may be
52 54
     * clobbered: set from an instruction that addresses the register file
53 55
     * directly until the held registers are reloaded. */
54 56
    bool spilled;
55 57
    u32  insns; /* Instructions through the current operation. */
56 -
    /* Each indexed load can fail three checks for each of four lanes. */
57 -
    u32 refunds[JIT_MAX_BLOCK_INSNS * 12];  /* Budget patch offsets. */
58 -
    u32 consumed[JIT_MAX_BLOCK_INSNS * 12]; /* Instructions at each patch. */
59 -
    u32 refunds_len;                        /* Number of guarded exits. */
60 58
61 -
    /* Exits from inside the block, on a branch or a checked return, emitted
62 -
     * after its main path. */
59 +
    /* Conditional branch exits, emitted after the main path. */
63 60
    struct {
64 -
        u32  jcc;    /* Offset of the jcc rel32 instruction. */
65 -
        u32  target; /* Guest PC of the branch target. */
66 -
        u32  insns;  /* Instructions executed when it is taken. */
67 -
        bool ret;    /* Return to ra instead of jumping to `target`. */
61 +
        u32 jcc;    /* Offset of the jcc rel32 instruction. */
62 +
        u32 target; /* Guest PC of the branch target. */
63 +
        u32 insns;  /* Instructions executed when it is taken. */
68 64
    } sides[JIT_MAX_SIDE_EXITS];
69 65
    u32 sides_len;
70 66
71 67
    /* Guest PC after the current instruction. */
72 68
    u32 next_pc;
76 72
    u32 run_start[JIT_MAX_FOLLOWS + 1];
77 73
    u32 run_end[JIT_MAX_FOLLOWS + 1];
78 74
    u32 runs;
79 75
    /* The previous instruction set `auipc_reg` to `auipc_value` with auipc,
80 76
     * or `auipc_reg` is zero. */
81 -
    u32 auipc_reg;
82 -
    u64 auipc_value;
83 -
    /* ra holds `ra_value`, set by a call the block followed. */
84 -
    bool ra_known;
85 -
    u32  ra_value;
86 -
77 +
    u32               auipc_reg;
78 +
    u64               auipc_value;
87 79
    u8               *mem;       /* Guest memory, for looking ahead. */
88 80
    u32               prog_base; /* Guest code bounds. */
89 81
    u32               prog_end;
90 82
    struct jit_state *jit;  /* For the exit stub and jump cache addresses. */
91 83
    u8               *dest; /* Address the buffer will be copied to. */
144 136
static inline u8 sib(u8 scale, u8 index, u8 base) {
145 137
    return (u8)((scale << 6) | ((index & 7) << 3) | (base & 7));
146 138
}
147 139
148 140
/* Pinned host registers. */
149 -
#define HREGS   X_R12 /* &regs[0]  */
150 141
#define HMEM    X_R13 /* memory    */
151 142
#define HIO     X_R14 /* &jit->io  */
152 143
#define HBUDGET X_R15 /* budget    */
153 144
154 145
/* Scratch registers for codegen (caller-saved, not pinned). */
166 157
    [2]  = X_RDI, /* sp */
167 158
    [30] = X_R8,  /* t5 */
168 159
    [7]  = X_R9,  /* t2 */
169 160
    [10] = X_R10, /* a0 */
170 161
    [28] = X_R11, /* t3 */
162 +
    [9]  = X_R12, /* s1 */
171 163
    [11] = X_RDX, /* a1 */
172 164
};
173 165
174 166
/* Host register holding `guest`, or zero if it lives in the register file. */
175 167
static inline u8 host_of(struct emitter *e, u32 guest) {
199 191
    emit_u8(e, modrm(3, ext, X_RAX));
200 192
}
201 193
202 194
/* ---------- Load/store guest register from register file --------------- */
203 195
204 -
/* Emit the ModR/M, SIB, and displacement for [r12 + off]. */
205 -
static void emit_r12_address(struct emitter *e, enum x86reg reg, u32 off) {
206 -
    if (off < 128) {
207 -
        emit_u8(e, modrm(1, reg, X_R12 & 7));
208 -
        emit_u8(e, sib(0, 4, X_R12 & 7));
209 -
        emit_u8(e, (u8)off);
196 +
/* Address byte offset `off` in the guest register file adjacent to io. */
197 +
static void emit_reg_address(struct emitter *e, enum x86reg reg, u32 off) {
198 +
    i32 disp = (i32)offsetof(struct jit_state, regs) -
199 +
               (i32)offsetof(struct jit_state, io) + (i32)off;
200 +
    if (disp >= INT8_MIN && disp <= INT8_MAX) {
201 +
        emit_u8(e, modrm(1, reg, HIO));
202 +
        emit_u8(e, (u8)disp);
210 203
    } else {
211 -
        emit_u8(e, modrm(2, reg, X_R12 & 7));
212 -
        emit_u8(e, sib(0, 4, X_R12 & 7));
213 -
        emit_u32(e, off);
204 +
        emit_u8(e, modrm(2, reg, HIO));
205 +
        emit_u32(e, (u32)disp);
214 206
    }
215 207
}
216 208
217 -
/* Emit mov with [r12 + disp] addressing (r12 needs SIB). */
218 -
static void emit_r12_disp(
209 +
/* Emit a 64-bit operation on a guest register in the register file. */
210 +
static void emit_reg_disp(
219 211
    struct emitter *e, u8 opcode, enum x86reg reg, u32 off
220 212
) {
221 -
    emit_u8(e, rex(true, reg >= 8, false, true)); /* B=1 for r12 */
213 +
    emit_u8(e, rex(true, reg >= 8, false, true));
222 214
    emit_u8(e, opcode);
223 -
    emit_r12_address(e, reg, off);
215 +
    emit_reg_address(e, reg, off);
224 216
}
225 217
226 218
/* mov dst, src (64-bit), omitted when they are the same register. */
227 219
static void emit_mov_rr(struct emitter *e, u8 dst, u8 src) {
228 220
    if (dst == src)
233 225
}
234 226
235 227
/* Load guest register `guest` into host `dst`. */
236 228
static void emit_load_guest(struct emitter *e, enum x86reg dst, u32 guest) {
237 229
    if (guest == 0) {
238 -
        /* x0 is always zero -- xor dst, dst. */
239 -
        emit_u8(e, rex(true, dst >= 8, false, dst >= 8));
230 +
        /* xor dst32, dst32 clears the full register. */
231 +
        if (dst >= 8)
232 +
            emit_u8(e, rex(false, true, false, true));
240 233
        emit_u8(e, 0x31);
241 234
        emit_u8(e, modrm(3, dst, dst));
242 235
        return;
243 236
    }
244 237
    u8 host = host_of(e, guest);
245 238
    if (host) {
246 239
        emit_mov_rr(e, dst, host);
247 240
        return;
248 241
    }
249 -
    emit_r12_disp(e, 0x8B, dst, guest * 8);
242 +
    emit_reg_disp(e, 0x8B, dst, guest * 8);
250 243
}
251 244
252 245
/* Load and zero-extend the low word of a guest register. */
253 246
static void emit_load_guest32(struct emitter *e, enum x86reg dst, u32 guest) {
254 247
    if (guest == 0) {
264 257
        emit_u8(e, modrm(3, host, dst));
265 258
        return;
266 259
    }
267 260
    emit_u8(e, rex(false, dst >= 8, false, true));
268 261
    emit_u8(e, 0x8b);
269 -
    emit_r12_address(e, dst, guest * 8);
262 +
    emit_reg_address(e, dst, guest * 8);
270 263
}
271 264
272 265
/* Store host `src` to guest register `guest`. */
273 266
static void emit_store_guest(struct emitter *e, u32 guest, enum x86reg src) {
274 267
    if (guest == 0)
276 269
    u8 host = host_of(e, guest);
277 270
    if (host) {
278 271
        emit_mov_rr(e, host, src);
279 272
        return;
280 273
    }
281 -
    emit_r12_disp(e, 0x89, src, guest * 8);
274 +
    emit_reg_disp(e, 0x89, src, guest * 8);
282 275
}
283 276
284 277
/* Every host register, as a mask for emit_transfer(). */
285 278
#define ALL_HOSTS 0xFFFFu
286 279
287 280
/* Store (`store`) or load the held guest registers whose host registers are
288 281
 * in `mask` to or from the register file. */
289 282
static void emit_transfer(struct emitter *e, u32 mask, bool store) {
290 283
    for (u32 g = 1; g < 32; g++) {
291 284
        if (guest_host[g] && (mask >> guest_host[g]) & 1)
292 -
            emit_r12_disp(e, store ? 0x89 : 0x8B, guest_host[g], g * 8);
285 +
            emit_reg_disp(e, store ? 0x89 : 0x8B, guest_host[g], g * 8);
293 286
    }
294 287
}
295 288
296 289
/* Make the register file hold every guest register before an instruction
297 290
 * that addresses it directly or clobbers host registers. */
313 306
    emit_u64(e, imm);
314 307
}
315 308
316 309
/* mov reg, imm32 (sign-extended to 64 via mov r/m64, imm32) */
317 310
static void emit_mov_imm32_sx(struct emitter *e, enum x86reg dst, i32 imm) {
311 +
    if (imm >= 0) {
312 +
        /* A 32-bit destination clears the high word. */
313 +
        if (dst >= 8)
314 +
            emit_u8(e, rex(false, false, false, true));
315 +
        emit_u8(e, 0xB8 | (dst & 7));
316 +
        emit_u32(e, (u32)imm);
317 +
        return;
318 +
    }
318 319
    emit_u8(e, rex(true, false, false, dst >= 8));
319 320
    emit_u8(e, 0xC7);
320 321
    emit_u8(e, modrm(3, 0, dst));
321 322
    emit_u32(e, (u32)imm);
322 323
}
323 324
325 +
/* Set a guest register to a signed 32-bit constant. */
326 +
static void emit_guest_imm32(struct emitter *e, u32 guest, i32 imm) {
327 +
    u8 host = host_of(e, guest);
328 +
    if (host) {
329 +
        emit_mov_imm32_sx(e, host, imm);
330 +
        return;
331 +
    }
332 +
    emit_u8(e, 0x49);
333 +
    emit_u8(e, 0xC7);
334 +
    emit_reg_address(e, 0, guest * 8);
335 +
    emit_u32(e, (u32)imm);
336 +
}
337 +
324 338
/* ---------- Jump patching ---------------------------------------------- */
325 339
326 340
/* Emit jnz rel32 (0F 85), return position for patching. */
327 341
static u32 emit_jnz_placeholder(struct emitter *e) {
328 342
    u32 pos = e->pos;
386 400
    emit_u8(e, 0xC7);
387 401
    emit_u8(e, modrm(0, 0, HIO));
388 402
    emit_u32(e, pc);
389 403
}
390 404
405 +
/* Charge the instructions on this path and set the budget flags. */
406 +
static void emit_charge(struct emitter *e, u32 insns) {
407 +
    if (insns == 0)
408 +
        return;
409 +
    bool small = insns <= INT8_MAX;
410 +
    emit_u8(e, 0x49);
411 +
    emit_u8(e, small ? 0x83 : 0x81);
412 +
    emit_u8(e, modrm(3, 5, HBUDGET));
413 +
    if (small)
414 +
        emit_u8(e, (u8)insns);
415 +
    else
416 +
        emit_u32(e, insns);
417 +
}
418 +
391 419
/* mov eax, reason; jmp exit_stub.  While spilled, the register
392 420
 * file is already current and the host registers may be stale. */
393 421
static void emit_exit(struct emitter *e, u32 exit_reason) {
394 422
    emit_u8(e, 0xB8);
395 423
    emit_u32(e, exit_reason); /* mov eax, reason */
398 426
399 427
/* Store next-PC to *r14 and leave the cache with `reason`. */
400 428
static void emit_block_exit(
401 429
    struct emitter *e, enum jit_exit reason, u32 next_pc
402 430
) {
431 +
    emit_charge(e, e->insns);
403 432
    emit_store_pc_imm(e, next_pc);
404 433
    emit_exit(e, (u32)reason);
405 434
}
406 435
407 -
/* Jump to the guest block at the constant PC `target`.  Emits a jmp rel32
408 -
 * that initially falls into a stub returning JIT_EXIT_CHAIN with the address
409 -
 * of its rel32 field in io.patch; see jit_patch_chain. */
410 -
static void emit_chain_exit(struct emitter *e, u32 target) {
411 -
    /* A linked jump enters the target with the guest registers held. */
412 -
    bool spilled = e->spilled;
413 -
    if (spilled)
414 -
        emit_reload(e);
415 -
416 -
    u32 site = e->pos;
417 -
    emit_u8(e, 0xE9);
418 -
    emit_u32(e, 0); /* jmp .stub (next instruction) */
419 -
436 +
/* Leave for C to link the rel32 branch at `site` to `target`. */
437 +
static void emit_chain_stub(struct emitter *e, u32 target, u32 site) {
420 438
    emit_store_pc_imm(e, target);
421 439
422 -
    /* lea rax, [rip + (site + 1) - next] */
440 +
    /* lea rax, [rip + site - next] */
423 441
    emit_u8(e, 0x48);
424 442
    emit_u8(e, 0x8D);
425 443
    emit_u8(e, modrm(0, X_RAX, 5));
426 -
    emit_u32(e, (site + 1) - (e->pos + 4));
444 +
    emit_u32(e, site - (e->pos + 4));
427 445
428 446
    /* mov [r14 + IO_PATCH], rax */
429 447
    emit_u8(e, 0x49);
430 448
    emit_u8(e, 0x89);
431 449
    emit_u8(e, modrm(1, X_RAX, HIO));
432 450
    emit_u8(e, IO_PATCH);
433 451
434 452
    emit_exit(e, JIT_EXIT_CHAIN);
453 +
}
454 +
455 +
/* Jump to the guest block at `target` with the budget flags set. */
456 +
static void emit_link(struct emitter *e, u32 target) {
457 +
    u32 site = emit_jmp_placeholder(e);
458 +
    emit_chain_stub(e, target, site + 1);
459 +
}
460 +
461 +
/* Charge the block and jump to `target` with all held registers loaded. */
462 +
static void emit_chain_exit(struct emitter *e, u32 target) {
463 +
    bool spilled = e->spilled;
464 +
    if (spilled)
465 +
        emit_reload(e);
466 +
    emit_charge(e, e->insns);
467 +
    emit_link(e, target);
435 468
    e->spilled = spilled;
436 469
}
437 470
471 +
/* A native call frame contains the host continuation and the guest link. */
472 +
#define JIT_CALL_LINK_SIZE  8
473 +
#define JIT_CALL_STACK_SIZE 4096
474 +
#define IO_STACK_BASE       ((u8)offsetof(struct jit_io, stack_base))
475 +
#define IO_STACK_FLOOR      ((u8)offsetof(struct jit_io, stack_floor))
476 +
477 +
/* Return from a native call and drop its guest link. */
478 +
static void emit_native_ret(struct emitter *e) {
479 +
    emit_u8(e, 0xC2); /* ret imm16 */
480 +
    emit_u8(e, JIT_CALL_LINK_SIZE);
481 +
    emit_u8(e, 0);
482 +
}
483 +
484 +
/* On an unwind, continue returning through the native call stack. */
485 +
static void emit_unwind_check(struct emitter *e) {
486 +
    emit_u8(e, 0x0F);
487 +
    emit_u8(e, 0x80); /* jo exit_unwind */
488 +
    emit_u32(e, (u32)(e->jit->exit_unwind - (e->dest + e->pos + 4)));
489 +
}
490 +
491 +
/* Translate a direct call with guest link `next`.  If the host call stack is
492 +
 * full, charge the block and jump without adding a frame. */
493 +
static void emit_native_call(struct emitter *e, u32 next, u32 target) {
494 +
    /* cmp rsp, [r14 + IO_STACK_FLOOR]; jbe .full */
495 +
    emit_u8(e, 0x49);
496 +
    emit_u8(e, 0x3B);
497 +
    emit_u8(e, modrm(1, X_RSP, HIO));
498 +
    emit_u8(e, IO_STACK_FLOOR);
499 +
    u32 full = e->pos;
500 +
    emit_u8(e, 0x0F);
501 +
    emit_u8(e, 0x86);
502 +
    emit_u32(e, 0);
503 +
504 +
    emit_u8(e, 0x68); /* push guest link */
505 +
    emit_u32(e, next);
506 +
    emit_charge(e, e->insns);
507 +
    u32 call = e->pos;
508 +
    emit_u8(e, 0xE8);
509 +
    emit_u32(e, 0);
510 +
511 +
    /* A guest return jumps to its continuation.  An unwind falls through. */
512 +
    u32 resume = e->pos;
513 +
    emit_u8(e, 0x0F);
514 +
    emit_u8(e, 0x81); /* jno */
515 +
    emit_u32(e, 0);
516 +
    emit_jmp_abs(e, e->jit->exit_unwind);
517 +
    patch_jcc(e, resume, e->pos);
518 +
    emit_chain_stub(e, next, resume + 2);
519 +
520 +
    patch_jmp(e, call, e->pos);
521 +
    emit_chain_stub(e, target, call + 1);
522 +
523 +
    patch_jcc(e, full, e->pos);
524 +
    emit_chain_exit(e, target);
525 +
}
526 +
438 527
/* Offset of the jump cache from the exchange area in r14. */
439 528
#define IO_JUMP_CACHE                                                          \
440 529
    ((u32)(offsetof(struct jit_state, jump_cache) -                            \
441 530
           offsetof(struct jit_state, io)))
442 531
443 -
/* Jump to the guest PC in rax (already even).  Stores it to io.pc, probes the
444 -
 * jump cache and jumps to the cached block on a hit, else exits to C.
445 -
 * Clobbers only rax and rcx, since the target expects held registers. */
532 +
/* Jump to the guest PC in rax (already even).  On a cache miss, store the PC
533 +
 * in io.pc and return to C.  Preserve the held guest registers for the target.
534 +
 */
446 535
static void emit_indirect_jump(struct emitter *e) {
447 536
    if (e->spilled)
448 537
        emit_reload(e);
449 538
450 -
    /* mov dword [r14], eax */
451 -
    emit_u8(e, 0x41);
452 -
    emit_u8(e, 0x89);
453 -
    emit_u8(e, modrm(0, X_RAX, HIO));
454 -
455 539
    /* mov ecx, eax; and ecx, mask << 2; shl ecx, 2 -- byte offset of the
456 540
     * entry, ((pc >> 2) & mask) * sizeof(struct jit_jump_entry). */
457 541
    emit_u8(e, 0x89);
458 542
    emit_u8(e, modrm(3, X_RAX, X_RCX));
459 543
    emit_u8(e, 0x81);
470 554
    emit_u8(e, sib(0, X_RCX, HIO));
471 555
    emit_u32(e, IO_JUMP_CACHE);
472 556
473 557
    u32 jne_pos = emit_jnz_placeholder(e);
474 558
559 +
    emit_charge(e, e->insns);
560 +
475 561
    /* jmp [r14 + rcx + IO_JUMP_CACHE + offsetof(code)] */
476 562
    emit_u8(e, 0x41);
477 563
    emit_u8(e, 0xFF);
478 564
    emit_u8(e, modrm(2, 4, 4));
479 565
    emit_u8(e, sib(0, X_RCX, HIO));
480 566
    emit_u32(e, IO_JUMP_CACHE + offsetof(struct jit_jump_entry, code));
481 567
482 -
    /* Miss: io.pc is set, let C dispatch. */
568 +
    /* Miss: save the target PC for C. */
483 569
    patch_jcc(e, jne_pos, e->pos);
570 +
    emit_charge(e, e->insns);
571 +
    emit_u8(e, 0x41);
572 +
    emit_u8(e, 0x89);
573 +
    emit_u8(e, modrm(0, X_RAX, HIO));
484 574
    emit_exit(e, JIT_EXIT_BRANCH);
485 575
}
486 576
487 577
/* ---------- Per-instruction translation helpers ------------------------ */
488 578
584 674
    emit_u8(e, 0x83);
585 675
    emit_u8(e, modrm(3, 7, X_RCX));
586 676
    emit_u8(e, 0xFF);
587 677
}
588 678
589 -
/* mov rax, r11 (via: REX.WR mov rax, r11) */
590 -
static void emit_mov_rax_r11(struct emitter *e) {
591 -
    emit_u8(e, 0x4C);
592 -
    emit_u8(e, 0x89);
593 -
    emit_u8(e, modrm(3, X_R11 & 7, X_RAX));
594 -
}
595 -
596 679
/* mov rax, rdx (64-bit) */
597 680
static void emit_mov_rax_rdx(struct emitter *e) {
598 681
    emit_rexw(e);
599 682
    emit_u8(e, 0x89);
600 683
    emit_u8(e, modrm(3, X_RDX, X_RAX));
647 730
    emit_u8(e, 0x0F);
648 731
    emit_u8(e, 0xB6);
649 732
    emit_u8(e, modrm(3, X_RAX, X_RAX));
650 733
}
651 734
652 -
/* and ecx, imm8 (for masking shift amounts). */
653 -
static void emit_and_ecx_imm8(struct emitter *e, u8 mask) {
654 -
    emit_u8(e, 0x83);
655 -
    emit_u8(e, modrm(3, 4, X_RCX));
656 -
    emit_u8(e, mask);
657 -
}
658 -
659 735
/* ---------- Division helpers ------------------------------------------- */
660 736
661 737
/*
662 738
 * 64-bit div/rem of rax by rcx, leaving the result in rax, with the RISC-V
663 739
 * results where x86 division would trap:
667 743
 *
668 744
 * `is_signed`: signed division.
669 745
 * `is_rem`:    return the remainder instead of the quotient.
670 746
 */
671 747
static void emit_div64(struct emitter *e, bool is_signed, bool is_rem) {
672 -
    /* For rem: save dividend to r11 (needed if divisor==0). */
673 -
    if (is_rem) {
674 -
        /* mov r11, rax */
675 -
        emit_u8(e, rex(true, false, false, true));
676 -
        emit_u8(e, 0x89);
677 -
        emit_u8(e, modrm(3, X_RAX, X_R11 & 7));
678 -
    }
679 -
680 748
    emit_test_rcx(e);
681 749
    u32 jnz = emit_jnz_placeholder(e);
682 750
683 751
    /* Divisor == 0. */
684 -
    if (is_rem) {
685 -
        emit_mov_rax_r11(e); /* result = dividend */
686 -
    } else {
752 +
    if (!is_rem) {
687 753
        emit_mov_imm32_sx(e, HTMP1, -1); /* result = -1 (all ones) */
688 754
    }
689 755
    u32 jmp_end1 = emit_jmp_placeholder(e);
690 756
691 757
    /* .nonzero: */
731 797
    }
732 798
}
733 799
734 800
/* Same pattern for 32-bit division (W-suffix instructions). */
735 801
static void emit_div32(struct emitter *e, bool is_signed, bool is_rem) {
736 -
    /* For rem: save dividend in edx. */
737 -
    if (is_rem) {
738 -
        /* mov edx, eax */
739 -
        emit_u8(e, 0x89);
740 -
        emit_u8(e, modrm(3, X_RAX, X_RDX));
741 -
    }
742 -
743 802
    emit_test_ecx(e);
744 803
    u32 jnz = emit_jnz_placeholder(e);
745 804
746 805
    /* Divisor == 0. */
747 -
    if (is_rem) {
748 -
        /* mov eax, edx (result = dividend) */
749 -
        emit_u8(e, 0x89);
750 -
        emit_u8(e, modrm(3, X_RDX, X_RAX));
751 -
    } else {
806 +
    if (!is_rem) {
752 807
        /* mov eax, -1 */
753 808
        emit_u8(e, 0xB8);
754 809
        emit_u32(e, 0xFFFFFFFF);
755 810
    }
756 811
    u32 jmp_end1 = emit_jmp_placeholder(e);
773 828
        jmp_end2 = emit_jmp_placeholder(e);
774 829
    }
775 830
776 831
    /* .safe: */
777 832
    u32 safe = e->pos;
778 -
    if (is_rem) {
779 -
        /* Restore dividend to eax from edx. */
780 -
        emit_u8(e, 0x89);
781 -
        emit_u8(e, modrm(3, X_RDX, X_RAX));
782 -
    }
783 833
    if (is_signed) {
784 834
        emit_u8(e, 0x99); /* cdq */
785 835
        emit_u8(e, 0xF7);
786 836
        emit_u8(e, modrm(3, 7, X_RCX)); /* idiv ecx */
787 837
    } else {
965 1015
            emit_load_rs1(e, ZERO);
966 1016
        } else {
967 1017
            emit_u8(e, rex(op != BM_ZEXTH, false, false, true));
968 1018
            emit_u8(e, 0x0f);
969 1019
            emit_u8(e, op == BM_SEXTB ? 0xbe : op == BM_SEXTH ? 0xbf : 0xb7);
970 -
            emit_r12_address(e, HTMP1, ins.r.rs1 * 8);
1020 +
            emit_reg_address(e, HTMP1, ins.r.rs1 * 8);
971 1021
        }
972 1022
        emit_store_rd(e, ins.r.rd);
973 1023
        return;
974 1024
    }
975 1025
    if (op == BM_ORCB) {
1020 1070
        }
1021 1071
        emit_u8(e, avx ? 0xc4 : 0xf3);
1022 1072
        emit_u8(e, avx ? 0xc1 : 0x41);
1023 1073
        emit_u8(e, avx ? 0x7a : 0x0f);
1024 1074
        emit_u8(e, 0x7e);
1025 -
        emit_r12_address(e, X_RAX, ins.r.rs1 * 8); /* vmovq xmm0, [r12 + off] */
1075 +
        emit_reg_address(
1076 +
            e, X_RAX, ins.r.rs1 * 8
1077 +
        ); /* vmovq xmm0, [r14 + disp] */
1026 1078
        for (u32 i = 0; i < sizeof(masks[0]); i++)
1027 1079
            emit_u8(e, masks[avx][i]);
1028 1080
        emit_u8(e, avx ? 0xc4 : 0x66);
1029 1081
        emit_u8(e, avx ? 0xc1 : 0x41);
1030 1082
        emit_u8(e, avx ? 0x79 : 0x0f);
1031 1083
        emit_u8(e, 0xd6);
1032 -
        emit_r12_address(e, X_RAX, ins.r.rd * 8); /* vmovq [r12 + off], xmm0 */
1084 +
        emit_reg_address(e, X_RAX, ins.r.rd * 8); /* vmovq [r14 + disp], xmm0 */
1033 1085
        return;
1034 1086
    }
1035 1087
    bool word        = op == BM_ROLW || op == BM_RORW || op == BM_RORIW;
1036 1088
    bool word_source = op == BM_ADDUW || op == BM_SH1ADDUW ||
1037 1089
                       op == BM_SH2ADDUW || op == BM_SH3ADDUW ||
1238 1290
        break;
1239 1291
    }
1240 1292
    emit_store_rd(e, ins.r.rd);
1241 1293
}
1242 1294
1243 -
/* Give back the budget of the instructions after the first `insns` of the
1244 -
 * block, on a path that leaves after them: add r15, refund.  The refund is
1245 -
 * patched in once the length of the block is known. */
1246 -
static void emit_refund(struct emitter *e, u32 insns) {
1247 -
    emit_u8(e, 0x49);
1248 -
    emit_u8(e, 0x81);
1249 -
    emit_u8(e, modrm(3, 0, HBUDGET));
1250 -
    e->refunds[e->refunds_len]    = e->pos;
1251 -
    e->consumed[e->refunds_len++] = insns;
1252 -
    emit_u32(e, 0);
1253 -
}
1254 -
1255 1295
/* ---------- Vector translation ----------------------------------------- */
1256 1296
1257 1297
/* Return a failed memory operation to the interpreter before its side effects
1258 1298
 * when the jcc `condition` (the second opcode byte) holds. */
1259 1299
static void emit_fault_if(struct emitter *e, u32 pc, u8 condition) {
1260 -
    u32 bad = e->pos;
1300 +
    u32 ok = e->pos;
1261 1301
    emit_u8(e, 0x0f);
1262 -
    emit_u8(e, condition);
1302 +
    emit_u8(e, condition ^ 1);
1263 1303
    emit_u32(e, 0);
1264 -
    u32 ok = emit_jmp_placeholder(e);
1265 -
    patch_jcc(e, bad, e->pos);
1266 -
    emit_refund(e, e->insns);
1267 1304
    emit_block_exit(e, JIT_EXIT_FAULT, pc);
1268 -
    patch_jmp(e, ok, e->pos);
1305 +
    patch_jcc(e, ok, e->pos);
1269 1306
}
1270 1307
1271 1308
/* Load the architectural vector state into rax. */
1272 1309
static void emit_vector_base(struct emitter *e) {
1273 1310
    emit_u8(e, 0x49);
1889 1926
        bad[i] = emit_jne_placeholder(e);
1890 1927
    }
1891 1928
    u32 ok = emit_jmp_placeholder(e);
1892 1929
    for (u32 i = 0; i < 3; i++)
1893 1930
        patch_jcc(e, bad[i], e->pos);
1894 -
    emit_u8(e, 0x49);
1895 -
    emit_u8(e, 0x83);
1896 -
    emit_u8(e, modrm(3, 5, HBUDGET));
1897 -
    emit_u8(e, 1);
1898 -
    emit_block_exit(e, JIT_EXIT_FAULT, pc);
1931 +
    emit_charge(e, 1);
1932 +
    emit_store_pc_imm(e, pc);
1933 +
    emit_exit(e, JIT_EXIT_FAULT);
1899 1934
    patch_jmp(e, ok, e->pos);
1900 1935
}
1901 1936
1902 1937
/* Set the flags as cmp rs1, rs2 would, using held registers in place. */
1903 1938
static void emit_compare(struct emitter *e, u32 rs1, u32 rs2) {
1915 1950
        emit_u8(e, rex(true, b >= 8, false, a >= 8));
1916 1951
        emit_u8(e, 0x39);
1917 1952
        emit_u8(e, modrm(3, b, a));
1918 1953
        return;
1919 1954
    }
1920 -
    /* cmp a, [r12 + rs2 * 8] */
1921 -
    emit_r12_disp(e, 0x3B, a, rs2 * 8);
1955 +
    /* cmp a, [guest register file + rs2 * 8] */
1956 +
    emit_reg_disp(e, 0x3B, a, rs2 * 8);
1922 1957
}
1923 1958
1924 1959
/* Whether the block already holds the instruction at `pc`. */
1925 1960
static bool translated(struct emitter *e, u32 pc) {
1926 1961
    for (u32 i = 0; i < e->runs; i++) {
1938 1973
    e->run_start[e->runs]   = target;
1939 1974
    e->runs++;
1940 1975
    e->next_pc = target;
1941 1976
}
1942 1977
1943 -
/* Jump to the constant `target`, linking `next` in `rd`.  Continues the
1944 -
 * block at the target unless that would loop, and returns whether the block
1945 -
 * ends. */
1978 +
/* Jump to the constant `target`, linking `next` in `rd`.  End the block for
1979 +
 * a native call or when the target cannot be translated in this block. */
1946 1980
static bool emit_direct_jump(struct emitter *e, u32 rd, u32 next, u32 target) {
1947 1981
    if (rd != 0) {
1948 -
        emit_mov_imm64(e, HTMP1, (u64)next);
1949 -
        emit_store_guest(e, rd, HTMP1);
1982 +
        if (next <= INT32_MAX) {
1983 +
            emit_guest_imm32(e, rd, (i32)next);
1984 +
        } else {
1985 +
            emit_mov_imm64(e, HTMP1, (u64)next);
1986 +
            emit_store_guest(e, rd, HTMP1);
1987 +
        }
1988 +
    }
1989 +
    if (rd == RA && next <= INT32_MAX) {
1990 +
        emit_native_call(e, next, target);
1991 +
        return true;
1950 1992
    }
1951 1993
    if (e->runs > JIT_MAX_FOLLOWS || translated(e, target)) {
1952 1994
        emit_chain_exit(e, target);
1953 1995
        return true;
1954 1996
    }
1955 -
    /* A return to `next` can then be checked in place. */
1956 -
    if (rd == RA && next <= INT32_MAX) {
1957 -
        e->ra_known = true;
1958 -
        e->ra_value = next;
1959 -
    }
1960 1997
    follow(e, next, target);
1961 1998
    return false;
1962 1999
}
1963 2000
1964 -
/* Return to ra: leave the program when it is zero, else jump to it. */
2001 +
/* Return to ra.  Use ret when ra matches the top native frame. */
1965 2002
static void emit_ret(struct emitter *e) {
1966 2003
    emit_load_guest(e, HTMP1, RA);
1967 2004
2005 +
    /* cmp rax, [rsp + 8]; jne .fallback */
2006 +
    emit_rexw(e);
2007 +
    emit_u8(e, 0x3B);
2008 +
    emit_u8(e, modrm(1, X_RAX, X_RSP));
2009 +
    emit_u8(e, sib(0, X_RSP, X_RSP));
2010 +
    emit_u8(e, 8);
2011 +
    u32 mismatch = emit_jnz_placeholder(e);
2012 +
    /* The budget was positive at entry, so this subtraction clears OF. */
2013 +
    emit_charge(e, e->insns);
2014 +
    emit_native_ret(e);
2015 +
    patch_jcc(e, mismatch, e->pos);
2016 +
1968 2017
    /* test rax, rax */
1969 2018
    emit_rexw(e);
1970 2019
    emit_u8(e, 0x85);
1971 2020
    emit_u8(e, modrm(3, X_RAX, X_RAX));
1972 2021
1983 2032
    emit_u8(e, 0xFE); /* and rax, ~1 */
1984 2033
1985 2034
    emit_indirect_jump(e);
1986 2035
}
1987 2036
1988 -
/* Emit the side exits recorded for taken branches and checked returns. */
2037 +
/* Emit the side exits recorded for conditional branches. */
1989 2038
static void emit_side_exits(struct emitter *e) {
1990 2039
    /* Branches are never translated spilled. */
1991 2040
    e->spilled = false;
1992 2041
    for (u32 i = 0; i < e->sides_len; i++) {
1993 2042
        patch_jcc(e, e->sides[i].jcc, e->pos);
1994 -
        emit_refund(e, e->sides[i].insns);
1995 -
        if (e->sides[i].ret)
1996 -
            emit_ret(e);
1997 -
        else
1998 -
            emit_chain_exit(e, e->sides[i].target);
2043 +
        e->insns = e->sides[i].insns;
2044 +
        emit_chain_exit(e, e->sides[i].target);
1999 2045
    }
2000 2046
}
2001 2047
2002 -
/* Leave the block through a side exit when the jcc `cc` (its second opcode
2003 -
 * byte) holds: to `target`, or by returning to ra when `ret` is set. */
2004 -
static void emit_side_exit(struct emitter *e, u8 cc, u32 target, bool ret) {
2048 +
/* Leave the block for `target` when the jcc `cc` condition is true. */
2049 +
static void emit_side_exit(struct emitter *e, u8 cc, u32 target) {
2005 2050
    e->sides[e->sides_len].jcc    = e->pos;
2006 2051
    e->sides[e->sides_len].target = target;
2007 2052
    e->sides[e->sides_len].insns  = e->insns;
2008 -
    e->sides[e->sides_len].ret    = ret;
2009 2053
    e->sides_len++;
2010 2054
    emit_u8(e, 0x0F);
2011 2055
    emit_u8(e, cc);
2012 2056
    emit_u32(e, 0);
2013 2057
}
2027 2071
 * the r/m operand, held or in the register file.  `src` is not x0. */
2028 2072
static void emit_op_guest(
2029 2073
    struct emitter *e, bool wide, bool escape, u8 opcode, u8 dst, u32 src
2030 2074
) {
2031 2075
    u8 host = host_of(e, src);
2032 -
    /* The register file is addressed through r12, which needs REX.B. */
2076 +
    /* The register file is addressed through r14, which needs REX.B. */
2033 2077
    emit_u8(e, rex(wide, dst >= 8, false, host ? host >= 8 : true));
2034 2078
    if (escape)
2035 2079
        emit_u8(e, 0x0F);
2036 2080
    emit_u8(e, opcode);
2037 2081
    if (host)
2038 2082
        emit_u8(e, modrm(3, dst, host));
2039 2083
    else
2040 -
        emit_r12_address(e, dst, src * 8);
2084 +
        emit_reg_address(e, dst, src * 8);
2041 2085
}
2042 2086
2043 2087
/* Emit the group-1 operation `ext` (0 add, 1 or, 4 and, 5 sub, 6 xor, 7 cmp)
2044 2088
 * of `dst` with a sign-extended immediate. */
2045 2089
static void emit_op_imm(struct emitter *e, bool wide, u8 ext, u8 dst, i32 imm) {
2114 2158
    u8 d   = dest_of(e, rd);
2115 2159
    u8 src = host_of(e, rs1);
2116 2160
2117 2161
    switch (ins.i.opcode) {
2118 2162
    case OP_LUI:
2119 -
        emit_mov_imm32_sx(e, d, (i32)(ins.u.imm_31_12 << 12));
2120 -
        break;
2163 +
        emit_guest_imm32(e, rd, (i32)(ins.u.imm_31_12 << 12));
2164 +
        return;
2121 2165
2122 2166
    case OP_AUIPC: {
2123 2167
        i64 value = (i64)pc + (i64)(i32)(ins.u.imm_31_12 << 12);
2124 -
        if (value >= INT32_MIN && value <= INT32_MAX)
2125 -
            emit_mov_imm32_sx(e, d, (i32)value);
2126 -
        else
2127 -
            emit_mov_imm64(e, d, (u64)value);
2168 +
        if (value >= INT32_MIN && value <= INT32_MAX) {
2169 +
            emit_guest_imm32(e, rd, (i32)value);
2170 +
            return;
2171 +
        }
2172 +
        emit_mov_imm64(e, d, (u64)value);
2128 2173
        break;
2129 2174
    }
2130 2175
2131 2176
    case OP_IMM:
2132 2177
        if (f3 == 0 && rs1 == 0) {
2133 -
            emit_mov_imm32_sx(e, d, imm); /* li */
2134 -
            break;
2178 +
            emit_guest_imm32(e, rd, imm); /* li */
2179 +
            return;
2135 2180
        }
2136 2181
        if (f3 == 0 && src) {
2137 2182
            if (imm)
2138 2183
                emit_lea(e, true, d, src, imm);
2139 -
            else
2140 -
                emit_mov_rr(e, d, src); /* mv */
2184 +
            else {
2185 +
                emit_store_guest(e, rd, src); /* mv */
2186 +
                return;
2187 +
            }
2141 2188
            break;
2142 2189
        }
2143 2190
        if (f3 == 2 || f3 == 3) {
2144 2191
            /* slti, sltiu: cmp a, imm; setl/setb al; movzx d, al */
2145 2192
            u8 a = emit_base(e, rs1);
2218 2265
    else
2219 2266
        return false;
2220 2267
    if (rd == 0 || rs1 == 0 || rs2 == 0)
2221 2268
        return false;
2222 2269
2223 -
    /* Compute in d, unless it holds rs2, which the first operand would
2224 -
     * overwrite, or the word result needs sign extension from eax. */
2270 +
    /* For a commutative operation, keep the destination operand in place. */
2271 +
    if (op != 0x2B && rd == rs2) {
2272 +
        rs2 = rs1;
2273 +
        rs1 = rd;
2274 +
    }
2275 +
2276 +
    /* Keep rs2 intact until the operation reads it. */
2225 2277
    u8 d = dest_of(e, rd);
2226 -
    u8 t = wide && (rd == rs1 || rd != rs2) ? d : X_RAX;
2278 +
    u8 t = rd == rs1 || rd != rs2 ? d : X_RAX;
2227 2279
    if (t == X_RAX || rd != rs1)
2228 2280
        emit_load_guest(e, t, rs1);
2229 2281
    emit_op_guest(e, wide, escape, op, t, rs2);
2230 2282
    if (wide)
2231 2283
        emit_mov_rr(e, d, t);
2232 2284
    else
2233 -
        emit_movsxd(e, d, X_RAX);
2285 +
        emit_movsxd(e, d, t);
2234 2286
    emit_finish(e, rd, d);
2235 2287
    return true;
2236 2288
}
2237 2289
2238 2290
/* Offset of the reservation in the exchange area. */
2351 2403
    case OP_JALR: {
2352 2404
        i32 imm = get_i_imm(ins);
2353 2405
2354 2406
        /* Special-case RET: jalr x0, ra, 0. */
2355 2407
        if (ins.i.rd == 0 && ins.i.rs1 == RA && imm == 0) {
2356 -
            /* Returning to the call this block followed: check that ra
2357 -
             * still holds its return address and continue there. */
2358 -
            if (e->ra_known && e->sides_len < JIT_MAX_SIDE_EXITS &&
2359 -
                e->runs <= JIT_MAX_FOLLOWS && !translated(e, e->ra_value)) {
2360 -
                emit_load_guest(e, HTMP1, RA);
2361 -
                emit_rexw(e);
2362 -
                emit_u8(e, 0x3D);
2363 -
                emit_u32(e, e->ra_value); /* cmp rax, ra_value */
2364 -
                emit_side_exit(e, 0x85, 0, true);
2365 -
                follow(e, pc_next, e->ra_value);
2366 -
                return false;
2367 -
            }
2368 2408
            emit_ret(e);
2369 2409
            return true;
2370 2410
        }
2371 2411
2372 2412
        /* A jump through the register the previous instruction set with
2430 2470
            /* A branch around a trap, as in a bounds check, is taken: continue
2431 2471
             * at the target and leave to the trap through the side exit,
2432 2472
             * with the inverse condition. */
2433 2473
            if (traps(e, pc_next) && e->runs <= JIT_MAX_FOLLOWS &&
2434 2474
                !translated(e, target)) {
2435 -
                emit_side_exit(e, cc ^ 1, pc_next, false);
2475 +
                emit_side_exit(e, cc ^ 1, pc_next);
2436 2476
                follow(e, pc_next, target);
2437 2477
                return false;
2438 2478
            }
2439 -
            emit_side_exit(e, cc, target, false);
2479 +
            emit_side_exit(e, cc, target);
2440 2480
            return false;
2441 2481
        }
2442 2482
2443 2483
        u32 jcc_pos = e->pos;
2444 2484
        emit_u8(e, 0x0F);
2477 2517
            switch (ins.r.funct3) {
2478 2518
            case 0x0:
2479 2519
                emit_alu_rax_rcx(e, 0x01);
2480 2520
                break; /* add */
2481 2521
            case 0x1:  /* sll */
2482 -
                emit_and_ecx_imm8(e, 0x3F);
2483 2522
                emit_shift_rax_cl(e, 4);
2484 2523
                break;
2485 2524
            case 0x2: /* slt */
2486 2525
                emit_cmp_rax_rcx(e);
2487 2526
                emit_setcc_rax(e, 0x9C); /* setl */
2492 2531
                break;
2493 2532
            case 0x4:
2494 2533
                emit_alu_rax_rcx(e, 0x31);
2495 2534
                break; /* xor */
2496 2535
            case 0x5:  /* srl */
2497 -
                emit_and_ecx_imm8(e, 0x3F);
2498 2536
                emit_shift_rax_cl(e, 5);
2499 2537
                break;
2500 2538
            case 0x6:
2501 2539
                emit_alu_rax_rcx(e, 0x09);
2502 2540
                break; /* or */
2510 2548
            switch (ins.r.funct3) {
2511 2549
            case 0x0:
2512 2550
                emit_alu_rax_rcx(e, 0x29);
2513 2551
                break; /* sub */
2514 2552
            case 0x5:  /* sra */
2515 -
                emit_and_ecx_imm8(e, 0x3F);
2516 2553
                emit_shift_rax_cl(e, 7);
2517 2554
                break;
2518 2555
            }
2519 2556
            break;
2520 2557
2559 2596
            case 0x0: /* addw */
2560 2597
                emit_u8(e, 0x01);
2561 2598
                emit_u8(e, modrm(3, X_RCX, X_RAX));
2562 2599
                break;
2563 2600
            case 0x1: /* sllw */
2564 -
                emit_and_ecx_imm8(e, 0x1F);
2565 2601
                emit_u8(e, 0xD3);
2566 2602
                emit_u8(e, modrm(3, 4, X_RAX));
2567 2603
                break;
2568 2604
            case 0x5: /* srlw */
2569 -
                emit_and_ecx_imm8(e, 0x1F);
2570 2605
                emit_u8(e, 0xD3);
2571 2606
                emit_u8(e, modrm(3, 5, X_RAX));
2572 2607
                break;
2573 2608
            }
2574 2609
            break;
2578 2613
            case 0x0: /* subw */
2579 2614
                emit_u8(e, 0x29);
2580 2615
                emit_u8(e, modrm(3, X_RCX, X_RAX));
2581 2616
                break;
2582 2617
            case 0x5: /* sraw */
2583 -
                emit_and_ecx_imm8(e, 0x1F);
2584 2618
                emit_u8(e, 0xD3);
2585 2619
                emit_u8(e, modrm(3, 7, X_RAX));
2586 2620
                break;
2587 2621
            }
2588 2622
            break;
2662 2696
    default:
2663 2697
        return false;
2664 2698
    }
2665 2699
}
2666 2700
2667 -
/* Host registers other than rax and rcx that an instruction clobbers: the
2668 -
 * high multiplications and divisions use rdx and r11. */
2701 +
/* High multiplications and divisions use rdx.  Mulhsu also uses r11. */
2669 2702
static u32 scratch_mask(instr_t ins) {
2670 2703
    u32 opcode = ins.r.opcode;
2671 2704
    if ((opcode == OP_OP || opcode == OP_OP_32) && ins.r.funct7 == FUNCT7_MUL &&
2672 2705
        ins.r.funct3 != 0 && bitmanip_decode(ins.raw) == BM_NONE)
2673 -
        return 1u << X_RDX | 1u << X_R11;
2706 +
        return (1u << X_RDX) | (ins.r.funct3 == 2 ? 1u << X_R11 : 0);
2674 2707
    return 0;
2675 2708
}
2676 2709
2677 2710
/* Whether a scalar instruction may be translated with every guest register
2678 2711
 * in the register file: it addresses them only through host_of(), clobbers
2754 2787
    u32 h              = (block->guest_pc >> 2) & (JIT_BLOCK_HASH_SIZE - 1);
2755 2788
    block->hash_next   = jit->block_hash[h];
2756 2789
    jit->block_hash[h] = block;
2757 2790
}
2758 2791
2759 -
/* Whether an instruction may write ra.  Conservative: any instruction with
2760 -
 * ra in the destination field except stores and branches, which have none.
2761 -
 */
2762 -
static bool writes_ra(instr_t ins) {
2763 -
    u32 opcode = ins.r.opcode;
2764 -
    if (opcode == OP_STORE || opcode == OP_BRANCH || opcode == 39)
2765 -
        return false;
2766 -
    return ins.r.rd == RA;
2767 -
}
2768 -
2769 2792
/* Translate the block at `guest_pc` into `em`.  The vector shape guard is
2770 2793
 * emitted only when `em->guard` is set. */
2771 2794
static void translate_block(
2772 2795
    struct emitter *em, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes
2773 2796
) {
2774 2797
    /* if (budget <= 0) goto .exhausted */
2775 -
    emit_u8(em, 0x4D);
2776 -
    emit_u8(em, 0x85);
2777 -
    emit_u8(em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */
2778 2798
    u32 jle_pos = emit_jle_placeholder(em);
2779 2799
2780 2800
    if (em->guard)
2781 2801
        emit_vector_guard(em, guest_pc);
2782 2802
2783 -
    /* sub r15, insn_count -- imm32 is patched once the count is known. */
2784 -
    emit_u8(em, 0x49);
2785 -
    emit_u8(em, 0x81);
2786 -
    emit_u8(em, modrm(3, 5, HBUDGET));
2787 -
    u32 count_pos = em->pos;
2788 -
    emit_u32(em, 0);
2789 -
2790 2803
    u32 pc           = guest_pc;
2791 2804
    u32 insn_count   = 0;
2792 2805
    em->run_start[0] = guest_pc;
2793 2806
    em->runs         = 1;
2794 2807
    u32 prog_end_pc  = prog_base + prog_bytes;
2801 2814
        instr_t ins;
2802 2815
        memcpy(&ins, &mem[pc], sizeof(instr_t));
2803 2816
        insn_count++;
2804 2817
        em->insns = insn_count;
2805 2818
2806 -
        if (writes_ra(ins))
2807 -
            em->ra_known = false;
2808 2819
        em->next_pc     = pc + INSTR_SIZE;
2809 2820
        bool ends_block = translate_insn(em, ins, pc);
2810 2821
2811 2822
        /* Remember the value of an auipc for a jalr right after it. */
2812 2823
        em->auipc_reg = 0;
2833 2844
    emit_side_exits(em);
2834 2845
    em->spilled = false;
2835 2846
2836 2847
    /* .exhausted: out of budget, leave before running anything. */
2837 2848
    patch_jcc(em, jle_pos, em->pos);
2838 -
    emit_block_exit(em, JIT_EXIT_BRANCH, guest_pc);
2839 -
2840 -
    for (u32 i = 0; i < 4; i++)
2841 -
        em->buf[count_pos + i] = (u8)(insn_count >> (8 * i));
2842 -
2843 -
    for (u32 i = 0; i < em->refunds_len; i++) {
2844 -
        u32 refund = insn_count - em->consumed[i];
2845 -
        for (u32 j = 0; j < 4; j++)
2846 -
            em->buf[em->refunds[i] + j] = (u8)(refund >> (8 * j));
2847 -
    }
2849 +
    emit_store_pc_imm(em, guest_pc);
2850 +
    emit_exit(em, JIT_EXIT_BRANCH);
2848 2851
}
2849 2852
2850 2853
struct jit_block *jit_compile_block(
2851 2854
    struct jit_state *jit, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes
2852 2855
) {
2926 2929
    jit->exit_raw = jit->code_cache + e.pos;
2927 2930
    emit_u8(&e, 0x4D);
2928 2931
    emit_u8(&e, 0x89);
2929 2932
    emit_u8(&e, modrm(1, HBUDGET, HIO));
2930 2933
    emit_u8(&e, IO_BUDGET); /* mov [r14 + IO_BUDGET], r15 */
2934 +
2935 +
    /* Return through all outstanding native call frames before returning to C.
2936 +
     */
2937 +
    jit->exit_unwind = jit->code_cache + e.pos;
2938 +
    emit_u8(&e, 0x49);
2939 +
    emit_u8(&e, 0x3B);
2940 +
    emit_u8(&e, modrm(1, X_RSP, HIO));
2941 +
    emit_u8(&e, IO_STACK_BASE); /* cmp rsp, [r14 + IO_STACK_BASE] */
2942 +
    u32 frames = emit_jnz_placeholder(&e);
2943 +
2931 2944
    emit_u8(&e, 0x41);
2932 2945
    emit_u8(&e, 0x5F); /* pop r15 */
2933 2946
    emit_u8(&e, 0x41);
2934 2947
    emit_u8(&e, 0x5E); /* pop r14 */
2935 2948
    emit_u8(&e, 0x41);
2938 2951
    emit_u8(&e, 0x5C); /* pop r12 */
2939 2952
    emit_u8(&e, 0x5D); /* pop rbp */
2940 2953
    emit_u8(&e, 0x5B); /* pop rbx */
2941 2954
    emit_u8(&e, 0xC3); /* ret */
2942 2955
2943 -
    /* Entry: (rdi=regs, rsi=memory, rdx=io, rcx=code, r8=budget).  Blocks
2944 -
     * make no calls, so the stack alignment is irrelevant. */
2956 +
    patch_jcc(&e, frames, e.pos);
2957 +
    /* Set OF for the return continuation. */
2958 +
    emit_u8(&e, 0xB1);
2959 +
    emit_u8(&e, INT8_MAX); /* mov cl, 127 */
2960 +
    emit_u8(&e, 0x80);
2961 +
    emit_u8(&e, modrm(3, 0, X_RCX));
2962 +
    emit_u8(&e, 1); /* add cl, 1 */
2963 +
    emit_native_ret(&e);
2964 +
2965 +
    /* Entry: (rdi=memory, rsi=io, rdx=code, rcx=budget). */
2945 2966
    jit->enter = (jit_enter_fn)(void *)(jit->code_cache + e.pos);
2946 2967
    emit_u8(&e, 0x53); /* push rbx */
2947 2968
    emit_u8(&e, 0x55); /* push rbp */
2948 2969
    emit_u8(&e, 0x41);
2949 2970
    emit_u8(&e, 0x54); /* push r12 */
2951 2972
    emit_u8(&e, 0x55); /* push r13 */
2952 2973
    emit_u8(&e, 0x41);
2953 2974
    emit_u8(&e, 0x56); /* push r14 */
2954 2975
    emit_u8(&e, 0x41);
2955 2976
    emit_u8(&e, 0x57); /* push r15 */
2977 +
    emit_mov_rr(&e, HMEM, X_RDI);
2978 +
    emit_mov_rr(&e, HIO, X_RSI);
2979 +
    emit_mov_rr(&e, HBUDGET, X_RCX);
2980 +
    emit_mov_rr(&e, X_RAX, X_RDX);
2981 +
2982 +
    /* Set the bounds of the native call stack. */
2956 2983
    emit_u8(&e, 0x49);
2957 2984
    emit_u8(&e, 0x89);
2958 -
    emit_u8(&e, 0xFC); /* mov r12, rdi */
2959 -
    emit_u8(&e, 0x49);
2960 -
    emit_u8(&e, 0x89);
2961 -
    emit_u8(&e, 0xF5); /* mov r13, rsi */
2962 -
    emit_u8(&e, 0x49);
2963 -
    emit_u8(&e, 0x89);
2964 -
    emit_u8(&e, 0xD6); /* mov r14, rdx */
2985 +
    emit_u8(&e, modrm(1, X_RSP, HIO));
2986 +
    emit_u8(&e, IO_STACK_BASE); /* mov [r14 + IO_STACK_BASE], rsp */
2987 +
    emit_u8(&e, 0x4C);
2988 +
    emit_u8(&e, 0x8D);
2989 +
    emit_u8(&e, modrm(2, X_R11, X_RSP));
2990 +
    emit_u8(&e, sib(0, X_RSP, X_RSP));
2991 +
    emit_u32(&e, (u32)-JIT_CALL_STACK_SIZE); /* lea r11, [rsp - stack size] */
2965 2992
    emit_u8(&e, 0x4D);
2966 2993
    emit_u8(&e, 0x89);
2967 -
    emit_u8(&e, 0xC7); /* mov r15, r8 */
2968 -
    emit_u8(&e, 0x48);
2969 -
    emit_u8(&e, 0x89);
2970 -
    emit_u8(&e, 0xC8); /* mov rax, rcx */
2994 +
    emit_u8(&e, modrm(1, X_R11, HIO));
2995 +
    emit_u8(&e, IO_STACK_FLOOR); /* mov [r14 + IO_STACK_FLOOR], r11 */
2996 +
2971 2997
    /* The argument registers are free now: load the held guest registers. */
2972 2998
    emit_transfer(&e, ALL_HOSTS, false);
2999 +
3000 +
    /* The entry frame returns to C when the guest returns to PC zero. */
3001 +
    emit_u8(&e, 0x6A);
3002 +
    emit_u8(&e, 0); /* push guest link */
3003 +
    emit_u8(&e, 0x4D);
3004 +
    emit_u8(&e, 0x85);
3005 +
    emit_u8(&e, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */
2973 3006
    emit_u8(&e, 0xFF);
2974 -
    emit_u8(&e, 0xE0); /* jmp rax */
3007 +
    emit_u8(&e, modrm(3, 2, X_RAX)); /* call rax */
3008 +
    emit_unwind_check(&e);
3009 +
    emit_store_pc_imm(&e, 0);
3010 +
    emit_exit(&e, JIT_EXIT_RET);
2975 3011
2976 3012
    memcpy(jit->code_cache, buf, e.pos);
2977 3013
    jit->code_cache_used = e.pos;
2978 3014
}
2979 3015
jit.h +22 -13
9 9
 * result, and execute the native code on subsequent visits.
10 10
 */
11 11
#ifndef JIT_H
12 12
#define JIT_H
13 13
14 +
#include <string.h>
15 +
14 16
#include "riscv.h"
15 17
#include "types.h"
16 18
17 19
/* Forward declarations. */
18 20
struct cpu;
56 58
enum jit_exit {
57 59
    JIT_EXIT_BRANCH = 0, /* Return to C, which continues at io.pc. */
58 60
    JIT_EXIT_ECALL  = 1, /* ECALL -- needs interpreter handling. */
59 61
    JIT_EXIT_EBREAK = 2, /* EBREAK -- needs interpreter handling. */
60 62
    JIT_EXIT_FAULT  = 3, /* Memory fault -- needs interpreter handling. */
61 -
    JIT_EXIT_CHAIN  = 4, /* Direct jump to a known PC; io.patch is its site. */
62 -
    JIT_EXIT_RET    = 5, /* Program returned (jalr x0, ra, 0 with ra==0). */
63 +
    JIT_EXIT_CHAIN = 4, /* Direct branch to a known PC; io.patch is its site. */
64 +
    JIT_EXIT_RET   = 5, /* Program returned (jalr x0, ra, 0 with ra==0). */
63 65
};
64 66
65 67
/* A translated block of native code. */
66 68
struct jit_block {
67 69
    u32               guest_pc;  /* Guest PC this block starts at. */
78 80
    struct vector_state *vector;      /* Architectural vector registers. */
79 81
    u64                  memory_size; /* Accessible hosted memory bytes. */
80 82
    /* LR reservation: the address in the low word and the access width in
81 83
     * the high word, or zero when none is held. */
82 84
    u64 reservation;
85 +
    /* Bounds of the host stack frames for translated calls. */
86 +
    u64 stack_base;
87 +
    u64 stack_floor;
83 88
};
84 89
85 90
/* Reservation value of an LR of `size` bytes at `address`. */
86 91
static inline u64 jit_reservation(u32 address, u32 size) {
87 92
    return (u64)size << 32 | address;
92 97
struct jit_jump_entry {
93 98
    u32 guest_pc; /* Guest PC, or JIT_JUMP_CACHE_EMPTY. */
94 99
    u8 *code;     /* Entry point of the block at that PC. */
95 100
};
96 101
97 -
/* Signature of the shared entry trampoline.  It saves host registers, pins
98 -
 * regs/memory/io/budget and jumps to `code`; the shared exit stub returns the
99 -
 * exit reason. */
102 +
/* Signature of the shared entry trampoline.  It loads guest registers from
103 +
 * the register file adjacent to io and calls the first translated block. */
100 104
typedef int (*jit_enter_fn)(
101 -
    u64 *regs, u8 *memory, struct jit_io *io, u8 *code, i64 budget
105 +
    u8 *memory, struct jit_io *io, u8 *code, i64 budget
102 106
);
103 107
104 108
/* Complete JIT state. */
105 109
struct jit_state {
106 110
    /* Executable code cache (mmap'd RWX). */
115 119
    u32              block_count;
116 120
117 121
    /* Shared entry trampoline and exit stubs, at the start of the cache.
118 122
     * `exit_stub` writes the guest registers held in host registers back to
119 123
     * the register file and falls into `exit_raw`, which returns without
120 -
     * writing them. */
124 +
     * writing them.  `exit_unwind` returns through pending native calls. */
121 125
    jit_enter_fn enter;
122 126
    u8          *exit_stub;
123 127
    u8          *exit_raw;
128 +
    u8          *exit_unwind;
124 129
125 -
    /* Exit information for the current run. */
130 +
    /* The register file and exit information share one host base register. */
131 +
    u64           regs[32];
126 132
    struct jit_io io;
127 133
128 134
    /* Direct-mapped cache of indirect jump targets, read by generated code
129 135
     * and filled by C. */
130 136
    struct jit_jump_entry jump_cache[JIT_JUMP_CACHE_SIZE];
194 200
        &jit->jump_cache[(block->guest_pc >> 2) & (JIT_JUMP_CACHE_SIZE - 1)];
195 201
    ent->guest_pc = block->guest_pc;
196 202
    ent->code     = block->code;
197 203
}
198 204
199 -
/* Point the direct jump whose rel32 field is at `site` to `target`. */
205 +
/* Point the direct branch whose rel32 field is at `site` to `target`. */
200 206
void jit_patch_chain(u8 *site, struct jit_block *target);
201 207
202 208
/* Run translated code starting at `block`, with room for `budget` more guest
203 -
 * instructions.  Generated code follows direct and cached indirect jumps
204 -
 * without returning.  Returns the exit reason; the next PC, the remaining
205 -
 * budget (which may be negative) and any patch site are left in jit->io. */
209 +
 * instructions.  Copy guest register state to the JIT at entry and back at
210 +
 * exit.  Return the exit reason and leave the next PC, remaining budget and
211 +
 * any patch site in jit->io. */
206 212
static inline int jit_run(
207 213
    struct jit_state *jit,
208 214
    struct jit_block *block,
209 215
    u64              *regs,
210 216
    u8               *memory,
211 217
    i64               budget
212 218
) {
213 -
    return jit->enter(regs, memory, &jit->io, block->code, budget);
219 +
    memcpy(jit->regs, regs, sizeof(jit->regs));
220 +
    int reason = jit->enter(memory, &jit->io, block->code, budget);
221 +
    memcpy(regs, jit->regs, sizeof(jit->regs));
222 +
    return reason;
214 223
}
215 224
216 225
#endif /* JIT_H */
test/jit_calls.c +20 -4
1 1
/* Writes a raw RV64 image for comparing translated and interpreted runs.
2 2
 *
3 3
 * The program makes calls through jal and through auipc and jalr, also with
4 4
 * a jalr that reads the register it writes.  It returns through ra, saves ra
5 5
 * in a non-leaf function, branches around ebreak as a bounds check does, and
6 -
 * uses LR and SC.  Translated code follows these
7 -
 * calls and checks the returns in place, so both runs must end with the
8 -
 * same status after the same number of instructions. */
6 +
 * uses LR and SC.  Recursive calls exceed the native call stack limit.
7 +
 * Both runs must end with the same status and instruction count. */
9 8
#include <stdint.h>
10 9
#include <stdio.h>
11 10
12 11
typedef uint32_t u32;
13 12
typedef int32_t  i32;
17 16
18 17
static u32 code[256];
19 18
static u32 len;
20 19
21 20
/* Labels and the instructions that refer to them. */
22 -
enum label { LOOP, LEAF, LEAF_OK, NEST, LABELS };
21 +
enum label { LOOP, LEAF, LEAF_OK, NEST, RECURSE, RECURSE_DONE, LABELS };
23 22
static u32 labels[LABELS];
24 23
static struct {
25 24
    u32        at;
26 25
    enum label to;
27 26
    u32        kind; /* 0 jal, 1 branch, 2 jalr after auipc */
87 86
        fprintf(stderr, "usage: %s <output>\n", argv[0]);
88 87
        return 1;
89 88
    }
90 89
    /* main: sum calls over 200 iterations into a0. */
91 90
    addi(A0, ZERO, 0);
91 +
    addi(A1, ZERO, 600);
92 +
    ref(0, RA << 7 | 0x6f, RECURSE);
93 +
    addi(A1, ZERO, 600);
94 +
    ref(0, RA << 7 | 0x6f, RECURSE);
92 95
    addi(S1, ZERO, 200);
93 96
    addi(T6, SP, -64);
94 97
    mark(LOOP);
95 98
    addi(A1, S1, 0);
96 99
    ref(0, RA << 7 | 0x6f, LEAF);             /* jal ra, leaf */
139 142
    ref(0, RA << 7 | 0x6f, LEAF);
140 143
    emit(i_type(0x03, RA, 3, SP, 8));
141 144
    addi(SP, SP, 16);
142 145
    emit(i_type(0x67, ZERO, 0, RA, 0));
143 146
147 +
    /* recurse(a1): add one to the result for each level after returning. */
148 +
    mark(RECURSE);
149 +
    ref(1, r_type(0x63, 0, 0, A1, ZERO, 0), RECURSE_DONE);
150 +
    addi(SP, SP, -16);
151 +
    emit(s_type(RA, SP, 3, 8));
152 +
    addi(A1, A1, -1);
153 +
    ref(0, RA << 7 | 0x6f, RECURSE);
154 +
    emit(i_type(0x03, RA, 3, SP, 8));
155 +
    addi(SP, SP, 16);
156 +
    addi(A0, A0, 1);
157 +
    mark(RECURSE_DONE);
158 +
    emit(i_type(0x67, ZERO, 0, RA, 0));
159 +
144 160
    resolve();
145 161
    FILE *f = fopen(argv[1], "wb");
146 162
    if (!f || fwrite(code, 4, len, f) != len || fclose(f)) {
147 163
        perror(argv[1]);
148 164
        return 1;
test/jit_vector.c +44 -0
271 271
    require_native = false;
272 272
    u32 invalid    = alu(25, 0, 9, 8, 16);
273 273
    check(&invalid, 1, 0xd3, 33, 0, 128);
274 274
}
275 275
276 +
/* Check the budget at entry and across linked and indirect block exits. */
277 +
static void check_budget(void) {
278 +
    static struct jit_state jit;
279 +
    u8                      mem[CODE + 24] = { 0 };
280 +
    u64                     x[32]          = { 0 };
281 +
    const u32               code[]         = {
282 +
        0x00000063, /* beq zero, zero, 0 */
283 +
        0x00100513, /* addi a0, zero, 1 */
284 +
        0x00008067, /* ret */
285 +
        0x00028067, /* jalr zero, t0, 0 */
286 +
        0x00200513, /* addi a0, zero, 2 */
287 +
        0x00008067, /* ret */
288 +
    };
289 +
    memcpy(mem + CODE, code, sizeof(code));
290 +
    assert(jit_init(&jit));
291 +
292 +
    struct jit_block *loop = jit_get_block(&jit, CODE, mem, CODE, sizeof(code));
293 +
    assert(loop);
294 +
    assert(jit_run(&jit, loop, x, mem, -1) == JIT_EXIT_BRANCH);
295 +
    assert(jit.io.pc == CODE && jit.io.budget == -1 && x[10] == 0);
296 +
    assert(jit_run(&jit, loop, x, mem, 5) == JIT_EXIT_CHAIN);
297 +
    assert(jit.io.pc == CODE && jit.io.budget == 4 && x[10] == 0);
298 +
    jit_patch_chain(jit.io.patch, loop);
299 +
    assert(jit_run(&jit, loop, x, mem, 5) == JIT_EXIT_BRANCH);
300 +
    assert(jit.io.pc == CODE && jit.io.budget == 0 && x[10] == 0);
301 +
302 +
    u32               target = CODE + 16;
303 +
    struct jit_block *jump =
304 +
        jit_get_block(&jit, CODE + 12, mem, CODE, sizeof(code));
305 +
    struct jit_block *dest =
306 +
        jit_get_block(&jit, target, mem, CODE, sizeof(code));
307 +
    assert(jump && dest);
308 +
    x[5] = target;
309 +
    assert(jit_run(&jit, jump, x, mem, 2) == JIT_EXIT_BRANCH);
310 +
    assert(jit.io.pc == target && jit.io.budget == 1 && x[10] == 0);
311 +
    jit_cache_jump(&jit, dest);
312 +
    assert(jit_run(&jit, jump, x, mem, 1) == JIT_EXIT_BRANCH);
313 +
    assert(jit.io.pc == target && jit.io.budget == 0 && x[10] == 0);
314 +
    assert(jit_run(&jit, jump, x, mem, 2) == JIT_EXIT_RET);
315 +
    assert(jit.io.budget == -1 && x[10] == 2);
316 +
    jit_destroy(&jit);
317 +
}
318 +
276 319
int main(void) {
320 +
    check_budget();
277 321
    check_bulk();
278 322
    check_word_groups();
279 323
    u32 ops[] = {
280 324
        alu(0, 0, 8, 10, 12),   alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0),
281 325
        alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31),   alu(9, 3, 8, 8, 3),