jit: Hold hot registers and translate superblocks

df02cea04c38773d5b79a1e05f510986a745fb619f62c97a637b371ba81257d8
Halves the time of Radiance self-compilation (1.50s to 0.75s, 51%
fewer cycles) with identical output.

Nine guest registers live in host registers while translated code
runs: the entry trampoline loads them and the exit stub stores them.
Instructions whose emitters address the register file directly run
spilled, sharing one spill and reload across neighbours.

Loads and stores fold their address into one instruction, and common
integer operations and branch compares work on held registers.

Blocks continue past conditional branches, leaving through side
exits that refund the budget, and past branches around ebreak on the
taken path. They follow jal and auipc+jalr calls, and check a return
to a followed call in place. LR and SC are translated, with the
reservation shared with the interpreter.

The jump cache grows to 64K entries and the code cache to 64 MB.

Assisted-By: Claude:opus-5.5
Change-Id: I700003506f45b2291e95c646580b4ba2be7734de
Alexis Sellier committed ago 1 parent edb99b53
Makefile +18 -2
62 62
# Compare hosted vector translation with architectural interpreter state.
63 63
bin/jit-vector-test: test/jit_vector.c jit.c jit.h vector.c vector.h riscv.c riscv.h riscv/debug.c riscv/debug.h bitmanip.h color.h types.h
64 64
	@mkdir -p bin
65 65
	@$(CC) $(CPPFLAGS) -std=c99 -O2 -g -Wall -Wextra -fsanitize=address,undefined test/jit_vector.c jit.c vector.c riscv.c riscv/debug.c -o $@
66 66
67 -
test-jit: bin/jit-vector-test
67 +
test-jit: bin/jit-vector-test test-jit-calls
68 68
	@ASAN_OPTIONS=detect_leaks=0 timeout 60 $<
69 69
70 -
.PHONY: test-jit
70 +
# Compare translated and interpreted runs of calls, returns and LR/SC.
71 +
bin/jit-calls-gen: test/jit_calls.c
72 +
	@mkdir -p bin
73 +
	@$(CC) $(CPPFLAGS) -std=c99 -Wall -Wextra -g $< -o $@
74 +
75 +
test-jit-calls: bin/emulator bin/jit-calls-gen
76 +
	@bin/jit-calls-gen bin/jit-calls.bin
77 +
	@jit=$$(bin/emulator -run -count-instructions bin/jit-calls.bin 2>&1; \
78 +
	        echo "status $$?"); \
79 +
	 int=$$(bin/emulator -run -count-instructions -no-jit bin/jit-calls.bin 2>&1; \
80 +
	        echo "status $$?"); \
81 +
	 if [ "$$jit" != "$$int" ]; then \
82 +
	     echo "JIT call checks failed: '$$jit' != '$$int'"; exit 1; \
83 +
	 fi; \
84 +
	 echo "JIT call checks passed"
85 +
86 +
.PHONY: test-jit test-jit-calls
emulator.c +19 -1
1556 1556
    else
1557 1557
        abi_linux_call(cpu, pc);
1558 1558
}
1559 1559
1560 1560
/* Execute a single instruction. */
1561 +
/* Take the LR reservation held by translated code. */
1562 +
static void reservation_from_jit(struct cpu *cpu, const struct jit_io *io) {
1563 +
    cpu->reserved         = io->reservation != 0;
1564 +
    cpu->reservation      = (u32)io->reservation;
1565 +
    cpu->reservation_size = (u32)(io->reservation >> 32);
1566 +
}
1567 +
1568 +
/* Hand the LR reservation back to translated code. */
1569 +
static void reservation_to_jit(const struct cpu *cpu, struct jit_io *io) {
1570 +
    io->reservation =
1571 +
        cpu->reserved ? jit_reservation(cpu->reservation, cpu->reservation_size)
1572 +
                      : 0;
1573 +
}
1574 +
1561 1575
static void cpu_execute(struct cpu *cpu, enum display display, bool headless) {
1562 1576
    if (cpu_out_of_bounds(cpu)) {
1563 1577
        cpu->running = false;
1564 1578
        emit_fault_diagnostics(cpu, cpu->pc);
1565 1579
        if (headless) {
2581 2595
                    &jit, cpu.pc, memory, program_base, program_bytes
2582 2596
                );
2583 2597
                if (!block) {
2584 2598
                    /* Cache full or compilation error -- fall back to
2585 2599
                     * interpreter for remainder. */
2600 +
                    reservation_from_jit(&cpu, &jit.io);
2586 2601
                    while (cpu.running && steps++ < max_steps) {
2587 2602
                        cpu_execute(&cpu, display, true);
2588 2603
                    }
2589 2604
                    break;
2590 2605
                }
2615 2630
                    break;
2616 2631
2617 2632
                default:
2618 2633
                    /* ECALL, EBREAK, FAULT -- interpreter handles it.
2619 2634
                     * The instruction is already counted above,
2620 -
                     * so don't increment steps again. */
2635 +
                     * so don't increment steps again.  It sees the
2636 +
                     * reservation of translated LR and SC. */
2637 +
                    reservation_from_jit(&cpu, &jit.io);
2621 2638
                    cpu_execute(&cpu, display, true);
2639 +
                    reservation_to_jit(&cpu, &jit.io);
2622 2640
                    break;
2623 2641
                }
2624 2642
            }
2625 2643
            jit_destroy(&jit);
2626 2644
        } else {
jit.c +937 -348
45 45
    u8  *buf;      /* Start of buffer. */
46 46
    u32  pos;      /* Current write position. */
47 47
    u32  capacity; /* Maximum bytes. */
48 48
    bool overflow; /* Set if we ran out of space. */
49 49
    bool vector;   /* Block uses four-lane 32-bit SIMD. */
50 -
    u32  insns;    /* Instructions through the current operation. */
50 +
    bool guard;    /* Emit the vector shape guard at block entry. */
51 +
    /* The register file holds every guest register and host registers may be
52 +
     * clobbered: set from an instruction that addresses the register file
53 +
     * directly until the held registers are reloaded. */
54 +
    bool spilled;
55 +
    u32  insns; /* Instructions through the current operation. */
51 56
    /* Each indexed load can fail three checks for each of four lanes. */
52 57
    u32 refunds[JIT_MAX_BLOCK_INSNS * 12];  /* Budget patch offsets. */
53 58
    u32 consumed[JIT_MAX_BLOCK_INSNS * 12]; /* Instructions at each patch. */
54 59
    u32 refunds_len;                        /* Number of guarded exits. */
55 60
61 +
    /* Exits from inside the block, on a branch or a checked return, emitted
62 +
     * after its main path. */
63 +
    struct {
64 +
        u32  jcc;    /* Offset of the jcc rel32 instruction. */
65 +
        u32  target; /* Guest PC of the branch target. */
66 +
        u32  insns;  /* Instructions executed when it is taken. */
67 +
        bool ret;    /* Return to ra instead of jumping to `target`. */
68 +
    } sides[JIT_MAX_SIDE_EXITS];
69 +
    u32 sides_len;
70 +
71 +
    /* Guest PC after the current instruction. */
72 +
    u32 next_pc;
73 +
    /* Runs of consecutive guest instructions translated so far: the block
74 +
     * starts one, and each direct jump it follows starts another.  The last
75 +
     * run ends at next_pc. */
76 +
    u32 run_start[JIT_MAX_FOLLOWS + 1];
77 +
    u32 run_end[JIT_MAX_FOLLOWS + 1];
78 +
    u32 runs;
79 +
    /* The previous instruction set `auipc_reg` to `auipc_value` with auipc,
80 +
     * or `auipc_reg` is zero. */
81 +
    u32 auipc_reg;
82 +
    u64 auipc_value;
83 +
    /* ra holds `ra_value`, set by a call the block followed. */
84 +
    bool ra_known;
85 +
    u32  ra_value;
86 +
87 +
    u8               *mem;       /* Guest memory, for looking ahead. */
88 +
    u32               prog_base; /* Guest code bounds. */
89 +
    u32               prog_end;
56 90
    struct jit_state *jit;  /* For the exit stub and jump cache addresses. */
57 91
    u8               *dest; /* Address the buffer will be copied to. */
58 92
};
59 93
60 94
static inline void emit_u8(struct emitter *e, u8 b) {
119 153
120 154
/* Scratch registers for codegen (caller-saved, not pinned). */
121 155
#define HTMP1 X_RAX
122 156
#define HTMP2 X_RCX
123 157
158 +
/* Guest registers held in host registers for the whole time generated code
159 +
 * runs, chosen by how often a compiler workload uses them.  The entry
160 +
 * trampoline loads them from the register file and the exit stub stores them
161 +
 * back, so C always sees the register file.  Zero entries are not held. */
162 +
static const u8 guest_host[32] = {
163 +
    [5]  = X_RBX, /* t0 */
164 +
    [29] = X_RBP, /* t4 */
165 +
    [6]  = X_RSI, /* t1 */
166 +
    [2]  = X_RDI, /* sp */
167 +
    [30] = X_R8,  /* t5 */
168 +
    [7]  = X_R9,  /* t2 */
169 +
    [10] = X_R10, /* a0 */
170 +
    [28] = X_R11, /* t3 */
171 +
    [11] = X_RDX, /* a1 */
172 +
};
173 +
174 +
/* Host register holding `guest`, or zero if it lives in the register file. */
175 +
static inline u8 host_of(struct emitter *e, u32 guest) {
176 +
    return e->spilled ? 0 : guest_host[guest];
177 +
}
178 +
124 179
/* ---------- Common encoding helpers ------------------------------------ */
125 180
126 181
/* Emit REX.W prefix (64-bit operand size, no extended regs). */
127 182
static inline void emit_rexw(struct emitter *e) {
128 183
    emit_u8(e, 0x48);
166 221
    emit_u8(e, rex(true, reg >= 8, false, true)); /* B=1 for r12 */
167 222
    emit_u8(e, opcode);
168 223
    emit_r12_address(e, reg, off);
169 224
}
170 225
171 -
/* Load guest register `guest` into host `dst` from [r12 + guest*8]. */
226 +
/* mov dst, src (64-bit), omitted when they are the same register. */
227 +
static void emit_mov_rr(struct emitter *e, u8 dst, u8 src) {
228 +
    if (dst == src)
229 +
        return;
230 +
    emit_u8(e, rex(true, src >= 8, false, dst >= 8));
231 +
    emit_u8(e, 0x89);
232 +
    emit_u8(e, modrm(3, src, dst));
233 +
}
234 +
235 +
/* Load guest register `guest` into host `dst`. */
172 236
static void emit_load_guest(struct emitter *e, enum x86reg dst, u32 guest) {
173 237
    if (guest == 0) {
174 238
        /* x0 is always zero -- xor dst, dst. */
175 239
        emit_u8(e, rex(true, dst >= 8, false, dst >= 8));
176 240
        emit_u8(e, 0x31);
177 241
        emit_u8(e, modrm(3, dst, dst));
178 242
        return;
179 243
    }
244 +
    u8 host = host_of(e, guest);
245 +
    if (host) {
246 +
        emit_mov_rr(e, dst, host);
247 +
        return;
248 +
    }
180 249
    emit_r12_disp(e, 0x8B, dst, guest * 8);
181 250
}
182 251
183 252
/* Load and zero-extend the low word of a guest register. */
184 253
static void emit_load_guest32(struct emitter *e, enum x86reg dst, u32 guest) {
185 254
    if (guest == 0) {
186 255
        emit_load_guest(e, dst, guest);
187 256
        return;
188 257
    }
258 +
    u8 host = host_of(e, guest);
259 +
    if (host) {
260 +
        /* mov dst32, host32 */
261 +
        if (dst >= 8 || host >= 8)
262 +
            emit_u8(e, rex(false, host >= 8, false, dst >= 8));
263 +
        emit_u8(e, 0x89);
264 +
        emit_u8(e, modrm(3, host, dst));
265 +
        return;
266 +
    }
189 267
    emit_u8(e, rex(false, dst >= 8, false, true));
190 268
    emit_u8(e, 0x8b);
191 269
    emit_r12_address(e, dst, guest * 8);
192 270
}
193 271
194 -
/* Store host `src` to guest register [r12 + guest*8]. */
272 +
/* Store host `src` to guest register `guest`. */
195 273
static void emit_store_guest(struct emitter *e, u32 guest, enum x86reg src) {
196 274
    if (guest == 0)
197 275
        return;
276 +
    u8 host = host_of(e, guest);
277 +
    if (host) {
278 +
        emit_mov_rr(e, host, src);
279 +
        return;
280 +
    }
198 281
    emit_r12_disp(e, 0x89, src, guest * 8);
199 282
}
200 283
284 +
/* Every host register, as a mask for emit_transfer(). */
285 +
#define ALL_HOSTS 0xFFFFu
286 +
287 +
/* Store (`store`) or load the held guest registers whose host registers are
288 +
 * in `mask` to or from the register file. */
289 +
static void emit_transfer(struct emitter *e, u32 mask, bool store) {
290 +
    for (u32 g = 1; g < 32; g++) {
291 +
        if (guest_host[g] && (mask >> guest_host[g]) & 1)
292 +
            emit_r12_disp(e, store ? 0x89 : 0x8B, guest_host[g], g * 8);
293 +
    }
294 +
}
295 +
296 +
/* Make the register file hold every guest register before an instruction
297 +
 * that addresses it directly or clobbers host registers. */
298 +
static void emit_spill(struct emitter *e) {
299 +
    emit_transfer(e, ALL_HOSTS, true);
300 +
    e->spilled = true;
301 +
}
302 +
303 +
/* Reload the held guest registers after such an instruction. */
304 +
static void emit_reload(struct emitter *e) {
305 +
    emit_transfer(e, ALL_HOSTS, false);
306 +
    e->spilled = false;
307 +
}
308 +
201 309
/* mov reg, imm64 (REX.W + B8+rd) */
202 310
static void emit_mov_imm64(struct emitter *e, enum x86reg dst, u64 imm) {
203 311
    emit_u8(e, rex(true, false, false, dst >= 8));
204 312
    emit_u8(e, (u8)(0xB8 + (dst & 7)));
205 313
    emit_u64(e, imm);
278 386
    emit_u8(e, 0xC7);
279 387
    emit_u8(e, modrm(0, 0, HIO));
280 388
    emit_u32(e, pc);
281 389
}
282 390
283 -
/* mov eax, reason; jmp exit_stub. */
391 +
/* mov eax, reason; jmp exit_stub.  While spilled, the register
392 +
 * file is already current and the host registers may be stale. */
284 393
static void emit_exit(struct emitter *e, u32 exit_reason) {
285 394
    emit_u8(e, 0xB8);
286 395
    emit_u32(e, exit_reason); /* mov eax, reason */
287 -
    emit_jmp_abs(e, e->jit->exit_stub);
396 +
    emit_jmp_abs(e, e->spilled ? e->jit->exit_raw : e->jit->exit_stub);
288 397
}
289 398
290 399
/* Store next-PC to *r14 and leave the cache with `reason`. */
291 400
static void emit_block_exit(
292 401
    struct emitter *e, enum jit_exit reason, u32 next_pc
297 406
298 407
/* Jump to the guest block at the constant PC `target`.  Emits a jmp rel32
299 408
 * that initially falls into a stub returning JIT_EXIT_CHAIN with the address
300 409
 * of its rel32 field in io.patch; see jit_patch_chain. */
301 410
static void emit_chain_exit(struct emitter *e, u32 target) {
411 +
    /* A linked jump enters the target with the guest registers held. */
412 +
    bool spilled = e->spilled;
413 +
    if (spilled)
414 +
        emit_reload(e);
415 +
302 416
    u32 site = e->pos;
303 417
    emit_u8(e, 0xE9);
304 418
    emit_u32(e, 0); /* jmp .stub (next instruction) */
305 419
306 420
    emit_store_pc_imm(e, target);
316 430
    emit_u8(e, 0x89);
317 431
    emit_u8(e, modrm(1, X_RAX, HIO));
318 432
    emit_u8(e, IO_PATCH);
319 433
320 434
    emit_exit(e, JIT_EXIT_CHAIN);
435 +
    e->spilled = spilled;
321 436
}
322 437
438 +
/* Offset of the jump cache from the exchange area in r14. */
439 +
#define IO_JUMP_CACHE                                                          \
440 +
    ((u32)(offsetof(struct jit_state, jump_cache) -                            \
441 +
           offsetof(struct jit_state, io)))
442 +
323 443
/* Jump to the guest PC in rax (already even).  Stores it to io.pc, probes the
324 -
 * jump cache and jumps to the cached block on a hit, else exits to C. */
444 +
 * jump cache and jumps to the cached block on a hit, else exits to C.
445 +
 * Clobbers only rax and rcx, since the target expects held registers. */
325 446
static void emit_indirect_jump(struct emitter *e) {
447 +
    if (e->spilled)
448 +
        emit_reload(e);
449 +
326 450
    /* mov dword [r14], eax */
327 451
    emit_u8(e, 0x41);
328 452
    emit_u8(e, 0x89);
329 453
    emit_u8(e, modrm(0, X_RAX, HIO));
330 454
337 461
    emit_u32(e, (JIT_JUMP_CACHE_SIZE - 1) << 2);
338 462
    emit_u8(e, 0xC1);
339 463
    emit_u8(e, modrm(3, 4, X_RCX));
340 464
    emit_u8(e, 2);
341 465
342 -
    emit_mov_imm64(e, X_RDX, (u64)(uintptr_t)e->jit->jump_cache);
343 -
344 -
    /* cmp eax, [rdx + rcx] */
466 +
    /* cmp eax, [r14 + rcx + IO_JUMP_CACHE] */
467 +
    emit_u8(e, 0x41);
345 468
    emit_u8(e, 0x3B);
346 -
    emit_u8(e, modrm(0, X_RAX, 4));
347 -
    emit_u8(e, sib(0, X_RCX, X_RDX));
469 +
    emit_u8(e, modrm(2, X_RAX, 4));
470 +
    emit_u8(e, sib(0, X_RCX, HIO));
471 +
    emit_u32(e, IO_JUMP_CACHE);
348 472
349 473
    u32 jne_pos = emit_jnz_placeholder(e);
350 474
351 -
    /* jmp [rdx + rcx + offsetof(code)] */
475 +
    /* jmp [r14 + rcx + IO_JUMP_CACHE + offsetof(code)] */
476 +
    emit_u8(e, 0x41);
352 477
    emit_u8(e, 0xFF);
353 -
    emit_u8(e, modrm(1, 4, 4));
354 -
    emit_u8(e, sib(0, X_RCX, X_RDX));
355 -
    emit_u8(e, offsetof(struct jit_jump_entry, code));
478 +
    emit_u8(e, modrm(2, 4, 4));
479 +
    emit_u8(e, sib(0, X_RCX, HIO));
480 +
    emit_u32(e, IO_JUMP_CACHE + offsetof(struct jit_jump_entry, code));
356 481
357 482
    /* Miss: io.pc is set, let C dispatch. */
358 483
    patch_jcc(e, jne_pos, e->pos);
359 484
    emit_exit(e, JIT_EXIT_BRANCH);
360 485
}
680 805
    }
681 806
}
682 807
683 808
/* ---------- Memory access helpers -------------------------------------- */
684 809
810 +
/* Emit `opcode` with `reg` and the memory operand [r13 + index + disp], where
811 +
 * `opcode` is one byte, or two when `escape` prefixes it with 0x0F.  `word`
812 +
 * selects a 16-bit operand. */
813 +
static void emit_guest_memory(
814 +
    struct emitter *e,
815 +
    bool            word,
816 +
    bool            wide,
817 +
    bool            escape,
818 +
    u8              opcode,
819 +
    u8              reg,
820 +
    u8              index,
821 +
    i32             disp
822 +
) {
823 +
    if (word)
824 +
        emit_u8(e, 0x66);
825 +
    /* Always present: r13 needs REX.B, and it selects sil/dil for bytes. */
826 +
    emit_u8(e, rex(wide, reg >= 8, index >= 8, true));
827 +
    if (escape)
828 +
        emit_u8(e, 0x0F);
829 +
    emit_u8(e, opcode);
830 +
    bool short_disp = disp >= -128 && disp <= 127;
831 +
    emit_u8(e, modrm(short_disp ? 1 : 2, reg, 4));
832 +
    emit_u8(e, sib(0, index, HMEM));
833 +
    if (short_disp)
834 +
        emit_u8(e, (u8)(i8)disp);
835 +
    else
836 +
        emit_u32(e, (u32)disp);
837 +
}
838 +
839 +
/* Host register holding the base address register `guest`, loading it into
840 +
 * rax unless it is held. */
841 +
static u8 emit_base(struct emitter *e, u32 guest) {
842 +
    u8 host = host_of(e, guest);
843 +
    if (host)
844 +
        return host;
845 +
    emit_load_guest(e, X_RAX, guest);
846 +
    return X_RAX;
847 +
}
848 +
849 +
/* Translate a scalar load.  Returns false for an invalid width. */
850 +
static bool emit_scalar_load(struct emitter *e, instr_t ins) {
851 +
    static const struct {
852 +
        bool valid, wide, escape;
853 +
        u8   opcode;
854 +
    } forms[8] = {
855 +
        { true, true, true, 0xBE },   /* lb:  movsx r64, byte */
856 +
        { true, true, true, 0xBF },   /* lh:  movsx r64, word */
857 +
        { true, true, false, 0x63 },  /* lw:  movsxd r64, dword */
858 +
        { true, true, false, 0x8B },  /* ld:  mov r64, qword */
859 +
        { true, false, true, 0xB6 },  /* lbu: movzx r32, byte */
860 +
        { true, false, true, 0xB7 },  /* lhu: movzx r32, word */
861 +
        { true, false, false, 0x8B }, /* lwu: mov r32, dword */
862 +
        { false, false, false, 0 },
863 +
    };
864 +
    u32 f = ins.i.funct3;
865 +
    if (!forms[f].valid)
866 +
        return false;
867 +
    if (ins.i.rd == 0)
868 +
        return true;
869 +
    u8 base = emit_base(e, ins.i.rs1);
870 +
    u8 dst  = host_of(e, ins.i.rd);
871 +
    if (!dst)
872 +
        dst = X_RAX;
873 +
    emit_guest_memory(
874 +
        e,
875 +
        false,
876 +
        forms[f].wide,
877 +
        forms[f].escape,
878 +
        forms[f].opcode,
879 +
        dst,
880 +
        base,
881 +
        get_i_imm(ins)
882 +
    );
883 +
    if (dst == X_RAX)
884 +
        emit_store_guest(e, ins.i.rd, X_RAX);
885 +
    return true;
886 +
}
887 +
888 +
/* Translate a scalar store.  Returns false for an invalid width. */
889 +
static bool emit_scalar_store(struct emitter *e, instr_t ins) {
890 +
    u32 f = ins.s.funct3;
891 +
    if (f > 3)
892 +
        return false;
893 +
    u8 base = emit_base(e, ins.s.rs1);
894 +
    u8 src  = host_of(e, ins.s.rs2);
895 +
    if (!src) {
896 +
        emit_load_guest(e, X_RCX, ins.s.rs2);
897 +
        src = X_RCX;
898 +
    }
899 +
    emit_guest_memory(
900 +
        e,
901 +
        f == 1,
902 +
        f == 3,
903 +
        false,
904 +
        f == 0 ? 0x88 : 0x89,
905 +
        src,
906 +
        base,
907 +
        get_s_imm(ins)
908 +
    );
909 +
    return true;
910 +
}
911 +
685 912
/* add rax, r13 (compute host address from guest address). */
686 913
static void emit_add_rax_r13(struct emitter *e) {
687 914
    emit_u8(e, 0x4C);
688 915
    emit_u8(e, 0x01);
689 916
    emit_u8(e, 0xE8);
690 917
}
691 918
692 -
/* Load from [r13+rax] into rax, with sign/zero extension. */
693 -
static void emit_load_mem_i8(struct emitter *e) {
694 -
    emit_add_rax_r13(e);
695 -
    emit_rexw(e);
696 -
    emit_u8(e, 0x0F);
697 -
    emit_u8(e, 0xBE);
698 -
    emit_u8(e, modrm(0, X_RAX, X_RAX));
699 -
}
700 -
701 -
static void emit_load_mem_u8(struct emitter *e) {
702 -
    emit_add_rax_r13(e);
703 -
    emit_u8(e, 0x0F);
704 -
    emit_u8(e, 0xB6);
705 -
    emit_u8(e, modrm(0, X_RAX, X_RAX));
706 -
}
707 -
708 -
static void emit_load_mem_i16(struct emitter *e) {
709 -
    emit_add_rax_r13(e);
710 -
    emit_rexw(e);
711 -
    emit_u8(e, 0x0F);
712 -
    emit_u8(e, 0xBF);
713 -
    emit_u8(e, modrm(0, X_RAX, X_RAX));
714 -
}
715 -
716 -
static void emit_load_mem_u16(struct emitter *e) {
717 -
    emit_add_rax_r13(e);
718 -
    emit_u8(e, 0x0F);
719 -
    emit_u8(e, 0xB7);
720 -
    emit_u8(e, modrm(0, X_RAX, X_RAX));
721 -
}
722 -
723 -
static void emit_load_mem_i32(struct emitter *e) {
724 -
    emit_add_rax_r13(e);
725 -
    emit_rexw(e);
726 -
    emit_u8(e, 0x63);
727 -
    emit_u8(e, modrm(0, X_RAX, X_RAX));
728 -
}
729 -
730 -
static void emit_load_mem_u32(struct emitter *e) {
731 -
    emit_add_rax_r13(e);
732 -
    emit_u8(e, 0x8B);
733 -
    emit_u8(e, modrm(0, X_RAX, X_RAX));
734 -
}
735 -
736 -
static void emit_load_mem_u64(struct emitter *e) {
737 -
    emit_add_rax_r13(e);
738 -
    emit_rexw(e);
739 -
    emit_u8(e, 0x8B);
740 -
    emit_u8(e, modrm(0, X_RAX, X_RAX));
741 -
}
742 -
743 -
/* Store from rcx to [r13+rax]. */
744 -
static void emit_store_mem_u8(struct emitter *e) {
745 -
    emit_add_rax_r13(e);
746 -
    emit_u8(e, 0x88);
747 -
    emit_u8(e, modrm(0, X_RCX, X_RAX));
748 -
}
749 -
750 -
static void emit_store_mem_u16(struct emitter *e) {
751 -
    emit_add_rax_r13(e);
752 -
    emit_u8(e, 0x66);
753 -
    emit_u8(e, 0x89);
754 -
    emit_u8(e, modrm(0, X_RCX, X_RAX));
755 -
}
756 -
757 -
static void emit_store_mem_u32(struct emitter *e) {
758 -
    emit_add_rax_r13(e);
759 -
    emit_u8(e, 0x89);
760 -
    emit_u8(e, modrm(0, X_RCX, X_RAX));
761 -
}
762 -
763 -
static void emit_store_mem_u64(struct emitter *e) {
764 -
    emit_add_rax_r13(e);
765 -
    emit_rexw(e);
766 -
    emit_u8(e, 0x89);
767 -
    emit_u8(e, modrm(0, X_RCX, X_RAX));
768 -
}
769 -
770 919
/* Count the low 32 or 64 bits in rax and return the result in rax. */
771 920
static void emit_popcount(struct emitter *e, bool wide) {
772 921
    if (__builtin_cpu_supports("popcnt")) {
773 922
        emit_u8(e, 0xf3);
774 923
        if (wide)
1089 1238
        break;
1090 1239
    }
1091 1240
    emit_store_rd(e, ins.r.rd);
1092 1241
}
1093 1242
1243 +
/* Give back the budget of the instructions after the first `insns` of the
1244 +
 * block, on a path that leaves after them: add r15, refund.  The refund is
1245 +
 * patched in once the length of the block is known. */
1246 +
static void emit_refund(struct emitter *e, u32 insns) {
1247 +
    emit_u8(e, 0x49);
1248 +
    emit_u8(e, 0x81);
1249 +
    emit_u8(e, modrm(3, 0, HBUDGET));
1250 +
    e->refunds[e->refunds_len]    = e->pos;
1251 +
    e->consumed[e->refunds_len++] = insns;
1252 +
    emit_u32(e, 0);
1253 +
}
1254 +
1094 1255
/* ---------- Vector translation ----------------------------------------- */
1095 1256
1096 -
/* Return a failed memory operation to the interpreter before its side effects.
1097 -
 */
1098 -
static void emit_vector_fault(struct emitter *e, u32 pc, u8 condition) {
1257 +
/* Return a failed memory operation to the interpreter before its side effects
1258 +
 * when the jcc `condition` (the second opcode byte) holds. */
1259 +
static void emit_fault_if(struct emitter *e, u32 pc, u8 condition) {
1099 1260
    u32 bad = e->pos;
1100 1261
    emit_u8(e, 0x0f);
1101 1262
    emit_u8(e, condition);
1102 1263
    emit_u32(e, 0);
1103 1264
    u32 ok = emit_jmp_placeholder(e);
1104 1265
    patch_jcc(e, bad, e->pos);
1105 -
    emit_u8(e, 0x49);
1106 -
    emit_u8(e, 0x81);
1107 -
    emit_u8(e, modrm(3, 0, HBUDGET));
1108 -
    e->refunds[e->refunds_len]    = e->pos;
1109 -
    e->consumed[e->refunds_len++] = e->insns;
1110 -
    emit_u32(e, 0);
1266 +
    emit_refund(e, e->insns);
1111 1267
    emit_block_exit(e, JIT_EXIT_FAULT, pc);
1112 1268
    patch_jmp(e, ok, e->pos);
1113 1269
}
1114 1270
1115 1271
/* Load the architectural vector state into rax. */
1140 1296
    emit_u32(e, ~0xc7u);
1141 1297
    emit_rexw(e);
1142 1298
    emit_u8(e, 0x83);
1143 1299
    emit_u8(e, 0xfa);
1144 1300
    emit_u8(e, width << 3);
1145 -
    emit_vector_fault(e, pc, 0x85);
1301 +
    emit_fault_if(e, pc, 0x85);
1146 1302
    emit_rexw(e);
1147 1303
    emit_u8(e, 0x83);
1148 1304
    emit_u8(e, 0xe1);
1149 1305
    emit_u8(e, 7);
1150 1306
    emit_rexw(e);
1151 1307
    emit_u8(e, 0x83);
1152 1308
    emit_u8(e, 0xf9);
1153 1309
    emit_u8(e, lm);
1154 -
    emit_vector_fault(e, pc, 0x87);
1310 +
    emit_fault_if(e, pc, 0x87);
1155 1311
    emit_mov_imm32_sx(e, X_RDX, VECTOR_BYTES >> width);
1156 1312
    emit_rexw(e);
1157 1313
    emit_u8(e, 0xd3);
1158 1314
    emit_u8(e, 0xe2); /* shl rdx, cl */
1159 1315
    emit_rexw(e);
1160 1316
    emit_u8(e, 0x83);
1161 1317
    emit_u8(e, modrm(2, 7, X_RAX));
1162 1318
    emit_u32(e, offsetof(struct vector_state, vstart));
1163 1319
    emit_u8(e, 0);
1164 -
    emit_vector_fault(e, pc, 0x85);
1320 +
    emit_fault_if(e, pc, 0x85);
1165 1321
    emit_rexw(e);
1166 1322
    emit_u8(e, 0x8b);
1167 1323
    emit_u8(e, modrm(2, X_RCX, X_RAX));
1168 1324
    emit_u32(e, offsetof(struct vector_state, vl));
1169 1325
    emit_rexw(e);
1170 1326
    emit_u8(e, 0x39);
1171 1327
    emit_u8(e, 0xd1); /* cmp rcx, rdx */
1172 -
    emit_vector_fault(e, pc, 0x87);
1328 +
    emit_fault_if(e, pc, 0x87);
1173 1329
}
1174 1330
1175 1331
/* Form a pointer into one architectural register group. */
1176 1332
static void emit_group_pointer(struct emitter *e, enum x86reg dst, u32 reg) {
1177 1333
    emit_rexw(e);
1256 1412
    emit_u32(e, index * VECTOR_BYTES + lane * 4);
1257 1413
    emit_load_guest(e, X_RAX, base);
1258 1414
    emit_u8(e, 0x48);
1259 1415
    emit_u8(e, 0x01);
1260 1416
    emit_u8(e, 0xd0);
1261 -
    emit_vector_fault(e, pc, 0x82); /* jc */
1417 +
    emit_fault_if(e, pc, 0x82); /* jc */
1262 1418
}
1263 1419
1264 1420
/* Check a complete transfer against the hosted memory bound. */
1265 1421
static void emit_vector_bound(struct emitter *e, u32 size, u32 pc) {
1266 1422
    emit_u8(e, 0x49);
1269 1425
    emit_u8(e, offsetof(struct jit_io, memory_size));
1270 1426
    emit_u8(e, 0x48);
1271 1427
    emit_u8(e, 0x83);
1272 1428
    emit_u8(e, 0xe9);
1273 1429
    emit_u8(e, size);
1274 -
    emit_vector_fault(e, pc, 0x82);
1430 +
    emit_fault_if(e, pc, 0x82);
1275 1431
    emit_u8(e, 0x48);
1276 1432
    emit_u8(e, 0x39);
1277 1433
    emit_u8(e, 0xc8);
1278 -
    emit_vector_fault(e, pc, 0x87); /* ja */
1434 +
    emit_fault_if(e, pc, 0x87); /* ja */
1279 1435
}
1280 1436
1281 1437
/* Transfer active bytes with complete bounds checks before any guest write. */
1282 1438
static bool emit_group_memory(struct emitter *e, u32 ins, u32 pc) {
1283 1439
    u32  op = ins & 127, reg = (ins >> 7) & 31;
1310 1466
        emit_u8(e, modrm(1, X_RDX, HIO));
1311 1467
        emit_u8(e, offsetof(struct jit_io, memory_size));
1312 1468
        emit_rexw(e);
1313 1469
        emit_u8(e, 0x29);
1314 1470
        emit_u8(e, 0xca); /* sub rdx, rcx */
1315 -
        emit_vector_fault(e, pc, 0x82);
1471 +
        emit_fault_if(e, pc, 0x82);
1316 1472
        emit_rexw(e);
1317 1473
        emit_u8(e, 0x39);
1318 1474
        emit_u8(e, 0xd0); /* cmp rax, rdx */
1319 -
        emit_vector_fault(e, pc, 0x87);
1475 +
        emit_fault_if(e, pc, 0x87);
1320 1476
        emit_add_rax_r13(e);
1321 1477
        emit_rexw(e);
1322 1478
        emit_u8(e, 0x89);
1323 1479
        emit_u8(e, store ? 0xc7 : 0xc6);
1324 1480
        emit_u8(e, 0xf3);
1336 1492
        emit_u8(e, 0xf7);
1337 1493
        emit_u8(e, 0xe6); /* mul rsi */
1338 1494
        emit_rexw(e);
1339 1495
        emit_u8(e, 0x85);
1340 1496
        emit_u8(e, 0xd2);
1341 -
        emit_vector_fault(e, pc, 0x85);
1497 +
        emit_fault_if(e, pc, 0x85);
1342 1498
        emit_rexw(e);
1343 1499
        emit_u8(e, 0x01);
1344 1500
        emit_u8(e, 0xf8); /* add rax, rdi */
1345 -
        emit_vector_fault(e, pc, 0x82);
1501 +
        emit_fault_if(e, pc, 0x82);
1346 1502
        emit_vector_bound(e, 1, pc);
1347 1503
        emit_rexw(e);
1348 1504
        emit_u8(e, 0x89);
1349 1505
        emit_u8(e, 0xf2); /* mov rdx, rsi */
1350 1506
        emit_vector_base(e);
1741 1897
    emit_u8(e, 1);
1742 1898
    emit_block_exit(e, JIT_EXIT_FAULT, pc);
1743 1899
    patch_jmp(e, ok, e->pos);
1744 1900
}
1745 1901
1902 +
/* Set the flags as cmp rs1, rs2 would, using held registers in place. */
1903 +
static void emit_compare(struct emitter *e, u32 rs1, u32 rs2) {
1904 +
    u8 a = emit_base(e, rs1);
1905 +
    if (rs2 == 0) {
1906 +
        /* test a, a */
1907 +
        emit_u8(e, rex(true, a >= 8, false, a >= 8));
1908 +
        emit_u8(e, 0x85);
1909 +
        emit_u8(e, modrm(3, a, a));
1910 +
        return;
1911 +
    }
1912 +
    u8 b = host_of(e, rs2);
1913 +
    if (b) {
1914 +
        /* cmp a, b */
1915 +
        emit_u8(e, rex(true, b >= 8, false, a >= 8));
1916 +
        emit_u8(e, 0x39);
1917 +
        emit_u8(e, modrm(3, b, a));
1918 +
        return;
1919 +
    }
1920 +
    /* cmp a, [r12 + rs2 * 8] */
1921 +
    emit_r12_disp(e, 0x3B, a, rs2 * 8);
1922 +
}
1923 +
1924 +
/* Whether the block already holds the instruction at `pc`. */
1925 +
static bool translated(struct emitter *e, u32 pc) {
1926 +
    for (u32 i = 0; i < e->runs; i++) {
1927 +
        u32 end = i + 1 == e->runs ? e->next_pc : e->run_end[i];
1928 +
        if (pc >= e->run_start[i] && pc < end)
1929 +
            return true;
1930 +
    }
1931 +
    return false;
1932 +
}
1933 +
1934 +
/* Continue translating the block at `target` after a jump whose next
1935 +
 * instruction is at `next`. */
1936 +
static void follow(struct emitter *e, u32 next, u32 target) {
1937 +
    e->run_end[e->runs - 1] = next;
1938 +
    e->run_start[e->runs]   = target;
1939 +
    e->runs++;
1940 +
    e->next_pc = target;
1941 +
}
1942 +
1943 +
/* Jump to the constant `target`, linking `next` in `rd`.  Continues the
1944 +
 * block at the target unless that would loop, and returns whether the block
1945 +
 * ends. */
1946 +
static bool emit_direct_jump(struct emitter *e, u32 rd, u32 next, u32 target) {
1947 +
    if (rd != 0) {
1948 +
        emit_mov_imm64(e, HTMP1, (u64)next);
1949 +
        emit_store_guest(e, rd, HTMP1);
1950 +
    }
1951 +
    if (e->runs > JIT_MAX_FOLLOWS || translated(e, target)) {
1952 +
        emit_chain_exit(e, target);
1953 +
        return true;
1954 +
    }
1955 +
    /* A return to `next` can then be checked in place. */
1956 +
    if (rd == RA && next <= INT32_MAX) {
1957 +
        e->ra_known = true;
1958 +
        e->ra_value = next;
1959 +
    }
1960 +
    follow(e, next, target);
1961 +
    return false;
1962 +
}
1963 +
1964 +
/* Return to ra: leave the program when it is zero, else jump to it. */
1965 +
static void emit_ret(struct emitter *e) {
1966 +
    emit_load_guest(e, HTMP1, RA);
1967 +
1968 +
    /* test rax, rax */
1969 +
    emit_rexw(e);
1970 +
    emit_u8(e, 0x85);
1971 +
    emit_u8(e, modrm(3, X_RAX, X_RAX));
1972 +
1973 +
    u32 jnz_pos = emit_jnz_placeholder(e);
1974 +
1975 +
    /* RA == 0: program exit. */
1976 +
    emit_block_exit(e, JIT_EXIT_RET, 0);
1977 +
1978 +
    /* RA != 0: compute target = RA & ~1. */
1979 +
    patch_jcc(e, jnz_pos, e->pos);
1980 +
    emit_rexw(e);
1981 +
    emit_u8(e, 0x83);
1982 +
    emit_u8(e, modrm(3, 4, X_RAX));
1983 +
    emit_u8(e, 0xFE); /* and rax, ~1 */
1984 +
1985 +
    emit_indirect_jump(e);
1986 +
}
1987 +
1988 +
/* Emit the side exits recorded for taken branches and checked returns. */
1989 +
static void emit_side_exits(struct emitter *e) {
1990 +
    /* Branches are never translated spilled. */
1991 +
    e->spilled = false;
1992 +
    for (u32 i = 0; i < e->sides_len; i++) {
1993 +
        patch_jcc(e, e->sides[i].jcc, e->pos);
1994 +
        emit_refund(e, e->sides[i].insns);
1995 +
        if (e->sides[i].ret)
1996 +
            emit_ret(e);
1997 +
        else
1998 +
            emit_chain_exit(e, e->sides[i].target);
1999 +
    }
2000 +
}
2001 +
2002 +
/* Leave the block through a side exit when the jcc `cc` (its second opcode
2003 +
 * byte) holds: to `target`, or by returning to ra when `ret` is set. */
2004 +
static void emit_side_exit(struct emitter *e, u8 cc, u32 target, bool ret) {
2005 +
    e->sides[e->sides_len].jcc    = e->pos;
2006 +
    e->sides[e->sides_len].target = target;
2007 +
    e->sides[e->sides_len].insns  = e->insns;
2008 +
    e->sides[e->sides_len].ret    = ret;
2009 +
    e->sides_len++;
2010 +
    emit_u8(e, 0x0F);
2011 +
    emit_u8(e, cc);
2012 +
    emit_u32(e, 0);
2013 +
}
2014 +
2015 +
/* Whether the instruction at `pc` is ebreak. */
2016 +
static bool traps(struct emitter *e, u32 pc) {
2017 +
    if (pc < e->prog_base || pc >= e->prog_end)
2018 +
        return false;
2019 +
    u32 raw;
2020 +
    memcpy(&raw, &e->mem[pc], sizeof(raw));
2021 +
    return raw == 0x00100073;
2022 +
}
2023 +
2024 +
/* ---------- Register-direct integer operations ------------------------- */
2025 +
2026 +
/* Emit `opcode` (an "op r, r/m" form) with `dst` and guest register `src` as
2027 +
 * the r/m operand, held or in the register file.  `src` is not x0. */
2028 +
static void emit_op_guest(
2029 +
    struct emitter *e, bool wide, bool escape, u8 opcode, u8 dst, u32 src
2030 +
) {
2031 +
    u8 host = host_of(e, src);
2032 +
    /* The register file is addressed through r12, which needs REX.B. */
2033 +
    emit_u8(e, rex(wide, dst >= 8, false, host ? host >= 8 : true));
2034 +
    if (escape)
2035 +
        emit_u8(e, 0x0F);
2036 +
    emit_u8(e, opcode);
2037 +
    if (host)
2038 +
        emit_u8(e, modrm(3, dst, host));
2039 +
    else
2040 +
        emit_r12_address(e, dst, src * 8);
2041 +
}
2042 +
2043 +
/* Emit the group-1 operation `ext` (0 add, 1 or, 4 and, 5 sub, 6 xor, 7 cmp)
2044 +
 * of `dst` with a sign-extended immediate. */
2045 +
static void emit_op_imm(struct emitter *e, bool wide, u8 ext, u8 dst, i32 imm) {
2046 +
    emit_u8(e, rex(wide, false, false, dst >= 8));
2047 +
    if (imm >= -128 && imm <= 127) {
2048 +
        emit_u8(e, 0x83);
2049 +
        emit_u8(e, modrm(3, ext, dst));
2050 +
        emit_u8(e, (u8)(i8)imm);
2051 +
    } else {
2052 +
        emit_u8(e, 0x81);
2053 +
        emit_u8(e, modrm(3, ext, dst));
2054 +
        emit_u32(e, (u32)imm);
2055 +
    }
2056 +
}
2057 +
2058 +
/* Emit the shift `ext` (4 shl, 5 shr, 7 sar) of `dst` by a constant. */
2059 +
static void emit_shift_imm(
2060 +
    struct emitter *e, bool wide, u8 ext, u8 dst, u8 amount
2061 +
) {
2062 +
    emit_u8(e, rex(wide, false, false, dst >= 8));
2063 +
    emit_u8(e, 0xC1);
2064 +
    emit_u8(e, modrm(3, ext, dst));
2065 +
    emit_u8(e, amount);
2066 +
}
2067 +
2068 +
/* lea dst, [base + disp] (64-bit, or 32-bit when `wide` is clear). */
2069 +
static void emit_lea(struct emitter *e, bool wide, u8 dst, u8 base, i32 disp) {
2070 +
    if (wide || dst >= 8 || base >= 8)
2071 +
        emit_u8(e, rex(wide, dst >= 8, false, base >= 8));
2072 +
    emit_u8(e, 0x8D);
2073 +
    bool short_disp = disp >= -128 && disp <= 127;
2074 +
    emit_u8(e, modrm(short_disp ? 1 : 2, dst, base));
2075 +
    if ((base & 7) == 4)
2076 +
        emit_u8(e, sib(0, 4, base));
2077 +
    if (short_disp)
2078 +
        emit_u8(e, (u8)(i8)disp);
2079 +
    else
2080 +
        emit_u32(e, (u32)disp);
2081 +
}
2082 +
2083 +
/* movsxd dst, src32 */
2084 +
static void emit_movsxd(struct emitter *e, u8 dst, u8 src) {
2085 +
    emit_u8(e, rex(true, dst >= 8, false, src >= 8));
2086 +
    emit_u8(e, 0x63);
2087 +
    emit_u8(e, modrm(3, dst, src));
2088 +
}
2089 +
2090 +
/* The shift extension of an immediate shift: funct3 1 shifts left, 5 right,
2091 +
 * arithmetically when bit 10 of the immediate is set. */
2092 +
static u8 shift_ext(u32 funct3, i32 imm) {
2093 +
    return funct3 == 1 ? 4 : (imm & 0x400) ? 7 : 5;
2094 +
}
2095 +
2096 +
/* Host register to compute `rd` in: the one holding it, or rax. */
2097 +
static u8 dest_of(struct emitter *e, u32 rd) {
2098 +
    u8 d = host_of(e, rd);
2099 +
    return d ? d : X_RAX;
2100 +
}
2101 +
2102 +
/* Write a result computed in `d`, chosen by dest_of(), to `rd`. */
2103 +
static void emit_finish(struct emitter *e, u32 rd, u8 d) {
2104 +
    if (d == X_RAX)
2105 +
        emit_store_guest(e, rd, X_RAX);
2106 +
}
2107 +
2108 +
/* Translate lui, auipc and the integer operations with an immediate. */
2109 +
static void emit_immediate(struct emitter *e, instr_t ins, u32 pc) {
2110 +
    u32 rd = ins.i.rd, rs1 = ins.i.rs1, f3 = ins.i.funct3;
2111 +
    i32 imm = get_i_imm(ins);
2112 +
    if (rd == 0)
2113 +
        return;
2114 +
    u8 d   = dest_of(e, rd);
2115 +
    u8 src = host_of(e, rs1);
2116 +
2117 +
    switch (ins.i.opcode) {
2118 +
    case OP_LUI:
2119 +
        emit_mov_imm32_sx(e, d, (i32)(ins.u.imm_31_12 << 12));
2120 +
        break;
2121 +
2122 +
    case OP_AUIPC: {
2123 +
        i64 value = (i64)pc + (i64)(i32)(ins.u.imm_31_12 << 12);
2124 +
        if (value >= INT32_MIN && value <= INT32_MAX)
2125 +
            emit_mov_imm32_sx(e, d, (i32)value);
2126 +
        else
2127 +
            emit_mov_imm64(e, d, (u64)value);
2128 +
        break;
2129 +
    }
2130 +
2131 +
    case OP_IMM:
2132 +
        if (f3 == 0 && rs1 == 0) {
2133 +
            emit_mov_imm32_sx(e, d, imm); /* li */
2134 +
            break;
2135 +
        }
2136 +
        if (f3 == 0 && src) {
2137 +
            if (imm)
2138 +
                emit_lea(e, true, d, src, imm);
2139 +
            else
2140 +
                emit_mov_rr(e, d, src); /* mv */
2141 +
            break;
2142 +
        }
2143 +
        if (f3 == 2 || f3 == 3) {
2144 +
            /* slti, sltiu: cmp a, imm; setl/setb al; movzx d, al */
2145 +
            u8 a = emit_base(e, rs1);
2146 +
            emit_op_imm(e, true, 7, a, imm);
2147 +
            emit_u8(e, 0x0F);
2148 +
            emit_u8(e, f3 == 2 ? 0x9C : 0x92);
2149 +
            emit_u8(e, modrm(3, 0, X_RAX));
2150 +
            if (d >= 8)
2151 +
                emit_u8(e, rex(false, true, false, false));
2152 +
            emit_u8(e, 0x0F);
2153 +
            emit_u8(e, 0xB6);
2154 +
            emit_u8(e, modrm(3, d, X_RAX));
2155 +
            break;
2156 +
        }
2157 +
        emit_load_guest(e, d, rs1);
2158 +
        switch (f3) {
2159 +
        case 0x0: /* addi */
2160 +
            if (imm)
2161 +
                emit_op_imm(e, true, 0, d, imm);
2162 +
            break;
2163 +
        case 0x1: /* slli */
2164 +
        case 0x5: /* srli, srai */
2165 +
            emit_shift_imm(e, true, shift_ext(f3, imm), d, (u8)(imm & 0x3F));
2166 +
            break;
2167 +
        case 0x4: /* xori */
2168 +
            emit_op_imm(e, true, 6, d, imm);
2169 +
            break;
2170 +
        case 0x6: /* ori */
2171 +
            emit_op_imm(e, true, 1, d, imm);
2172 +
            break;
2173 +
        case 0x7: /* andi */
2174 +
            emit_op_imm(e, true, 4, d, imm);
2175 +
            break;
2176 +
        }
2177 +
        break;
2178 +
2179 +
    case OP_IMM_32:
2180 +
        /* Compute in eax and sign-extend into d. */
2181 +
        if (f3 == 0 && src) {
2182 +
            emit_lea(e, false, X_RAX, src, imm); /* addiw */
2183 +
        } else {
2184 +
            emit_load_guest(e, X_RAX, rs1);
2185 +
            if (f3 == 0)
2186 +
                emit_op_imm(e, false, 0, X_RAX, imm); /* addiw */
2187 +
            else if (f3 == 1 || f3 == 5)
2188 +
                emit_shift_imm(
2189 +
                    e, false, shift_ext(f3, imm), X_RAX, (u8)(imm & 0x1F)
2190 +
                ); /* slliw, srliw, sraiw */
2191 +
        }
2192 +
        emit_movsxd(e, d, X_RAX);
2193 +
        break;
2194 +
    }
2195 +
    emit_finish(e, rd, d);
2196 +
}
2197 +
2198 +
/* Translate add, sub, xor, or, and, mul and the word forms of add, sub and
2199 +
 * mul of two registers other than x0, on held registers in place.  Returns
2200 +
 * false for the other register operations. */
2201 +
static bool emit_register_op(struct emitter *e, instr_t ins) {
2202 +
    u32  rd = ins.r.rd, rs1 = ins.r.rs1, rs2 = ins.r.rs2;
2203 +
    bool wide   = ins.r.opcode == OP_OP;
2204 +
    bool escape = false;
2205 +
    u8   op;
2206 +
    if (ins.r.funct7 == FUNCT7_NORMAL && ins.r.funct3 == 0)
2207 +
        op = 0x03; /* add */
2208 +
    else if (ins.r.funct7 == FUNCT7_SUB && ins.r.funct3 == 0)
2209 +
        op = 0x2B; /* sub */
2210 +
    else if (ins.r.funct7 == FUNCT7_MUL && ins.r.funct3 == 0)
2211 +
        op = 0xAF, escape = true; /* imul */
2212 +
    else if (wide && ins.r.funct7 == FUNCT7_NORMAL && ins.r.funct3 == 4)
2213 +
        op = 0x33; /* xor */
2214 +
    else if (wide && ins.r.funct7 == FUNCT7_NORMAL && ins.r.funct3 == 6)
2215 +
        op = 0x0B; /* or */
2216 +
    else if (wide && ins.r.funct7 == FUNCT7_NORMAL && ins.r.funct3 == 7)
2217 +
        op = 0x23; /* and */
2218 +
    else
2219 +
        return false;
2220 +
    if (rd == 0 || rs1 == 0 || rs2 == 0)
2221 +
        return false;
2222 +
2223 +
    /* Compute in d, unless it holds rs2, which the first operand would
2224 +
     * overwrite, or the word result needs sign extension from eax. */
2225 +
    u8 d = dest_of(e, rd);
2226 +
    u8 t = wide && (rd == rs1 || rd != rs2) ? d : X_RAX;
2227 +
    if (t == X_RAX || rd != rs1)
2228 +
        emit_load_guest(e, t, rs1);
2229 +
    emit_op_guest(e, wide, escape, op, t, rs2);
2230 +
    if (wide)
2231 +
        emit_mov_rr(e, d, t);
2232 +
    else
2233 +
        emit_movsxd(e, d, X_RAX);
2234 +
    emit_finish(e, rd, d);
2235 +
    return true;
2236 +
}
2237 +
2238 +
/* Offset of the reservation in the exchange area. */
2239 +
#define IO_RESERVATION ((u8)offsetof(struct jit_io, reservation))
2240 +
2241 +
/* Translate LR and SC of a word or doubleword for a single hart.  Only LR,
2242 +
 * SC and AMOs touch the reservation, as in the interpreter.  A misaligned
2243 +
 * address leaves for the interpreter, which reports it.  Returns false for
2244 +
 * the operations left to the interpreter. */
2245 +
static bool emit_reserved(struct emitter *e, instr_t ins, u32 pc) {
2246 +
    u32 op = ins.raw >> 27;
2247 +
    if ((ins.r.funct3 != 2 && ins.r.funct3 != 3) || (op != 2 && op != 3))
2248 +
        return false;
2249 +
    u32  size = ins.r.funct3 == 2 ? 4 : 8;
2250 +
    bool lr   = op == 2;
2251 +
2252 +
    emit_load_guest(e, X_RAX, ins.r.rs1);
2253 +
    emit_u8(e, 0xA8);
2254 +
    emit_u8(e, (u8)(size - 1)); /* test al, size - 1 */
2255 +
    emit_fault_if(e, pc, 0x85);
2256 +
    emit_u8(e, 0x89);
2257 +
    emit_u8(e, 0xC0); /* mov eax, eax */
2258 +
2259 +
    /* mov rcx, size << 32; or rcx, rax -- the reservation value. */
2260 +
    emit_mov_imm64(e, X_RCX, jit_reservation(0, size));
2261 +
    emit_u8(e, 0x48);
2262 +
    emit_u8(e, 0x09);
2263 +
    emit_u8(e, modrm(3, X_RAX, X_RCX));
2264 +
2265 +
    if (lr) {
2266 +
        /* mov [r14 + IO_RESERVATION], rcx */
2267 +
        emit_u8(e, 0x49);
2268 +
        emit_u8(e, 0x89);
2269 +
        emit_u8(e, modrm(1, X_RCX, HIO));
2270 +
        emit_u8(e, IO_RESERVATION);
2271 +
        if (ins.r.rd != 0) {
2272 +
            u8 d = host_of(e, ins.r.rd);
2273 +
            if (!d)
2274 +
                d = X_RAX;
2275 +
            /* movsxd d, [r13 + rax] or mov d, [r13 + rax] */
2276 +
            emit_guest_memory(
2277 +
                e, false, true, false, size == 4 ? 0x63 : 0x8B, d, X_RAX, 0
2278 +
            );
2279 +
            if (d == X_RAX)
2280 +
                emit_store_guest(e, ins.r.rd, X_RAX);
2281 +
        }
2282 +
        return true;
2283 +
    }
2284 +
    /* cmp [r14 + IO_RESERVATION], rcx */
2285 +
    emit_u8(e, 0x49);
2286 +
    emit_u8(e, 0x39);
2287 +
    emit_u8(e, modrm(1, X_RCX, HIO));
2288 +
    emit_u8(e, IO_RESERVATION);
2289 +
    /* mov qword [r14 + IO_RESERVATION], 0 -- keeps the flags. */
2290 +
    emit_u8(e, 0x49);
2291 +
    emit_u8(e, 0xC7);
2292 +
    emit_u8(e, modrm(1, 0, HIO));
2293 +
    emit_u8(e, IO_RESERVATION);
2294 +
    emit_u32(e, 0);
2295 +
    u32 fail = emit_jne_placeholder(e);
2296 +
2297 +
    emit_load_guest(e, X_RCX, ins.r.rs2);
2298 +
    emit_guest_memory(e, false, size == 8, false, 0x89, X_RCX, X_RAX, 0);
2299 +
    emit_mov_imm32_sx(e, X_RAX, 0);
2300 +
    u32 done = emit_jmp_placeholder(e);
2301 +
2302 +
    patch_jcc(e, fail, e->pos);
2303 +
    emit_mov_imm32_sx(e, X_RAX, 1);
2304 +
    patch_jmp(e, done, e->pos);
2305 +
    emit_store_guest(e, ins.r.rd, X_RAX);
2306 +
    return true;
2307 +
}
2308 +
1746 2309
/* ---------- Translate one RV64I instruction ----------------------------- */
1747 2310
1748 2311
/*
1749 2312
 * Returns: true if the instruction ends the basic block (branch/jump/ecall),
1750 2313
 *          false if execution should continue to the next instruction.
1751 2314
 */
1752 -
static bool translate_insn(struct emitter *e, instr_t ins, u32 pc) {
2315 +
static bool translate_body(struct emitter *e, instr_t ins, u32 pc) {
1753 2316
    u32 opcode  = ins.r.opcode;
1754 2317
    u32 pc_next = pc + INSTR_SIZE;
1755 2318
1756 2319
    enum bitmanip_op bm = bitmanip_decode(ins.raw);
1757 2320
    if (bm != BM_NONE) {
1773 2336
        return false;
1774 2337
1775 2338
    switch (opcode) {
1776 2339
1777 2340
    case OP_LUI:
1778 -
        if (ins.u.rd != 0) {
1779 -
            emit_mov_imm32_sx(e, HTMP1, (i32)(ins.u.imm_31_12 << 12));
1780 -
            emit_store_rd(e, ins.u.rd);
1781 -
        }
1782 -
        return false;
1783 -
1784 2341
    case OP_AUIPC:
1785 -
        if (ins.u.rd != 0) {
1786 -
            i64 result = (i64)pc + (i64)(i32)(ins.u.imm_31_12 << 12);
1787 -
            emit_mov_imm64(e, HTMP1, (u64)result);
1788 -
            emit_store_rd(e, ins.u.rd);
1789 -
        }
2342 +
    case OP_IMM:
2343 +
    case OP_IMM_32:
2344 +
        emit_immediate(e, ins, pc);
1790 2345
        return false;
1791 2346
1792 2347
    case OP_JAL: {
1793 -
        u32 target = pc + (u32)get_j_imm(ins);
1794 -
        if (ins.j.rd != 0) {
1795 -
            emit_mov_imm64(e, HTMP1, (u64)pc_next);
1796 -
            emit_store_guest(e, ins.j.rd, HTMP1);
1797 -
        }
1798 -
        emit_chain_exit(e, target);
1799 -
        return true;
2348 +
        return emit_direct_jump(e, ins.j.rd, pc_next, pc + get_j_imm(ins));
1800 2349
    }
1801 2350
1802 2351
    case OP_JALR: {
1803 2352
        i32 imm = get_i_imm(ins);
1804 2353
1805 2354
        /* Special-case RET: jalr x0, ra, 0. */
1806 2355
        if (ins.i.rd == 0 && ins.i.rs1 == RA && imm == 0) {
1807 -
            emit_load_guest(e, HTMP1, RA);
1808 -
1809 -
            /* test rax, rax */
1810 -
            emit_rexw(e);
1811 -
            emit_u8(e, 0x85);
1812 -
            emit_u8(e, modrm(3, X_RAX, X_RAX));
1813 -
1814 -
            u32 jnz_pos = emit_jnz_placeholder(e);
1815 -
1816 -
            /* RA == 0: program exit. */
1817 -
            emit_block_exit(e, JIT_EXIT_RET, 0);
1818 -
1819 -
            /* RA != 0: compute target = RA & ~1. */
1820 -
            patch_jcc(e, jnz_pos, e->pos);
1821 -
            emit_rexw(e);
1822 -
            emit_u8(e, 0x83);
1823 -
            emit_u8(e, modrm(3, 4, X_RAX));
1824 -
            emit_u8(e, 0xFE); /* and rax, ~1 */
1825 -
1826 -
            emit_indirect_jump(e);
2356 +
            /* Returning to the call this block followed: check that ra
2357 +
             * still holds its return address and continue there. */
2358 +
            if (e->ra_known && e->sides_len < JIT_MAX_SIDE_EXITS &&
2359 +
                e->runs <= JIT_MAX_FOLLOWS && !translated(e, e->ra_value)) {
2360 +
                emit_load_guest(e, HTMP1, RA);
2361 +
                emit_rexw(e);
2362 +
                emit_u8(e, 0x3D);
2363 +
                emit_u32(e, e->ra_value); /* cmp rax, ra_value */
2364 +
                emit_side_exit(e, 0x85, 0, true);
2365 +
                follow(e, pc_next, e->ra_value);
2366 +
                return false;
2367 +
            }
2368 +
            emit_ret(e);
1827 2369
            return true;
1828 2370
        }
1829 2371
2372 +
        /* A jump through the register the previous instruction set with
2373 +
         * auipc has a constant target: jump there directly. */
2374 +
        if (e->auipc_reg && ins.i.rs1 == e->auipc_reg) {
2375 +
            u32 target = (u32)(e->auipc_value + (u64)(i64)imm) & ~1u;
2376 +
            return emit_direct_jump(e, ins.i.rd, pc_next, target);
2377 +
        }
2378 +
1830 2379
        /* General JALR: target = (rs1 + imm) & ~1. */
1831 2380
        emit_load_rs1(e, ins.i.rs1);
1832 2381
        emit_add_rax_imm32(e, imm);
1833 2382
        emit_rexw(e);
1834 2383
        emit_u8(e, 0x83);
1845 2394
    }
1846 2395
1847 2396
    case OP_BRANCH: {
1848 2397
        u32 target = pc + (u32)get_b_imm(ins);
1849 2398
1850 -
        emit_load_rs1(e, ins.b.rs1);
1851 -
        emit_load_rs2(e, ins.b.rs2);
1852 -
        emit_cmp_rax_rcx(e);
2399 +
        emit_compare(e, ins.b.rs1, ins.b.rs2);
1853 2400
1854 2401
        /* jCC to .taken */
1855 2402
        u8 cc;
1856 2403
        switch (ins.b.funct3) {
1857 2404
        case 0x0:
1875 2422
        default:
1876 2423
            cc = 0x84;
1877 2424
            break;
1878 2425
        }
1879 2426
2427 +
        /* Continue the block on the fall-through path; the taken path
2428 +
         * leaves through a side exit after the block's main path. */
2429 +
        if (e->sides_len < JIT_MAX_SIDE_EXITS) {
2430 +
            /* A branch around a trap, as in a bounds check, is taken: continue
2431 +
             * at the target and leave to the trap through the side exit,
2432 +
             * with the inverse condition. */
2433 +
            if (traps(e, pc_next) && e->runs <= JIT_MAX_FOLLOWS &&
2434 +
                !translated(e, target)) {
2435 +
                emit_side_exit(e, cc ^ 1, pc_next, false);
2436 +
                follow(e, pc_next, target);
2437 +
                return false;
2438 +
            }
2439 +
            emit_side_exit(e, cc, target, false);
2440 +
            return false;
2441 +
        }
2442 +
1880 2443
        u32 jcc_pos = e->pos;
1881 2444
        emit_u8(e, 0x0F);
1882 2445
        emit_u8(e, cc);
1883 2446
        emit_u32(e, 0);
1884 2447
1889 2452
        patch_jcc(e, jcc_pos, e->pos);
1890 2453
        emit_chain_exit(e, target);
1891 2454
        return true;
1892 2455
    }
1893 2456
1894 -
    case OP_LOAD: {
1895 -
        i32 imm = get_i_imm(ins);
1896 -
        if (ins.i.rd == 0)
2457 +
    case OP_LOAD:
2458 +
        if (emit_scalar_load(e, ins))
1897 2459
            return false;
2460 +
        emit_block_exit(e, JIT_EXIT_FAULT, pc);
2461 +
        return true;
1898 2462
1899 -
        emit_load_rs1(e, ins.i.rs1);
1900 -
        emit_add_rax_imm32(e, imm);
1901 -
1902 -
        switch (ins.i.funct3) {
1903 -
        case 0x0:
1904 -
            emit_load_mem_i8(e);
1905 -
            break; /* lb */
1906 -
        case 0x1:
1907 -
            emit_load_mem_i16(e);
1908 -
            break; /* lh */
1909 -
        case 0x2:
1910 -
            emit_load_mem_i32(e);
1911 -
            break; /* lw */
1912 -
        case 0x3:
1913 -
            emit_load_mem_u64(e);
1914 -
            break; /* ld */
1915 -
        case 0x4:
1916 -
            emit_load_mem_u8(e);
1917 -
            break; /* lbu */
1918 -
        case 0x5:
1919 -
            emit_load_mem_u16(e);
1920 -
            break; /* lhu */
1921 -
        case 0x6:
1922 -
            emit_load_mem_u32(e);
1923 -
            break; /* lwu */
1924 -
        default:
1925 -
            emit_block_exit(e, JIT_EXIT_FAULT, pc);
1926 -
            return true;
1927 -
        }
1928 -
        emit_store_rd(e, ins.i.rd);
1929 -
        return false;
1930 -
    }
1931 -
1932 -
    case OP_STORE: {
1933 -
        i32 imm = get_s_imm(ins);
1934 -
        emit_load_guest(e, HTMP1, ins.s.rs1);
1935 -
        emit_add_rax_imm32(e, imm);
1936 -
        emit_load_guest(e, HTMP2, ins.s.rs2);
1937 -
1938 -
        switch (ins.s.funct3) {
1939 -
        case 0x0:
1940 -
            emit_store_mem_u8(e);
1941 -
            break; /* sb */
1942 -
        case 0x1:
1943 -
            emit_store_mem_u16(e);
1944 -
            break; /* sh */
1945 -
        case 0x2:
1946 -
            emit_store_mem_u32(e);
1947 -
            break; /* sw */
1948 -
        case 0x3:
1949 -
            emit_store_mem_u64(e);
1950 -
            break; /* sd */
1951 -
        default:
1952 -
            emit_block_exit(e, JIT_EXIT_FAULT, pc);
1953 -
            return true;
1954 -
        }
1955 -
        return false;
1956 -
    }
1957 -
1958 -
    case OP_IMM: {
1959 -
        i32 imm = get_i_imm(ins);
1960 -
        if (ins.i.rd == 0)
1961 -
            return false;
1962 -
1963 -
        emit_load_rs1(e, ins.i.rs1);
1964 -
1965 -
        switch (ins.i.funct3) {
1966 -
        case 0x0: /* addi */
1967 -
            emit_add_rax_imm32(e, imm);
1968 -
            break;
1969 -
        case 0x1: /* slli */
1970 -
            emit_rexw(e);
1971 -
            emit_u8(e, 0xC1);
1972 -
            emit_u8(e, modrm(3, 4, X_RAX));
1973 -
            emit_u8(e, (u8)(imm & 0x3F));
1974 -
            break;
1975 -
        case 0x2: /* slti */
1976 -
            emit_mov_imm32_sx(e, HTMP2, imm);
1977 -
            emit_cmp_rax_rcx(e);
1978 -
            emit_setcc_rax(e, 0x9C); /* setl */
1979 -
            break;
1980 -
        case 0x3: /* sltiu */
1981 -
            emit_mov_imm32_sx(e, HTMP2, imm);
1982 -
            emit_cmp_rax_rcx(e);
1983 -
            emit_setcc_rax(e, 0x92); /* setb */
1984 -
            break;
1985 -
        case 0x4: /* xori */
1986 -
            emit_mov_imm32_sx(e, HTMP2, imm);
1987 -
            emit_alu_rax_rcx(e, 0x31); /* xor */
1988 -
            break;
1989 -
        case 0x5: /* srli/srai */
1990 -
            emit_rexw(e);
1991 -
            emit_u8(e, 0xC1);
1992 -
            emit_u8(e, modrm(3, (imm & 0x400) ? 7 : 5, X_RAX));
1993 -
            emit_u8(e, (u8)(imm & 0x3F));
1994 -
            break;
1995 -
        case 0x6: /* ori */
1996 -
            emit_mov_imm32_sx(e, HTMP2, imm);
1997 -
            emit_alu_rax_rcx(e, 0x09); /* or */
1998 -
            break;
1999 -
        case 0x7: /* andi */
2000 -
            emit_mov_imm32_sx(e, HTMP2, imm);
2001 -
            emit_alu_rax_rcx(e, 0x21); /* and */
2002 -
            break;
2003 -
        }
2004 -
        emit_store_rd(e, ins.i.rd);
2005 -
        return false;
2006 -
    }
2007 -
2008 -
    case OP_IMM_32: {
2009 -
        i32 imm = get_i_imm(ins);
2010 -
        if (ins.i.rd == 0)
2463 +
    case OP_STORE:
2464 +
        if (emit_scalar_store(e, ins))
2011 2465
            return false;
2012 -
2013 -
        emit_load_rs1(e, ins.i.rs1);
2014 -
2015 -
        switch (ins.i.funct3) {
2016 -
        case 0x0: /* addiw */
2017 -
            emit_add_rax_imm32(e, imm);
2018 -
            break;
2019 -
        case 0x1: /* slliw */
2020 -
            emit_u8(e, 0xC1);
2021 -
            emit_u8(e, modrm(3, 4, X_RAX));
2022 -
            emit_u8(e, (u8)(imm & 0x1F));
2023 -
            break;
2024 -
        case 0x5: /* srliw/sraiw */
2025 -
            emit_u8(e, 0xC1);
2026 -
            emit_u8(e, modrm(3, (imm & 0x400) ? 7 : 5, X_RAX));
2027 -
            emit_u8(e, (u8)(imm & 0x1F));
2028 -
            break;
2029 -
        }
2030 -
        emit_movsxd_rax_eax(e);
2031 -
        emit_store_rd(e, ins.i.rd);
2032 -
        return false;
2033 -
    }
2466 +
        emit_block_exit(e, JIT_EXIT_FAULT, pc);
2467 +
        return true;
2034 2468
2035 2469
    case OP_OP: {
2036 -
        if (ins.r.rd == 0)
2470 +
        if (emit_register_op(e, ins) || ins.r.rd == 0)
2037 2471
            return false;
2038 2472
        emit_load_rs1(e, ins.r.rs1);
2039 2473
        emit_load_rs2(e, ins.r.rs2);
2040 2474
2041 2475
        switch (ins.r.funct7) {
2112 2546
        emit_store_rd(e, ins.r.rd);
2113 2547
        return false;
2114 2548
    }
2115 2549
2116 2550
    case OP_OP_32: {
2117 -
        if (ins.r.rd == 0)
2551 +
        if (emit_register_op(e, ins) || ins.r.rd == 0)
2118 2552
            return false;
2119 2553
        emit_load_rs1(e, ins.r.rs1);
2120 2554
        emit_load_rs2(e, ins.r.rs2);
2121 2555
2122 2556
        switch (ins.r.funct7) {
2194 2628
2195 2629
    case OP_FENCE:
2196 2630
        return false;
2197 2631
2198 2632
    case OP_AMO:
2199 -
        /* Atomics are rare; the interpreter runs them. */
2633 +
        if (emit_reserved(e, ins, pc))
2634 +
            return false;
2635 +
        /* Other atomics are rare; the interpreter runs them. */
2200 2636
        emit_block_exit(e, JIT_EXIT_FAULT, pc);
2201 2637
        return true;
2202 2638
2203 2639
    default:
2204 2640
        emit_block_exit(e, JIT_EXIT_FAULT, pc);
2205 2641
        return true;
2206 2642
    }
2207 2643
}
2208 2644
2645 +
/* Whether an instruction is translated by an emitter that addresses the
2646 +
 * register file directly or clobbers host registers other than rax, rcx and
2647 +
 * those in `scratch_mask`. */
2648 +
static bool needs_spill(instr_t ins) {
2649 +
    u32 opcode = ins.r.opcode;
2650 +
    if (opcode == 0x57 || opcode == 7 || opcode == 39)
2651 +
        return true;
2652 +
    switch (bitmanip_decode(ins.raw)) {
2653 +
    case BM_NONE:
2654 +
        return false;
2655 +
    case BM_SEXTB:
2656 +
    case BM_SEXTH:
2657 +
    case BM_ZEXTH:
2658 +
    case BM_ORCB:
2659 +
    case BM_CPOP:
2660 +
    case BM_CPOPW:
2661 +
        return true;
2662 +
    default:
2663 +
        return false;
2664 +
    }
2665 +
}
2666 +
2667 +
/* Host registers other than rax and rcx that an instruction clobbers: the
2668 +
 * high multiplications and divisions use rdx and r11. */
2669 +
static u32 scratch_mask(instr_t ins) {
2670 +
    u32 opcode = ins.r.opcode;
2671 +
    if ((opcode == OP_OP || opcode == OP_OP_32) && ins.r.funct7 == FUNCT7_MUL &&
2672 +
        ins.r.funct3 != 0 && bitmanip_decode(ins.raw) == BM_NONE)
2673 +
        return 1u << X_RDX | 1u << X_R11;
2674 +
    return 0;
2675 +
}
2676 +
2677 +
/* Whether a scalar instruction may be translated with every guest register
2678 +
 * in the register file: it addresses them only through host_of(), clobbers
2679 +
 * nothing beyond rax and rcx, and does not end the block. */
2680 +
static bool spill_neutral(instr_t ins) {
2681 +
    switch (ins.r.opcode) {
2682 +
    case OP_LUI:
2683 +
    case OP_AUIPC:
2684 +
    case OP_IMM:
2685 +
    case OP_IMM_32:
2686 +
    case OP_LOAD:
2687 +
    case OP_STORE:
2688 +
        return true;
2689 +
    case OP_OP:
2690 +
    case OP_OP_32:
2691 +
        return scratch_mask(ins) == 0;
2692 +
    default:
2693 +
        return false;
2694 +
    }
2695 +
}
2696 +
2697 +
/* Instructions a spill-neutral run may span before the next instruction that
2698 +
 * needs the register file, so that it is cheaper to stay spilled. */
2699 +
#define SPILL_LOOKAHEAD 4
2700 +
2701 +
/* Whether an instruction that needs the register file follows within a few
2702 +
 * spill-neutral instructions of `pc`. */
2703 +
static bool spill_follows(struct emitter *e, u32 pc) {
2704 +
    for (u32 i = 1; i <= SPILL_LOOKAHEAD; i++) {
2705 +
        u32 next = pc + i * INSTR_SIZE;
2706 +
        if (next < e->prog_base || next >= e->prog_end)
2707 +
            return false;
2708 +
        instr_t ins;
2709 +
        memcpy(&ins, &e->mem[next], sizeof(ins));
2710 +
        if (needs_spill(ins))
2711 +
            return true;
2712 +
        if (!spill_neutral(ins))
2713 +
            return false;
2714 +
    }
2715 +
    return false;
2716 +
}
2717 +
2718 +
/* Translate one instruction, keeping the held guest registers coherent
2719 +
 * around those whose emitters address the register file or clobber them. */
2720 +
static bool translate_insn(struct emitter *e, instr_t ins, u32 pc) {
2721 +
    /* Consecutive such instructions share one spill and reload. */
2722 +
    if (needs_spill(ins)) {
2723 +
        if (!e->spilled)
2724 +
            emit_spill(e);
2725 +
        return translate_body(e, ins, pc);
2726 +
    }
2727 +
    if (e->spilled && spill_neutral(ins) && spill_follows(e, pc))
2728 +
        return translate_body(e, ins, pc);
2729 +
    if (e->spilled)
2730 +
        emit_reload(e);
2731 +
    u32 mask = scratch_mask(ins);
2732 +
    u32 rd   = ins.r.rd;
2733 +
    if (!mask || rd == 0)
2734 +
        return translate_body(e, ins, pc);
2735 +
2736 +
    /* These clobber the scratch registers after reading their operands and
2737 +
     * write the destination last: save the scratch registers and restore
2738 +
     * all but the destination. */
2739 +
    emit_transfer(e, mask, true);
2740 +
    bool ends = translate_body(e, ins, pc);
2741 +
    emit_transfer(e, mask & ~(1u << host_of(e, rd)), false);
2742 +
    return ends;
2743 +
}
2744 +
2209 2745
/* ---------- Block compiler --------------------------------------------- */
2210 2746
2211 2747
static struct jit_block *jit_alloc_block(struct jit_state *jit) {
2212 2748
    if (jit->block_count >= JIT_MAX_BLOCKS)
2213 2749
        return NULL;
2218 2754
    u32 h              = (block->guest_pc >> 2) & (JIT_BLOCK_HASH_SIZE - 1);
2219 2755
    block->hash_next   = jit->block_hash[h];
2220 2756
    jit->block_hash[h] = block;
2221 2757
}
2222 2758
2223 -
struct jit_block *jit_compile_block(
2224 -
    struct jit_state *jit, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes
2225 -
) {
2226 -
2227 -
#define JIT_EMIT_BUF_SIZE (JIT_MAX_BLOCK_INSNS * JIT_MAX_INSN_BYTES + 256)
2228 -
    static u8 emit_buf[JIT_EMIT_BUF_SIZE];
2229 -
2230 -
    struct emitter em = {
2231 -
        .buf      = emit_buf,
2232 -
        .pos      = 0,
2233 -
        .capacity = JIT_EMIT_BUF_SIZE,
2234 -
        .overflow = false,
2235 -
        .jit      = jit,
2236 -
        .dest     = jit->code_cache + jit->code_cache_used,
2237 -
    };
2759 +
/* Whether an instruction may write ra.  Conservative: any instruction with
2760 +
 * ra in the destination field except stores and branches, which have none.
2761 +
 */
2762 +
static bool writes_ra(instr_t ins) {
2763 +
    u32 opcode = ins.r.opcode;
2764 +
    if (opcode == OP_STORE || opcode == OP_BRANCH || opcode == 39)
2765 +
        return false;
2766 +
    return ins.r.rd == RA;
2767 +
}
2238 2768
2769 +
/* Translate the block at `guest_pc` into `em`.  The vector shape guard is
2770 +
 * emitted only when `em->guard` is set. */
2771 +
static void translate_block(
2772 +
    struct emitter *em, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes
2773 +
) {
2239 2774
    /* if (budget <= 0) goto .exhausted */
2240 -
    emit_u8(&em, 0x4D);
2241 -
    emit_u8(&em, 0x85);
2242 -
    emit_u8(&em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */
2243 -
    u32 jle_pos = emit_jle_placeholder(&em);
2775 +
    emit_u8(em, 0x4D);
2776 +
    emit_u8(em, 0x85);
2777 +
    emit_u8(em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */
2778 +
    u32 jle_pos = emit_jle_placeholder(em);
2244 2779
2245 -
    u32 skip_vector  = emit_jmp_placeholder(&em);
2246 -
    u32 vector_guard = em.pos;
2247 -
    emit_vector_guard(&em, guest_pc);
2248 -
    u32 vector_body = em.pos;
2780 +
    if (em->guard)
2781 +
        emit_vector_guard(em, guest_pc);
2249 2782
2250 2783
    /* sub r15, insn_count -- imm32 is patched once the count is known. */
2251 -
    emit_u8(&em, 0x49);
2252 -
    emit_u8(&em, 0x81);
2253 -
    emit_u8(&em, modrm(3, 5, HBUDGET));
2254 -
    u32 count_pos = em.pos;
2255 -
    emit_u32(&em, 0);
2256 -
2257 -
    u32 pc          = guest_pc;
2258 -
    u32 insn_count  = 0;
2259 -
    u32 prog_end_pc = prog_base + prog_bytes;
2784 +
    emit_u8(em, 0x49);
2785 +
    emit_u8(em, 0x81);
2786 +
    emit_u8(em, modrm(3, 5, HBUDGET));
2787 +
    u32 count_pos = em->pos;
2788 +
    emit_u32(em, 0);
2789 +
2790 +
    u32 pc           = guest_pc;
2791 +
    u32 insn_count   = 0;
2792 +
    em->run_start[0] = guest_pc;
2793 +
    em->runs         = 1;
2794 +
    u32 prog_end_pc  = prog_base + prog_bytes;
2260 2795
2261 2796
    while (insn_count < JIT_MAX_BLOCK_INSNS) {
2262 2797
        if (pc < prog_base || pc >= prog_end_pc) {
2263 -
            emit_block_exit(&em, JIT_EXIT_FAULT, pc);
2798 +
            emit_block_exit(em, JIT_EXIT_FAULT, pc);
2264 2799
            break;
2265 2800
        }
2266 2801
        instr_t ins;
2267 2802
        memcpy(&ins, &mem[pc], sizeof(instr_t));
2268 2803
        insn_count++;
2269 -
        em.insns = insn_count;
2270 -
2271 -
        bool ends_block  = translate_insn(&em, ins, pc);
2272 -
        pc              += INSTR_SIZE;
2804 +
        em->insns = insn_count;
2805 +
2806 +
        if (writes_ra(ins))
2807 +
            em->ra_known = false;
2808 +
        em->next_pc     = pc + INSTR_SIZE;
2809 +
        bool ends_block = translate_insn(em, ins, pc);
2810 +
2811 +
        /* Remember the value of an auipc for a jalr right after it. */
2812 +
        em->auipc_reg = 0;
2813 +
        if (ins.u.opcode == OP_AUIPC && ins.u.rd != 0) {
2814 +
            em->auipc_reg   = ins.u.rd;
2815 +
            em->auipc_value = pc + (u64)(i64)(i32)(ins.u.imm_31_12 << 12);
2816 +
        }
2817 +
        pc = em->next_pc;
2273 2818
2274 2819
        if (ends_block)
2275 2820
            break;
2276 2821
2277 -
        if (em.pos + JIT_MAX_INSN_BYTES + 64 > em.capacity) {
2278 -
            emit_chain_exit(&em, pc);
2822 +
        if (em->pos + JIT_MAX_INSN_BYTES + 64 > em->capacity) {
2823 +
            emit_chain_exit(em, pc);
2279 2824
            break;
2280 2825
        }
2281 2826
    }
2282 2827
2283 2828
    if (insn_count >= JIT_MAX_BLOCK_INSNS) {
2284 -
        emit_chain_exit(&em, pc);
2829 +
        emit_chain_exit(em, pc);
2285 2830
    }
2286 2831
2832 +
    /* The side exits and the budget exit start with held registers. */
2833 +
    emit_side_exits(em);
2834 +
    em->spilled = false;
2835 +
2287 2836
    /* .exhausted: out of budget, leave before running anything. */
2288 -
    patch_jcc(&em, jle_pos, em.pos);
2289 -
    emit_block_exit(&em, JIT_EXIT_BRANCH, guest_pc);
2837 +
    patch_jcc(em, jle_pos, em->pos);
2838 +
    emit_block_exit(em, JIT_EXIT_BRANCH, guest_pc);
2290 2839
2291 -
    patch_jmp(&em, skip_vector, em.vector ? vector_guard : vector_body);
2292 -
    if (em.overflow)
2293 -
        return NULL;
2294 2840
    for (u32 i = 0; i < 4; i++)
2295 -
        em.buf[count_pos + i] = (u8)(insn_count >> (8 * i));
2841 +
        em->buf[count_pos + i] = (u8)(insn_count >> (8 * i));
2296 2842
2297 -
    for (u32 i = 0; i < em.refunds_len; i++) {
2298 -
        u32 refund = insn_count - em.consumed[i];
2843 +
    for (u32 i = 0; i < em->refunds_len; i++) {
2844 +
        u32 refund = insn_count - em->consumed[i];
2299 2845
        for (u32 j = 0; j < 4; j++)
2300 -
            em.buf[em.refunds[i] + j] = (u8)(refund >> (8 * j));
2846 +
            em->buf[em->refunds[i] + j] = (u8)(refund >> (8 * j));
2847 +
    }
2848 +
}
2849 +
2850 +
struct jit_block *jit_compile_block(
2851 +
    struct jit_state *jit, u32 guest_pc, u8 *mem, u32 prog_base, u32 prog_bytes
2852 +
) {
2853 +
2854 +
#define JIT_EMIT_BUF_SIZE (JIT_MAX_BLOCK_INSNS * JIT_MAX_INSN_BYTES + 256)
2855 +
    static u8             emit_buf[JIT_EMIT_BUF_SIZE];
2856 +
    static struct emitter em;
2857 +
2858 +
    /* A block that turns out to use four-lane SIMD is translated again with
2859 +
     * the guard that checks the vector shape on entry. */
2860 +
    for (u32 pass = 0; pass < 2; pass++) {
2861 +
        memset(&em, 0, sizeof(em));
2862 +
        em.buf       = emit_buf;
2863 +
        em.capacity  = JIT_EMIT_BUF_SIZE;
2864 +
        em.jit       = jit;
2865 +
        em.dest      = jit->code_cache + jit->code_cache_used;
2866 +
        em.guard     = pass == 1;
2867 +
        em.mem       = mem;
2868 +
        em.prog_base = prog_base;
2869 +
        em.prog_end  = prog_base + prog_bytes;
2870 +
2871 +
        translate_block(&em, guest_pc, mem, prog_base, prog_bytes);
2872 +
        if (!em.vector)
2873 +
            break;
2301 2874
    }
2875 +
    if (em.overflow)
2876 +
        return NULL;
2877 +
2302 2878
    u32 code_size = em.pos;
2303 2879
    if (jit->code_cache_used + code_size > JIT_CODE_CACHE_SIZE)
2304 2880
        return NULL;
2305 2881
2306 2882
    u8 *dest = jit->code_cache + jit->code_cache_used;
2328 2904
        site[i] = (u8)(rel >> (8 * i));
2329 2905
}
2330 2906
2331 2907
/* ---------- Init / Destroy -------------------------------------------- */
2332 2908
2333 -
/* Emit the shared exit stub and entry trampoline at the cache start. */
2909 +
/* Emit the shared exit stubs and entry trampoline at the cache start. */
2334 2910
static void jit_emit_trampolines(struct jit_state *jit) {
2335 -
    u8             buf[128];
2911 +
    u8             buf[512];
2336 2912
    struct emitter e = {
2337 2913
        .buf      = buf,
2338 2914
        .pos      = 0,
2339 2915
        .capacity = sizeof(buf),
2340 2916
        .overflow = false,
2341 2917
        .jit      = jit,
2342 2918
        .dest     = jit->code_cache,
2343 2919
    };
2344 2920
2345 -
    /* Exit stub: reason is in eax. */
2921 +
    /* Exit stub: reason is in eax.  Write back the held guest registers. */
2346 2922
    jit->exit_stub = jit->code_cache;
2923 +
    emit_transfer(&e, ALL_HOSTS, true);
2924 +
2925 +
    /* Raw exit: the register file is current. */
2926 +
    jit->exit_raw = jit->code_cache + e.pos;
2347 2927
    emit_u8(&e, 0x4D);
2348 2928
    emit_u8(&e, 0x89);
2349 2929
    emit_u8(&e, modrm(1, HBUDGET, HIO));
2350 2930
    emit_u8(&e, IO_BUDGET); /* mov [r14 + IO_BUDGET], r15 */
2351 2931
    emit_u8(&e, 0x41);
2354 2934
    emit_u8(&e, 0x5E); /* pop r14 */
2355 2935
    emit_u8(&e, 0x41);
2356 2936
    emit_u8(&e, 0x5D); /* pop r13 */
2357 2937
    emit_u8(&e, 0x41);
2358 2938
    emit_u8(&e, 0x5C); /* pop r12 */
2939 +
    emit_u8(&e, 0x5D); /* pop rbp */
2940 +
    emit_u8(&e, 0x5B); /* pop rbx */
2359 2941
    emit_u8(&e, 0xC3); /* ret */
2360 2942
2361 2943
    /* Entry: (rdi=regs, rsi=memory, rdx=io, rcx=code, r8=budget).  Blocks
2362 2944
     * make no calls, so the stack alignment is irrelevant. */
2363 2945
    jit->enter = (jit_enter_fn)(void *)(jit->code_cache + e.pos);
2946 +
    emit_u8(&e, 0x53); /* push rbx */
2947 +
    emit_u8(&e, 0x55); /* push rbp */
2364 2948
    emit_u8(&e, 0x41);
2365 2949
    emit_u8(&e, 0x54); /* push r12 */
2366 2950
    emit_u8(&e, 0x41);
2367 2951
    emit_u8(&e, 0x55); /* push r13 */
2368 2952
    emit_u8(&e, 0x41);
2379 2963
    emit_u8(&e, 0x89);
2380 2964
    emit_u8(&e, 0xD6); /* mov r14, rdx */
2381 2965
    emit_u8(&e, 0x4D);
2382 2966
    emit_u8(&e, 0x89);
2383 2967
    emit_u8(&e, 0xC7); /* mov r15, r8 */
2968 +
    emit_u8(&e, 0x48);
2969 +
    emit_u8(&e, 0x89);
2970 +
    emit_u8(&e, 0xC8); /* mov rax, rcx */
2971 +
    /* The argument registers are free now: load the held guest registers. */
2972 +
    emit_transfer(&e, ALL_HOSTS, false);
2384 2973
    emit_u8(&e, 0xFF);
2385 -
    emit_u8(&e, 0xE1); /* jmp rcx */
2974 +
    emit_u8(&e, 0xE0); /* jmp rax */
2386 2975
2387 2976
    memcpy(jit->code_cache, buf, e.pos);
2388 2977
    jit->code_cache_used = e.pos;
2389 2978
}
2390 2979
jit.h +23 -4
20 20
21 21
/* ---------------------------------------------------------------------------
22 22
 * Tuning constants
23 23
 * ------------------------------------------------------------------------- */
24 24
25 -
/* Size of the JIT code cache (16 MB). */
26 -
#define JIT_CODE_CACHE_SIZE (16 * 1024 * 1024)
25 +
/* Size of the JIT code cache (64 MB).  Only the pages used are committed. */
26 +
#define JIT_CODE_CACHE_SIZE (64 * 1024 * 1024)
27 27
28 28
/* Maximum number of x86-64 bytes emitted per guest instruction.
29 29
 * Conservative upper bound -- most instructions compile to <30 bytes. */
30 30
#define JIT_MAX_INSN_BYTES 1024
31 31
32 32
/* Maximum guest instructions in a single translated block. */
33 33
#define JIT_MAX_BLOCK_INSNS 256
34 34
35 +
/* Maximum conditional branches a translated block continues past.  Each
36 +
 * taken branch leaves the block through a side exit. */
37 +
#define JIT_MAX_SIDE_EXITS 8
38 +
39 +
/* Maximum direct jumps a translated block follows to their targets. */
40 +
#define JIT_MAX_FOLLOWS 4
41 +
35 42
/* Number of hash-table buckets for block lookup (must be power of 2). */
36 43
#define JIT_BLOCK_HASH_SIZE (64 * 1024)
37 44
38 45
/* Maximum number of translated blocks. */
39 46
#define JIT_MAX_BLOCKS (256 * 1024)
40 47
41 48
/* Number of jump cache entries (must be power of 2). */
42 -
#define JIT_JUMP_CACHE_SIZE 16384
49 +
#define JIT_JUMP_CACHE_SIZE 65536
43 50
44 51
/* Guest PC that marks an empty jump cache entry.  Indirect jump targets are
45 52
 * always even, so this never matches. */
46 53
#define JIT_JUMP_CACHE_EMPTY 0xFFFFFFFFu
47 54
68 75
    u32                  pc;          /* Next guest PC on exit. */
69 76
    i64                  budget;      /* Remaining guest instructions. */
70 77
    u8                  *patch;       /* Chain jump rel32 field. */
71 78
    struct vector_state *vector;      /* Architectural vector registers. */
72 79
    u64                  memory_size; /* Accessible hosted memory bytes. */
80 +
    /* LR reservation: the address in the low word and the access width in
81 +
     * the high word, or zero when none is held. */
82 +
    u64 reservation;
73 83
};
74 84
85 +
/* Reservation value of an LR of `size` bytes at `address`. */
86 +
static inline u64 jit_reservation(u32 address, u32 size) {
87 +
    return (u64)size << 32 | address;
88 +
}
89 +
75 90
/* One entry of the indirect jump cache.  Generated code indexes the cache
76 91
 * assuming 16-byte entries. */
77 92
struct jit_jump_entry {
78 93
    u32 guest_pc; /* Guest PC, or JIT_JUMP_CACHE_EMPTY. */
79 94
    u8 *code;     /* Entry point of the block at that PC. */
97 112
98 113
    /* Block storage (static pool). */
99 114
    struct jit_block blocks[JIT_MAX_BLOCKS];
100 115
    u32              block_count;
101 116
102 -
    /* Shared entry trampoline and exit stub, at the start of the cache. */
117 +
    /* Shared entry trampoline and exit stubs, at the start of the cache.
118 +
     * `exit_stub` writes the guest registers held in host registers back to
119 +
     * the register file and falls into `exit_raw`, which returns without
120 +
     * writing them. */
103 121
    jit_enter_fn enter;
104 122
    u8          *exit_stub;
123 +
    u8          *exit_raw;
105 124
106 125
    /* Exit information for the current run. */
107 126
    struct jit_io io;
108 127
109 128
    /* Direct-mapped cache of indirect jump targets, read by generated code
test/jit_calls.c added +148 -0
1 +
/* Writes a raw RV64 image for comparing translated and interpreted runs.
2 +
 *
3 +
 * The program makes calls through jal and through auipc and jalr, returns
4 +
 * through ra, saves ra in a non-leaf function, branches around ebreak as a
5 +
 * bounds check does, and uses LR and SC.  Translated code follows these
6 +
 * calls and checks the returns in place, so both runs must end with the
7 +
 * same status after the same number of instructions. */
8 +
#include <stdint.h>
9 +
#include <stdio.h>
10 +
11 +
typedef uint32_t u32;
12 +
typedef int32_t  i32;
13 +
14 +
enum { ZERO = 0, RA = 1, SP = 2, T0 = 5, T1 = 6, T2 = 7, S1 = 9 };
15 +
enum { A0 = 10, A1, A2, A3, A4, A5, A7 = 17, T5 = 30, T6 = 31 };
16 +
17 +
static u32 code[256];
18 +
static u32 len;
19 +
20 +
/* Labels and the instructions that refer to them. */
21 +
enum label { LOOP, LEAF, LEAF_OK, NEST, LABELS };
22 +
static u32 labels[LABELS];
23 +
static struct {
24 +
    u32        at;
25 +
    enum label to;
26 +
    u32        kind; /* 0 jal, 1 branch, 2 jalr after auipc */
27 +
    u32        ins;  /* Instruction without its offset. */
28 +
} fixups[32];
29 +
static u32 nfixups;
30 +
31 +
static u32 r_type(u32 op, u32 rd, u32 f3, u32 rs1, u32 rs2, u32 f7) {
32 +
    return f7 << 25 | rs2 << 20 | rs1 << 15 | f3 << 12 | rd << 7 | op;
33 +
}
34 +
35 +
static u32 i_type(u32 op, u32 rd, u32 f3, u32 rs1, i32 imm) {
36 +
    return ((u32)imm & 0xfff) << 20 | rs1 << 15 | f3 << 12 | rd << 7 | op;
37 +
}
38 +
39 +
static u32 s_type(u32 rs2, u32 rs1, u32 f3, i32 imm) {
40 +
    u32 v = (u32)imm;
41 +
    return (v >> 5 & 0x7f) << 25 | rs2 << 20 | rs1 << 15 | f3 << 12 |
42 +
           (v & 0x1f) << 7 | 0x23;
43 +
}
44 +
45 +
static void emit(u32 ins) {
46 +
    code[len++] = ins;
47 +
}
48 +
49 +
static void addi(u32 rd, u32 rs1, i32 imm) {
50 +
    emit(i_type(0x13, rd, 0, rs1, imm));
51 +
}
52 +
53 +
static void ref(u32 kind, u32 ins, enum label to) {
54 +
    fixups[nfixups].at   = len;
55 +
    fixups[nfixups].to   = to;
56 +
    fixups[nfixups].kind = kind;
57 +
    fixups[nfixups].ins  = ins;
58 +
    nfixups++;
59 +
    emit(0);
60 +
}
61 +
62 +
static void mark(enum label l) {
63 +
    labels[l] = len * 4;
64 +
}
65 +
66 +
static void resolve(void) {
67 +
    for (u32 i = 0; i < nfixups; i++) {
68 +
        u32 pc  = fixups[i].at * 4;
69 +
        u32 off = labels[fixups[i].to] - pc;
70 +
        u32 ins = fixups[i].ins;
71 +
        if (fixups[i].kind == 0) {
72 +
            ins |= (off >> 20 & 1) << 31 | (off >> 1 & 0x3ff) << 21 |
73 +
                   (off >> 11 & 1) << 20 | (off >> 12 & 0xff) << 12;
74 +
        } else if (fixups[i].kind == 1) {
75 +
            ins |= (off >> 12 & 1) << 31 | (off >> 5 & 0x3f) << 25 |
76 +
                   (off >> 1 & 0xf) << 8 | (off >> 11 & 1) << 7;
77 +
        } else {
78 +
            ins |= ((off + 4) & 0xfff) << 20; /* Relative to the auipc. */
79 +
        }
80 +
        code[fixups[i].at] = ins;
81 +
    }
82 +
}
83 +
84 +
int main(int argc, char **argv) {
85 +
    if (argc != 2) {
86 +
        fprintf(stderr, "usage: %s <output>\n", argv[0]);
87 +
        return 1;
88 +
    }
89 +
    /* main: sum calls over 200 iterations into a0. */
90 +
    addi(A0, ZERO, 0);
91 +
    addi(S1, ZERO, 200);
92 +
    addi(T6, SP, -64);
93 +
    mark(LOOP);
94 +
    addi(A1, S1, 0);
95 +
    ref(0, RA << 7 | 0x6f, LEAF);             /* jal ra, leaf */
96 +
    ref(0, RA << 7 | 0x6f, NEST);             /* jal ra, nest */
97 +
    emit(T5 << 7 | 0x17);                     /* auipc t5, 0 */
98 +
    ref(2, i_type(0x67, RA, 0, T5, 0), LEAF); /* jalr ra, leaf(t5) */
99 +
    addi(S1, S1, -1);
100 +
    ref(1, r_type(0x63, 0, 1, S1, ZERO, 0), LOOP); /* bnez s1, loop */
101 +
102 +
    /* Add 7 to a doubleword with LR and SC, then fold it into a0. */
103 +
    emit(0x1000302f | T6 << 15 | T0 << 7); /* lr.d t0, (t6) */
104 +
    addi(T0, T0, 7);
105 +
    emit(0x1800302f | T6 << 15 | T0 << 20 | T1 << 7); /* sc.d t1, t0, (t6) */
106 +
    emit(r_type(0x33, A0, 0, A0, T1, 0));
107 +
    emit(i_type(0x03, T2, 3, T6, 0)); /* ld t2, 0(t6) */
108 +
    emit(r_type(0x33, A0, 0, A0, T2, 0));
109 +
    emit(i_type(0x13, A0, 7, A0, 0xff));
110 +
    addi(A7, ZERO, 0x400); /* exit(a0) */
111 +
    emit(0x73);
112 +
113 +
    /* leaf(a1): bounds check, then mix a1 into a0. */
114 +
    mark(LEAF);
115 +
    addi(A2, ZERO, 1000);
116 +
    ref(1, r_type(0x63, 0, 6, A1, A2, 0), LEAF_OK); /* bltu a1, a2, ok */
117 +
    emit(0x00100073);                               /* ebreak */
118 +
    mark(LEAF_OK);
119 +
    addi(A3, A1, 0);
120 +
    emit(r_type(0x33, A3, 1, A3, A3, 0)); /* sll a3, a3, a3 */
121 +
    addi(A4, ZERO, 3);
122 +
    emit(r_type(0x33, A4, 0, A1, A4, 1)); /* mul a4, a1, a4 */
123 +
    emit(i_type(0x13, A5, 5, A1, 1));     /* srli a5, a1, 1 */
124 +
    emit(r_type(0x33, A4, 4, A4, A5, 0)); /* xor a4, a4, a5 */
125 +
    emit(r_type(0x33, A0, 0, A0, A4, 0));
126 +
    emit(r_type(0x3b, A0, 0, A0, A3, 0x20)); /* subw a0, a0, a3 */
127 +
    emit(i_type(0x67, ZERO, 0, RA, 0));      /* ret */
128 +
129 +
    /* nest: save ra, call leaf twice, restore ra and return. */
130 +
    mark(NEST);
131 +
    addi(SP, SP, -16);
132 +
    emit(s_type(RA, SP, 3, 8));
133 +
    addi(A1, A1, 5);
134 +
    ref(0, RA << 7 | 0x6f, LEAF);
135 +
    addi(A1, A1, -3);
136 +
    ref(0, RA << 7 | 0x6f, LEAF);
137 +
    emit(i_type(0x03, RA, 3, SP, 8));
138 +
    addi(SP, SP, 16);
139 +
    emit(i_type(0x67, ZERO, 0, RA, 0));
140 +
141 +
    resolve();
142 +
    FILE *f = fopen(argv[1], "wb");
143 +
    if (!f || fwrite(code, 4, len, f) != len || fclose(f)) {
144 +
        perror(argv[1]);
145 +
        return 1;
146 +
    }
147 +
    return 0;
148 +
}