jit: Translate four-lane integer vector operations

137a6e3cfcae862c914862e6821909081b800906ff2fe0563422cc985346ff23
Guard e32,m2 blocks with vl=4 and vstart=0. Emit native integer
arithmetic, bitwise operations, rotates and gathers, with checked
unit-stride transfers and indexed loads. Return unsupported states and
faulting accesses to the interpreter with accurate instruction counts.

Add sanitizer-backed differential checks for register state, memory,
faults and instruction accounting. Include the JIT checks in test-vector.

Assisted-by: Codex:gpt-6
Change-Id: I1231c0279f09a03a5446f9024d5125460eb6852d
Alexis Sellier committed ago 1 parent 117807f8
Makefile +11 -1
51 51
52 52
bin/vector-machine-test: test/vector_machine.c vector.c vector.h machine_cpu.c machine_csr.c machine_devices.c machine.c machine_fdt.c machine_internal.h machine.h types.h bitmanip.h muldiv.h
53 53
	@mkdir -p bin
54 54
	@$(CC) $(CPPFLAGS) -std=c99 -Wall -Wextra -g -fsanitize=address,undefined test/vector_machine.c vector.c machine_cpu.c machine_csr.c machine_devices.c machine.c machine_fdt.c -o $@
55 55
56 -
test-vector: bin/vector-test bin/vector-machine-test
56 +
test-vector: bin/vector-test bin/vector-machine-test test-jit
57 57
	@ASAN_OPTIONS=detect_leaks=0 timeout 30 bin/vector-test
58 58
	@ASAN_OPTIONS=detect_leaks=0 timeout 30 bin/vector-machine-test
59 59
60 60
.PHONY: test-vector
61 +
62 +
# Compare hosted vector translation with architectural interpreter state.
63 +
bin/jit-vector-test: test/jit_vector.c jit.c jit.h vector.c vector.h riscv.c riscv.h riscv/debug.c riscv/debug.h bitmanip.h color.h types.h
64 +
	@mkdir -p bin
65 +
	@$(CC) $(CPPFLAGS) -std=c99 -O2 -g -Wall -Wextra -fsanitize=address,undefined test/jit_vector.c jit.c vector.c riscv.c riscv/debug.c -o $@
66 +
67 +
test-jit: bin/jit-vector-test
68 +
	@ASAN_OPTIONS=detect_leaks=0 timeout 60 $<
69 +
70 +
.PHONY: test-jit
emulator.c +2 -0
2569 2569
            !g_opts.trace_print_instructions && !g_opts.watch_enabled) {
2570 2570
            use_jit = jit_init(&jit);
2571 2571
        }
2572 2572
2573 2573
        if (use_jit) {
2574 +
            jit.io.vector      = &cpu.vector;
2575 +
            jit.io.memory_size = guest_sizes().memory;
2574 2576
            /* ---- JIT execution loop ---- */
2575 2577
            u8 *patch = NULL; /* Direct jump to link to the next block. */
2576 2578
2577 2579
            while (cpu.running && steps < max_steps) {
2578 2580
                struct jit_block *block = jit_get_block(
jit.c +276 -0
11 11
 *       r12 = &regs[0]   (guest register file)
12 12
 *       r13 = memory base
13 13
 *       r14 = &jit->io   (exit information; pc is its first field)
14 14
 *       r15 = budget     (guest instructions left, signed)
15 15
 *   - Guest registers are loaded/stored on demand from/to the array.
16 +
 *   - Vector blocks guard the four-word e32,m2 shape before any side effect.
17 +
 *     Other shapes return to the interpreter.
16 18
 *   - Each block starts with "if budget <= 0 exit at this PC; budget -=
17 19
 *     insn_count".  Every exit is the block's last instruction, so a block
18 20
 *     always accounts exactly the instructions it executes.
19 21
 *   - Direct jumps (jal, both sides of a branch, block split) are a jmp
20 22
 *     rel32 that first falls into a stub returning JIT_EXIT_CHAIN.  C then
32 34
33 35
#include "bitmanip.h"
34 36
#include "jit.h"
35 37
#include "riscv.h"
36 38
#include "types.h"
39 +
#include "vector.h"
37 40
38 41
/* ---------- x86-64 code emitter helpers --------------------------------- */
39 42
40 43
/* A small code buffer that we fill up, then copy into the cache. */
41 44
struct emitter {
42 45
    u8  *buf;      /* Start of buffer. */
43 46
    u32  pos;      /* Current write position. */
44 47
    u32  capacity; /* Maximum bytes. */
45 48
    bool overflow; /* Set if we ran out of space. */
49 +
    bool vector;   /* Block uses four-lane 32-bit SIMD. */
50 +
    u32  insns;    /* Instructions through the current operation. */
51 +
    /* Each indexed load can fail three checks for each of four lanes. */
52 +
    u32 refunds[JIT_MAX_BLOCK_INSNS * 12];  /* Budget patch offsets. */
53 +
    u32 consumed[JIT_MAX_BLOCK_INSNS * 12]; /* Instructions at each patch. */
54 +
    u32 refunds_len;                        /* Number of guarded exits. */
46 55
47 56
    struct jit_state *jit;  /* For the exit stub and jump cache addresses. */
48 57
    u8               *dest; /* Address the buffer will be copied to. */
49 58
};
50 59
1075 1084
        break;
1076 1085
    }
1077 1086
    emit_store_rd(e, ins.r.rd);
1078 1087
}
1079 1088
1089 +
/* ---------- Vector translation ----------------------------------------- */
1090 +
1091 +
/* Return a failed memory operation to the interpreter before its side effects.
1092 +
 */
1093 +
static void emit_vector_fault(struct emitter *e, u32 pc, u8 condition) {
1094 +
    u32 bad = e->pos;
1095 +
    emit_u8(e, 0x0f);
1096 +
    emit_u8(e, condition);
1097 +
    emit_u32(e, 0);
1098 +
    u32 ok = emit_jmp_placeholder(e);
1099 +
    patch_jcc(e, bad, e->pos);
1100 +
    emit_u8(e, 0x49);
1101 +
    emit_u8(e, 0x81);
1102 +
    emit_u8(e, modrm(3, 0, HBUDGET));
1103 +
    e->refunds[e->refunds_len]    = e->pos;
1104 +
    e->consumed[e->refunds_len++] = e->insns;
1105 +
    emit_u32(e, 0);
1106 +
    emit_block_exit(e, JIT_EXIT_FAULT, pc);
1107 +
    patch_jmp(e, ok, e->pos);
1108 +
}
1109 +
1110 +
/* Load the architectural vector state into rax. */
1111 +
static void emit_vector_base(struct emitter *e) {
1112 +
    emit_u8(e, 0x49);
1113 +
    emit_u8(e, 0x8b);
1114 +
    emit_u8(e, modrm(1, X_RAX, HIO));
1115 +
    emit_u8(e, offsetof(struct jit_io, vector));
1116 +
}
1117 +
1118 +
/* Form one indexed guest address, with overflow checked before any transfer. */
1119 +
static void emit_vector_address(struct emitter *e, u32 ins, u32 lane, u32 pc) {
1120 +
    u32 base = (ins >> 15) & 31, index = (ins >> 20) & 31;
1121 +
    emit_vector_base(e);
1122 +
    emit_u8(e, 0x8b);
1123 +
    emit_u8(e, modrm(2, X_RDX, X_RAX));
1124 +
    emit_u32(e, index * VECTOR_BYTES + lane * 4);
1125 +
    emit_load_guest(e, X_RAX, base);
1126 +
    emit_u8(e, 0x48);
1127 +
    emit_u8(e, 0x01);
1128 +
    emit_u8(e, 0xd0);
1129 +
    emit_vector_fault(e, pc, 0x82); /* jc */
1130 +
}
1131 +
1132 +
/* Check a complete transfer against the hosted memory bound. */
1133 +
static void emit_vector_bound(struct emitter *e, u32 size, u32 pc) {
1134 +
    emit_u8(e, 0x49);
1135 +
    emit_u8(e, 0x8b);
1136 +
    emit_u8(e, modrm(1, X_RCX, HIO));
1137 +
    emit_u8(e, offsetof(struct jit_io, memory_size));
1138 +
    emit_u8(e, 0x48);
1139 +
    emit_u8(e, 0x83);
1140 +
    emit_u8(e, 0xe9);
1141 +
    emit_u8(e, size);
1142 +
    emit_vector_fault(e, pc, 0x82);
1143 +
    emit_u8(e, 0x48);
1144 +
    emit_u8(e, 0x39);
1145 +
    emit_u8(e, 0xc8);
1146 +
    emit_vector_fault(e, pc, 0x87); /* ja */
1147 +
}
1148 +
1149 +
/* Emit an SSE instruction on two XMM registers. */
1150 +
static void emit_xmm(struct emitter *e, u8 opcode, u32 d, u32 src) {
1151 +
    emit_u8(e, 0x66);
1152 +
    emit_u8(e, 0x0f);
1153 +
    emit_u8(e, opcode);
1154 +
    emit_u8(e, modrm(3, d, src));
1155 +
}
1156 +
1157 +
/* Transfer four words between an XMM register and architectural state in rax.
1158 +
 */
1159 +
static void emit_xmm_memory(struct emitter *e, u32 xmm, u32 reg, bool store) {
1160 +
    emit_u8(e, 0xf3);
1161 +
    emit_u8(e, 0x0f);
1162 +
    emit_u8(e, store ? 0x7f : 0x6f);
1163 +
    emit_u8(e, modrm(2, xmm, X_RAX));
1164 +
    emit_u32(e, reg * VECTOR_BYTES);
1165 +
}
1166 +
1167 +
/* Emit checked unmasked word loads and stores for one register group. */
1168 +
static bool emit_vector_memory(struct emitter *e, u32 ins, u32 pc) {
1169 +
    u32  op = ins & 127, reg = (ins >> 7) & 31, index = (ins >> 20) & 31;
1170 +
    u32  mode  = (ins >> 26) & 3;
1171 +
    bool store = op == 39;
1172 +
    if ((op != 7 && !store) || ((ins >> 12) & 7) != 6 || !(ins & (1u << 25)) ||
1173 +
        (ins >> 29) || (ins & (1u << 28)) || (reg & 1) ||
1174 +
        !((mode == 0 && index == 0) || (mode == 1 && !store && !(index & 1))))
1175 +
        return false;
1176 +
    if (mode == 1 && !store && reg == index)
1177 +
        return false;
1178 +
    e->vector = true;
1179 +
    if (mode == 0) {
1180 +
        emit_load_guest(e, X_RAX, (ins >> 15) & 31);
1181 +
        emit_vector_bound(e, 16, pc);
1182 +
        if (store) {
1183 +
            emit_vector_base(e);
1184 +
            emit_xmm_memory(e, 0, reg, false);
1185 +
            emit_load_guest(e, X_RAX, (ins >> 15) & 31);
1186 +
        }
1187 +
        emit_add_rax_r13(e);
1188 +
        emit_u8(e, 0xf3);
1189 +
        emit_u8(e, 0x0f);
1190 +
        emit_u8(e, store ? 0x7f : 0x6f);
1191 +
        emit_u8(e, 0x00);
1192 +
        if (!store) {
1193 +
            emit_vector_base(e);
1194 +
            emit_xmm_memory(e, 0, reg, true);
1195 +
        }
1196 +
    } else {
1197 +
        /* Preflight every address so an exceptional access has no partial
1198 +
         * writes. */
1199 +
        for (u32 i = 0; i < 4; i++) {
1200 +
            emit_vector_address(e, ins, i, pc);
1201 +
            emit_vector_bound(e, 4, pc);
1202 +
        }
1203 +
        for (u32 i = 0; i < 4; i++) {
1204 +
            emit_vector_base(e);
1205 +
            emit_u8(e, 0x8b);
1206 +
            emit_u8(e, modrm(2, X_RDX, X_RAX));
1207 +
            emit_u32(e, index * VECTOR_BYTES + i * 4);
1208 +
            emit_load_guest(e, X_RAX, (ins >> 15) & 31);
1209 +
            emit_u8(e, 0x48);
1210 +
            emit_u8(e, 0x01);
1211 +
            emit_u8(e, 0xd0);
1212 +
            emit_add_rax_r13(e);
1213 +
            emit_u8(e, 0x8b);
1214 +
            emit_u8(e, 0x10); /* mov edx, [rax] */
1215 +
            emit_vector_base(e);
1216 +
            emit_u8(e, 0x89);
1217 +
            emit_u8(e, modrm(2, X_RDX, X_RAX));
1218 +
            emit_u32(e, reg * VECTOR_BYTES + i * 4);
1219 +
        }
1220 +
    }
1221 +
    return true;
1222 +
}
1223 +
1224 +
/* Gather four words from a validated, non-overlapping register group. */
1225 +
static void emit_vector_gather(struct emitter *e, u32 d, u32 a, u32 b) {
1226 +
    emit_vector_base(e);
1227 +
    for (u32 i = 0; i < 4; i++) {
1228 +
        emit_u8(e, 0x8b);
1229 +
        emit_u8(e, modrm(2, X_RDX, X_RAX));
1230 +
        emit_u32(e, b * VECTOR_BYTES + i * 4);
1231 +
        emit_u8(e, 0x31);
1232 +
        emit_u8(e, 0xc9); /* xor ecx, ecx */
1233 +
        emit_u8(e, 0x83);
1234 +
        emit_u8(e, 0xfa);
1235 +
        emit_u8(e, 16);
1236 +
        u32 outside = e->pos;
1237 +
        emit_u8(e, 0x0f);
1238 +
        emit_u8(e, 0x83);
1239 +
        emit_u32(e, 0);
1240 +
        emit_u8(e, 0x8b);
1241 +
        emit_u8(e, modrm(2, X_RCX, X_RSP));
1242 +
        emit_u8(e, sib(2, X_RDX, X_RAX));
1243 +
        emit_u32(e, a * VECTOR_BYTES);
1244 +
        patch_jcc(e, outside, e->pos);
1245 +
        emit_u8(e, 0x89);
1246 +
        emit_u8(e, modrm(2, X_RCX, X_RAX));
1247 +
        emit_u32(e, d * VECTOR_BYTES + i * 4);
1248 +
    }
1249 +
}
1250 +
1251 +
/* Emit unmasked four-word integer operations under the block's state guard. */
1252 +
static bool emit_vector(struct emitter *e, u32 ins) {
1253 +
    u32 fn = ins >> 26, form = (ins >> 12) & 7;
1254 +
    u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31;
1255 +
    if (!(ins & (1u << 25)) || (d & 1) || (a & 1))
1256 +
        return false;
1257 +
    if (!((form == 0 && !(b & 1) &&
1258 +
           (fn == 0 || fn == 9 || fn == 11 ||
1259 +
            (fn == 12 && d != a && d != b))) ||
1260 +
          (form == 3 &&
1261 +
           (fn == 0 || fn == 9 || fn == 11 || fn == 20 || fn == 21))))
1262 +
        return false;
1263 +
    e->vector = true;
1264 +
    if (form == 0 && fn == 12) {
1265 +
        emit_vector_gather(e, d, a, b);
1266 +
        return true;
1267 +
    }
1268 +
    emit_vector_base(e);
1269 +
    emit_xmm_memory(e, 0, a, false);
1270 +
    if (form == 3 && (fn == 20 || fn == 21)) {
1271 +
        emit_xmm(e, 0x6f, 1, 0);
1272 +
        emit_xmm(e, 0x72, 2, 0);
1273 +
        emit_u8(e, b);
1274 +
        emit_xmm(e, 0x72, 6, 1);
1275 +
        emit_u8(e, 32 - b);
1276 +
        emit_xmm(e, 0xeb, 0, 1);
1277 +
    } else {
1278 +
        if (form == 0) {
1279 +
            emit_xmm_memory(e, 1, b, false);
1280 +
        } else {
1281 +
            emit_u8(e, 0xb9);
1282 +
            emit_u32(e, (u32)((i32)(b << 27) >> 27));
1283 +
            emit_xmm(e, 0x6e, 1, X_RCX);
1284 +
            emit_xmm(e, 0x70, 1, 1);
1285 +
            emit_u8(e, 0);
1286 +
        }
1287 +
        emit_xmm(e, fn == 0 ? 0xfe : fn == 9 ? 0xdb : 0xef, 0, 1);
1288 +
    }
1289 +
    emit_xmm_memory(e, 0, d, true);
1290 +
    return true;
1291 +
}
1292 +
1293 +
/* Guard the vector shape before any instruction in the block has side effects.
1294 +
 */
1295 +
static void emit_vector_guard(struct emitter *e, u32 pc) {
1296 +
    emit_u8(e, 0x49);
1297 +
    emit_u8(e, 0x8b);
1298 +
    emit_u8(e, modrm(1, X_RAX, HIO));
1299 +
    emit_u8(e, offsetof(struct jit_io, vector));
1300 +
    u32       bad[3];
1301 +
    const u32 offsets[3] = { offsetof(struct vector_state, vtype),
1302 +
                             offsetof(struct vector_state, vl),
1303 +
                             offsetof(struct vector_state, vstart) };
1304 +
    const u32 values[3]  = { 0x11, 4, 0 };
1305 +
    for (u32 i = 0; i < 3; i++) {
1306 +
        if (i == 0) {
1307 +
            /* Tail and mask policies do not change the four active lanes. */
1308 +
            emit_u8(e, 0x48);
1309 +
            emit_u8(e, 0x8b);
1310 +
            emit_u8(e, modrm(2, X_RCX, X_RAX));
1311 +
            emit_u32(e, offsets[i]);
1312 +
            emit_u8(e, 0x48);
1313 +
            emit_u8(e, 0x81);
1314 +
            emit_u8(e, 0xe1);
1315 +
            emit_u32(e, ~0xc0u);
1316 +
            emit_u8(e, 0x48);
1317 +
            emit_u8(e, 0x81);
1318 +
            emit_u8(e, 0xf9);
1319 +
        } else {
1320 +
            emit_u8(e, 0x48);
1321 +
            emit_u8(e, 0x81);
1322 +
            emit_u8(e, modrm(2, 7, X_RAX));
1323 +
            emit_u32(e, offsets[i]);
1324 +
        }
1325 +
        emit_u32(e, values[i]);
1326 +
        bad[i] = emit_jne_placeholder(e);
1327 +
    }
1328 +
    u32 ok = emit_jmp_placeholder(e);
1329 +
    for (u32 i = 0; i < 3; i++)
1330 +
        patch_jcc(e, bad[i], e->pos);
1331 +
    emit_u8(e, 0x49);
1332 +
    emit_u8(e, 0x83);
1333 +
    emit_u8(e, modrm(3, 5, HBUDGET));
1334 +
    emit_u8(e, 1);
1335 +
    emit_block_exit(e, JIT_EXIT_FAULT, pc);
1336 +
    patch_jmp(e, ok, e->pos);
1337 +
}
1338 +
1080 1339
/* ---------- Translate one RV64I instruction ----------------------------- */
1081 1340
1082 1341
/*
1083 1342
 * Returns: true if the instruction ends the basic block (branch/jump/ecall),
1084 1343
 *          false if execution should continue to the next instruction.
1091 1350
    if (bm != BM_NONE) {
1092 1351
        emit_bitmanip(e, ins, bm);
1093 1352
        return false;
1094 1353
    }
1095 1354
1355 +
    if ((opcode == 7 || opcode == 39) && emit_vector_memory(e, ins.raw, pc))
1356 +
        return false;
1357 +
    if (opcode == 0x57 && emit_vector(e, ins.raw))
1358 +
        return false;
1359 +
1096 1360
    switch (opcode) {
1097 1361
1098 1362
    case OP_LUI:
1099 1363
        if (ins.u.rd != 0) {
1100 1364
            emit_mov_imm32_sx(e, HTMP1, (i32)(ins.u.imm_31_12 << 12));
1561 1825
    emit_u8(&em, 0x4D);
1562 1826
    emit_u8(&em, 0x85);
1563 1827
    emit_u8(&em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */
1564 1828
    u32 jle_pos = emit_jle_placeholder(&em);
1565 1829
1830 +
    u32 skip_vector  = emit_jmp_placeholder(&em);
1831 +
    u32 vector_guard = em.pos;
1832 +
    emit_vector_guard(&em, guest_pc);
1833 +
    u32 vector_body = em.pos;
1834 +
1566 1835
    /* sub r15, insn_count -- imm32 is patched once the count is known. */
1567 1836
    emit_u8(&em, 0x49);
1568 1837
    emit_u8(&em, 0x81);
1569 1838
    emit_u8(&em, modrm(3, 5, HBUDGET));
1570 1839
    u32 count_pos = em.pos;
1580 1849
            break;
1581 1850
        }
1582 1851
        instr_t ins;
1583 1852
        memcpy(&ins, &mem[pc], sizeof(instr_t));
1584 1853
        insn_count++;
1854 +
        em.insns = insn_count;
1585 1855
1586 1856
        bool ends_block  = translate_insn(&em, ins, pc);
1587 1857
        pc              += INSTR_SIZE;
1588 1858
1589 1859
        if (ends_block)
1601 1871
1602 1872
    /* .exhausted: out of budget, leave before running anything. */
1603 1873
    patch_jcc(&em, jle_pos, em.pos);
1604 1874
    emit_block_exit(&em, JIT_EXIT_BRANCH, guest_pc);
1605 1875
1876 +
    patch_jmp(&em, skip_vector, em.vector ? vector_guard : vector_body);
1606 1877
    if (em.overflow)
1607 1878
        return NULL;
1608 1879
    for (u32 i = 0; i < 4; i++)
1609 1880
        em.buf[count_pos + i] = (u8)(insn_count >> (8 * i));
1610 1881
1882 +
    for (u32 i = 0; i < em.refunds_len; i++) {
1883 +
        u32 refund = insn_count - em.consumed[i];
1884 +
        for (u32 j = 0; j < 4; j++)
1885 +
            em.buf[em.refunds[i] + j] = (u8)(refund >> (8 * j));
1886 +
    }
1611 1887
    u32 code_size = em.pos;
1612 1888
    if (jit->code_cache_used + code_size > JIT_CODE_CACHE_SIZE)
1613 1889
        return NULL;
1614 1890
1615 1891
    u8 *dest = jit->code_cache + jit->code_cache_used;
jit.h +7 -4
14 14
#include "riscv.h"
15 15
#include "types.h"
16 16
17 17
/* Forward declarations. */
18 18
struct cpu;
19 +
struct vector_state;
19 20
20 21
/* ---------------------------------------------------------------------------
21 22
 * Tuning constants
22 23
 * ------------------------------------------------------------------------- */
23 24
24 25
/* Size of the JIT code cache (16 MB). */
25 26
#define JIT_CODE_CACHE_SIZE (16 * 1024 * 1024)
26 27
27 28
/* Maximum number of x86-64 bytes emitted per guest instruction.
28 29
 * Conservative upper bound -- most instructions compile to <30 bytes. */
29 -
#define JIT_MAX_INSN_BYTES 128
30 +
#define JIT_MAX_INSN_BYTES 1024
30 31
31 32
/* Maximum guest instructions in a single translated block. */
32 33
#define JIT_MAX_BLOCK_INSNS 256
33 34
34 35
/* Number of hash-table buckets for block lookup (must be power of 2). */
62 63
};
63 64
64 65
/* Exchange area between generated code and C.  Generated code keeps a pointer
65 66
 * to it in r14. */
66 67
struct jit_io {
67 -
    u32 pc;     /* Next guest PC on exit. */
68 -
    i64 budget; /* Remaining guest instructions; in on entry, out on exit. */
69 -
    u8 *patch;  /* On JIT_EXIT_CHAIN, the rel32 field of the jmp to patch. */
68 +
    u32                  pc;          /* Next guest PC on exit. */
69 +
    i64                  budget;      /* Remaining guest instructions. */
70 +
    u8                  *patch;       /* Chain jump rel32 field. */
71 +
    struct vector_state *vector;      /* Architectural vector registers. */
72 +
    u64                  memory_size; /* Accessible hosted memory bytes. */
70 73
};
71 74
72 75
/* One entry of the indirect jump cache.  Generated code indexes the cache
73 76
 * assuming 16-byte entries. */
74 77
struct jit_jump_entry {
test/jit_vector.c added +173 -0
1 +
/* Differential checks of hosted vector translation and interpreter fallback. */
2 +
#include <assert.h>
3 +
#include <stdio.h>
4 +
#include <stdlib.h>
5 +
6 +
#include "jit.h"
7 +
#include "vector.h"
8 +
9 +
/* Test code lies outside the accessible data region. */
10 +
#define DATA 4096
11 +
#define CODE 8192
12 +
13 +
/* Transfer one interpreter element with the hosted bounds contract. */
14 +
static bool transfer(void *ctx, u64 addr, u32 size, bool store, u64 *value) {
15 +
    u8 *mem = ctx;
16 +
    if (addr > DATA || size > DATA - addr)
17 +
        return false;
18 +
    if (!store)
19 +
        *value = 0;
20 +
    for (u32 i = 0; i < size; i++) {
21 +
        if (store)
22 +
            mem[addr + i] = *value >> (i * 8);
23 +
        else
24 +
            *value |= (u64)mem[addr + i] << (i * 8);
25 +
    }
26 +
    return true;
27 +
}
28 +
29 +
/* A reproducible stream of operands and initial register bytes. */
30 +
static u32 random_word(void) {
31 +
    static u32 state  = 0x25ab64d1;
32 +
    state            ^= state << 13;
33 +
    state            ^= state >> 17;
34 +
    state            ^= state << 5;
35 +
    return state;
36 +
}
37 +
38 +
/* Encode one unmasked vector operation. */
39 +
static u32 alu(u32 fn, u32 form, u32 d, u32 a, u32 b) {
40 +
    return fn << 26 | 1u << 25 | a << 20 | b << 15 | form << 12 | d << 7 | 0x57;
41 +
}
42 +
43 +
/* Execute the scalar address update used before vector memory operations. */
44 +
static enum vector_result step(
45 +
    struct vector_state *v, u64 x[32], u32 ins, u8 *mem
46 +
) {
47 +
    if ((ins & 0x707f) == 0x13) {
48 +
        u32 rd = (ins >> 7) & 31, rs = (ins >> 15) & 31;
49 +
        if (rd)
50 +
            x[rd] = x[rs] + (i64)((i32)ins >> 20);
51 +
        return VECTOR_OK;
52 +
    }
53 +
    return vector_execute(v, x, ins, transfer, mem);
54 +
}
55 +
56 +
/* Compare all state, memory, faults and instruction accounting for one block.
57 +
 */
58 +
static void check(
59 +
    const u32 *code, u32 count, u32 type, u32 vl, u32 start, u64 base
60 +
) {
61 +
    static struct jit_state jit;
62 +
    u8                      actual[16384] = { 0 }, expected[16384] = { 0 };
63 +
    struct vector_state     a     = { 0 }, b;
64 +
    u64                     x[32] = { 0 }, y[32] = { 0 };
65 +
    for (u32 i = 0; i < DATA; i++)
66 +
        actual[i] = expected[i] = random_word();
67 +
    for (u32 i = 0; i < sizeof(a.reg); i++)
68 +
        ((u8 *)a.reg)[i] = random_word();
69 +
    /* Gather indexes include the last valid index and out-of-range indexes. */
70 +
    u32 indexes[8] = { 3, 15, 16, 0xffffffff, 2, 1, 0, 4 };
71 +
    memcpy(a.reg[24], indexes, sizeof(indexes));
72 +
    u32 offsets[8] = { 0, 4, 8, 12, 16, 20, 24, 28 };
73 +
    memcpy(a.reg[26], offsets, sizeof(offsets));
74 +
    a.vtype  = type;
75 +
    a.vl     = vl;
76 +
    a.vstart = start;
77 +
    b        = a;
78 +
    x[10] = y[10] = base;
79 +
    memcpy(actual + CODE, code, count * 4);
80 +
    u32 ret = 0x00008067;
81 +
    memcpy(actual + CODE + count * 4, &ret, 4);
82 +
    assert(jit_init(&jit));
83 +
    jit.io.vector      = &a;
84 +
    jit.io.memory_size = DATA;
85 +
    u32  pc            = CODE;
86 +
    i64  budget        = 100000;
87 +
    bool failed        = false;
88 +
    for (u32 i = 0; i < count + 2; i++) {
89 +
        struct jit_block *block =
90 +
            jit_get_block(&jit, pc, actual, CODE, (count + 1) * 4);
91 +
        assert(block);
92 +
        int reason = jit_run(&jit, block, x, actual, budget);
93 +
        budget     = jit.io.budget;
94 +
        pc         = jit.io.pc;
95 +
        if (reason == JIT_EXIT_RET)
96 +
            break;
97 +
        if (reason == JIT_EXIT_FAULT) {
98 +
            u32 ins;
99 +
            memcpy(&ins, actual + pc, 4);
100 +
            enum vector_result result = step(&a, x, ins, actual);
101 +
            if (result != VECTOR_OK) {
102 +
                failed = true;
103 +
                break;
104 +
            }
105 +
            pc += 4;
106 +
        }
107 +
    }
108 +
    bool ref_failed = false;
109 +
    u32  used       = 0;
110 +
    for (; used < count; used++) {
111 +
        if (step(&b, y, code[used], expected) != VECTOR_OK) {
112 +
            ref_failed = true;
113 +
            used++;
114 +
            break;
115 +
        }
116 +
    }
117 +
    if (!ref_failed)
118 +
        used++;
119 +
    assert(failed == ref_failed);
120 +
    assert(budget == 100000 - used);
121 +
    assert(memcmp(&a, &b, sizeof(a)) == 0);
122 +
    assert(memcmp(x, y, sizeof(x)) == 0);
123 +
    assert(memcmp(actual, expected, DATA) == 0);
124 +
    if ((count == 8 || (count == 7 && base == 128)) &&
125 +
        (type & ~0xc0u) == 0x11 && vl == 4 && start == 0)
126 +
        assert(jit.block_count == 1);
127 +
    jit_destroy(&jit);
128 +
}
129 +
130 +
int main(void) {
131 +
    u32 ops[] = {
132 +
        alu(0, 0, 8, 10, 12),   alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0),
133 +
        alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31),   alu(9, 3, 8, 8, 3),
134 +
        alu(12, 0, 16, 14, 24), alu(0, 0, 8, 8, 16),
135 +
    };
136 +
    u32 load   = 1u << 25 | 10 << 15 | 6 << 12 | 8 << 7 | 7;
137 +
    u32 store  = (load & ~127u) | 39;
138 +
    u32 gather = load | 1u << 26 | 26 << 20;
139 +
    u32 mem[]  = { ops[0], load, ops[1], gather, ops[2], store, ops[3] };
140 +
    for (u32 i = 0; i < 200; i++) {
141 +
        check(ops, 8, 0xd1, 4, 0, 128);
142 +
        check(mem, 7, 0xd1, 4, 0, 128);
143 +
    }
144 +
    for (u32 vl = 0; vl <= 16; vl++) {
145 +
        check(ops, 8, 0xd1, vl, 0, 128);
146 +
        check(mem, 7, 0xd1, vl, 0, 128);
147 +
    }
148 +
    for (u32 start = 1; start <= 4; start++)
149 +
        check(mem, 7, 0xd1, 4, start, 128);
150 +
    for (u32 type = 0x11; type <= 0xd1; type += 0x40)
151 +
        check(mem, 7, type, 4, 0, 128);
152 +
    u32 mixed[8] = { 0x00450513 };
153 +
    memcpy(mixed + 1, mem, sizeof(mem));
154 +
    check(mixed, 8, 0xd1, 4, 0, 128);
155 +
    check(mixed, 8, 0xd1, 4, 1, 128);
156 +
    check(mixed, 8, 0x10, 4, 0, 128);
157 +
    check(mixed, 8, 0x12, 4, 0, 128);
158 +
    check(mixed, 8, 0x80000000, 4, 0, 128);
159 +
    u64 bounds[] = { DATA - 16, DATA - 15, DATA - 4, DATA - 3,
160 +
                     DATA,      DATA + 1,  ~0ULL - 7 };
161 +
    for (u32 i = 0; i < sizeof(bounds) / sizeof(bounds[0]); i++) {
162 +
        check(mem, 7, 0xd1, 4, 0, bounds[i]);
163 +
        check(&gather, 1, 0xd1, 4, 0, bounds[i]);
164 +
        check(&store, 1, 0xd1, 4, 0, bounds[i]);
165 +
    }
166 +
    u32 invalid[] = { alu(0, 0, 9, 10, 12),
167 +
                      alu(12, 0, 8, 8, 24),
168 +
                      ops[0] & ~(1u << 25) };
169 +
    for (u32 i = 0; i < 3; i++)
170 +
        check(invalid + i, 1, 0xd1, 4, 0, 128);
171 +
    puts("JIT vector differential checks passed");
172 +
    return 0;
173 +
}