jit: Translate four-lane integer vector operations
137a6e3cfcae862c914862e6821909081b800906ff2fe0563422cc985346ff23
Guard e32,m2 blocks with vl=4 and vstart=0. Emit native integer arithmetic, bitwise operations, rotates and gathers, with checked unit-stride transfers and indexed loads. Return unsupported states and faulting accesses to the interpreter with accurate instruction counts. Add sanitizer-backed differential checks for register state, memory, faults and instruction accounting. Include the JIT checks in test-vector. Assisted-by: Codex:gpt-6 Change-Id: I1231c0279f09a03a5446f9024d5125460eb6852d
1 parent
117807f8
Makefile
+11 -1
| 51 | 51 | ||
| 52 | 52 | bin/vector-machine-test: test/vector_machine.c vector.c vector.h machine_cpu.c machine_csr.c machine_devices.c machine.c machine_fdt.c machine_internal.h machine.h types.h bitmanip.h muldiv.h |
|
| 53 | 53 | @mkdir -p bin |
|
| 54 | 54 | @$(CC) $(CPPFLAGS) -std=c99 -Wall -Wextra -g -fsanitize=address,undefined test/vector_machine.c vector.c machine_cpu.c machine_csr.c machine_devices.c machine.c machine_fdt.c -o $@ |
|
| 55 | 55 | ||
| 56 | - | test-vector: bin/vector-test bin/vector-machine-test |
|
| 56 | + | test-vector: bin/vector-test bin/vector-machine-test test-jit |
|
| 57 | 57 | @ASAN_OPTIONS=detect_leaks=0 timeout 30 bin/vector-test |
|
| 58 | 58 | @ASAN_OPTIONS=detect_leaks=0 timeout 30 bin/vector-machine-test |
|
| 59 | 59 | ||
| 60 | 60 | .PHONY: test-vector |
|
| 61 | + | ||
| 62 | + | # Compare hosted vector translation with architectural interpreter state. |
|
| 63 | + | bin/jit-vector-test: test/jit_vector.c jit.c jit.h vector.c vector.h riscv.c riscv.h riscv/debug.c riscv/debug.h bitmanip.h color.h types.h |
|
| 64 | + | @mkdir -p bin |
|
| 65 | + | @$(CC) $(CPPFLAGS) -std=c99 -O2 -g -Wall -Wextra -fsanitize=address,undefined test/jit_vector.c jit.c vector.c riscv.c riscv/debug.c -o $@ |
|
| 66 | + | ||
| 67 | + | test-jit: bin/jit-vector-test |
|
| 68 | + | @ASAN_OPTIONS=detect_leaks=0 timeout 60 $< |
|
| 69 | + | ||
| 70 | + | .PHONY: test-jit |
emulator.c
+2 -0
| 2569 | 2569 | !g_opts.trace_print_instructions && !g_opts.watch_enabled) { |
|
| 2570 | 2570 | use_jit = jit_init(&jit); |
|
| 2571 | 2571 | } |
|
| 2572 | 2572 | ||
| 2573 | 2573 | if (use_jit) { |
|
| 2574 | + | jit.io.vector = &cpu.vector; |
|
| 2575 | + | jit.io.memory_size = guest_sizes().memory; |
|
| 2574 | 2576 | /* ---- JIT execution loop ---- */ |
|
| 2575 | 2577 | u8 *patch = NULL; /* Direct jump to link to the next block. */ |
|
| 2576 | 2578 | ||
| 2577 | 2579 | while (cpu.running && steps < max_steps) { |
|
| 2578 | 2580 | struct jit_block *block = jit_get_block( |
jit.c
+276 -0
| 11 | 11 | * r12 = ®s[0] (guest register file) |
|
| 12 | 12 | * r13 = memory base |
|
| 13 | 13 | * r14 = &jit->io (exit information; pc is its first field) |
|
| 14 | 14 | * r15 = budget (guest instructions left, signed) |
|
| 15 | 15 | * - Guest registers are loaded/stored on demand from/to the array. |
|
| 16 | + | * - Vector blocks guard the four-word e32,m2 shape before any side effect. |
|
| 17 | + | * Other shapes return to the interpreter. |
|
| 16 | 18 | * - Each block starts with "if budget <= 0 exit at this PC; budget -= |
|
| 17 | 19 | * insn_count". Every exit is the block's last instruction, so a block |
|
| 18 | 20 | * always accounts exactly the instructions it executes. |
|
| 19 | 21 | * - Direct jumps (jal, both sides of a branch, block split) are a jmp |
|
| 20 | 22 | * rel32 that first falls into a stub returning JIT_EXIT_CHAIN. C then |
| 32 | 34 | ||
| 33 | 35 | #include "bitmanip.h" |
|
| 34 | 36 | #include "jit.h" |
|
| 35 | 37 | #include "riscv.h" |
|
| 36 | 38 | #include "types.h" |
|
| 39 | + | #include "vector.h" |
|
| 37 | 40 | ||
| 38 | 41 | /* ---------- x86-64 code emitter helpers --------------------------------- */ |
|
| 39 | 42 | ||
| 40 | 43 | /* A small code buffer that we fill up, then copy into the cache. */ |
|
| 41 | 44 | struct emitter { |
|
| 42 | 45 | u8 *buf; /* Start of buffer. */ |
|
| 43 | 46 | u32 pos; /* Current write position. */ |
|
| 44 | 47 | u32 capacity; /* Maximum bytes. */ |
|
| 45 | 48 | bool overflow; /* Set if we ran out of space. */ |
|
| 49 | + | bool vector; /* Block uses four-lane 32-bit SIMD. */ |
|
| 50 | + | u32 insns; /* Instructions through the current operation. */ |
|
| 51 | + | /* Each indexed load can fail three checks for each of four lanes. */ |
|
| 52 | + | u32 refunds[JIT_MAX_BLOCK_INSNS * 12]; /* Budget patch offsets. */ |
|
| 53 | + | u32 consumed[JIT_MAX_BLOCK_INSNS * 12]; /* Instructions at each patch. */ |
|
| 54 | + | u32 refunds_len; /* Number of guarded exits. */ |
|
| 46 | 55 | ||
| 47 | 56 | struct jit_state *jit; /* For the exit stub and jump cache addresses. */ |
|
| 48 | 57 | u8 *dest; /* Address the buffer will be copied to. */ |
|
| 49 | 58 | }; |
|
| 50 | 59 |
| 1075 | 1084 | break; |
|
| 1076 | 1085 | } |
|
| 1077 | 1086 | emit_store_rd(e, ins.r.rd); |
|
| 1078 | 1087 | } |
|
| 1079 | 1088 | ||
| 1089 | + | /* ---------- Vector translation ----------------------------------------- */ |
|
| 1090 | + | ||
| 1091 | + | /* Return a failed memory operation to the interpreter before its side effects. |
|
| 1092 | + | */ |
|
| 1093 | + | static void emit_vector_fault(struct emitter *e, u32 pc, u8 condition) { |
|
| 1094 | + | u32 bad = e->pos; |
|
| 1095 | + | emit_u8(e, 0x0f); |
|
| 1096 | + | emit_u8(e, condition); |
|
| 1097 | + | emit_u32(e, 0); |
|
| 1098 | + | u32 ok = emit_jmp_placeholder(e); |
|
| 1099 | + | patch_jcc(e, bad, e->pos); |
|
| 1100 | + | emit_u8(e, 0x49); |
|
| 1101 | + | emit_u8(e, 0x81); |
|
| 1102 | + | emit_u8(e, modrm(3, 0, HBUDGET)); |
|
| 1103 | + | e->refunds[e->refunds_len] = e->pos; |
|
| 1104 | + | e->consumed[e->refunds_len++] = e->insns; |
|
| 1105 | + | emit_u32(e, 0); |
|
| 1106 | + | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 1107 | + | patch_jmp(e, ok, e->pos); |
|
| 1108 | + | } |
|
| 1109 | + | ||
| 1110 | + | /* Load the architectural vector state into rax. */ |
|
| 1111 | + | static void emit_vector_base(struct emitter *e) { |
|
| 1112 | + | emit_u8(e, 0x49); |
|
| 1113 | + | emit_u8(e, 0x8b); |
|
| 1114 | + | emit_u8(e, modrm(1, X_RAX, HIO)); |
|
| 1115 | + | emit_u8(e, offsetof(struct jit_io, vector)); |
|
| 1116 | + | } |
|
| 1117 | + | ||
| 1118 | + | /* Form one indexed guest address, with overflow checked before any transfer. */ |
|
| 1119 | + | static void emit_vector_address(struct emitter *e, u32 ins, u32 lane, u32 pc) { |
|
| 1120 | + | u32 base = (ins >> 15) & 31, index = (ins >> 20) & 31; |
|
| 1121 | + | emit_vector_base(e); |
|
| 1122 | + | emit_u8(e, 0x8b); |
|
| 1123 | + | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1124 | + | emit_u32(e, index * VECTOR_BYTES + lane * 4); |
|
| 1125 | + | emit_load_guest(e, X_RAX, base); |
|
| 1126 | + | emit_u8(e, 0x48); |
|
| 1127 | + | emit_u8(e, 0x01); |
|
| 1128 | + | emit_u8(e, 0xd0); |
|
| 1129 | + | emit_vector_fault(e, pc, 0x82); /* jc */ |
|
| 1130 | + | } |
|
| 1131 | + | ||
| 1132 | + | /* Check a complete transfer against the hosted memory bound. */ |
|
| 1133 | + | static void emit_vector_bound(struct emitter *e, u32 size, u32 pc) { |
|
| 1134 | + | emit_u8(e, 0x49); |
|
| 1135 | + | emit_u8(e, 0x8b); |
|
| 1136 | + | emit_u8(e, modrm(1, X_RCX, HIO)); |
|
| 1137 | + | emit_u8(e, offsetof(struct jit_io, memory_size)); |
|
| 1138 | + | emit_u8(e, 0x48); |
|
| 1139 | + | emit_u8(e, 0x83); |
|
| 1140 | + | emit_u8(e, 0xe9); |
|
| 1141 | + | emit_u8(e, size); |
|
| 1142 | + | emit_vector_fault(e, pc, 0x82); |
|
| 1143 | + | emit_u8(e, 0x48); |
|
| 1144 | + | emit_u8(e, 0x39); |
|
| 1145 | + | emit_u8(e, 0xc8); |
|
| 1146 | + | emit_vector_fault(e, pc, 0x87); /* ja */ |
|
| 1147 | + | } |
|
| 1148 | + | ||
| 1149 | + | /* Emit an SSE instruction on two XMM registers. */ |
|
| 1150 | + | static void emit_xmm(struct emitter *e, u8 opcode, u32 d, u32 src) { |
|
| 1151 | + | emit_u8(e, 0x66); |
|
| 1152 | + | emit_u8(e, 0x0f); |
|
| 1153 | + | emit_u8(e, opcode); |
|
| 1154 | + | emit_u8(e, modrm(3, d, src)); |
|
| 1155 | + | } |
|
| 1156 | + | ||
| 1157 | + | /* Transfer four words between an XMM register and architectural state in rax. |
|
| 1158 | + | */ |
|
| 1159 | + | static void emit_xmm_memory(struct emitter *e, u32 xmm, u32 reg, bool store) { |
|
| 1160 | + | emit_u8(e, 0xf3); |
|
| 1161 | + | emit_u8(e, 0x0f); |
|
| 1162 | + | emit_u8(e, store ? 0x7f : 0x6f); |
|
| 1163 | + | emit_u8(e, modrm(2, xmm, X_RAX)); |
|
| 1164 | + | emit_u32(e, reg * VECTOR_BYTES); |
|
| 1165 | + | } |
|
| 1166 | + | ||
| 1167 | + | /* Emit checked unmasked word loads and stores for one register group. */ |
|
| 1168 | + | static bool emit_vector_memory(struct emitter *e, u32 ins, u32 pc) { |
|
| 1169 | + | u32 op = ins & 127, reg = (ins >> 7) & 31, index = (ins >> 20) & 31; |
|
| 1170 | + | u32 mode = (ins >> 26) & 3; |
|
| 1171 | + | bool store = op == 39; |
|
| 1172 | + | if ((op != 7 && !store) || ((ins >> 12) & 7) != 6 || !(ins & (1u << 25)) || |
|
| 1173 | + | (ins >> 29) || (ins & (1u << 28)) || (reg & 1) || |
|
| 1174 | + | !((mode == 0 && index == 0) || (mode == 1 && !store && !(index & 1)))) |
|
| 1175 | + | return false; |
|
| 1176 | + | if (mode == 1 && !store && reg == index) |
|
| 1177 | + | return false; |
|
| 1178 | + | e->vector = true; |
|
| 1179 | + | if (mode == 0) { |
|
| 1180 | + | emit_load_guest(e, X_RAX, (ins >> 15) & 31); |
|
| 1181 | + | emit_vector_bound(e, 16, pc); |
|
| 1182 | + | if (store) { |
|
| 1183 | + | emit_vector_base(e); |
|
| 1184 | + | emit_xmm_memory(e, 0, reg, false); |
|
| 1185 | + | emit_load_guest(e, X_RAX, (ins >> 15) & 31); |
|
| 1186 | + | } |
|
| 1187 | + | emit_add_rax_r13(e); |
|
| 1188 | + | emit_u8(e, 0xf3); |
|
| 1189 | + | emit_u8(e, 0x0f); |
|
| 1190 | + | emit_u8(e, store ? 0x7f : 0x6f); |
|
| 1191 | + | emit_u8(e, 0x00); |
|
| 1192 | + | if (!store) { |
|
| 1193 | + | emit_vector_base(e); |
|
| 1194 | + | emit_xmm_memory(e, 0, reg, true); |
|
| 1195 | + | } |
|
| 1196 | + | } else { |
|
| 1197 | + | /* Preflight every address so an exceptional access has no partial |
|
| 1198 | + | * writes. */ |
|
| 1199 | + | for (u32 i = 0; i < 4; i++) { |
|
| 1200 | + | emit_vector_address(e, ins, i, pc); |
|
| 1201 | + | emit_vector_bound(e, 4, pc); |
|
| 1202 | + | } |
|
| 1203 | + | for (u32 i = 0; i < 4; i++) { |
|
| 1204 | + | emit_vector_base(e); |
|
| 1205 | + | emit_u8(e, 0x8b); |
|
| 1206 | + | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1207 | + | emit_u32(e, index * VECTOR_BYTES + i * 4); |
|
| 1208 | + | emit_load_guest(e, X_RAX, (ins >> 15) & 31); |
|
| 1209 | + | emit_u8(e, 0x48); |
|
| 1210 | + | emit_u8(e, 0x01); |
|
| 1211 | + | emit_u8(e, 0xd0); |
|
| 1212 | + | emit_add_rax_r13(e); |
|
| 1213 | + | emit_u8(e, 0x8b); |
|
| 1214 | + | emit_u8(e, 0x10); /* mov edx, [rax] */ |
|
| 1215 | + | emit_vector_base(e); |
|
| 1216 | + | emit_u8(e, 0x89); |
|
| 1217 | + | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1218 | + | emit_u32(e, reg * VECTOR_BYTES + i * 4); |
|
| 1219 | + | } |
|
| 1220 | + | } |
|
| 1221 | + | return true; |
|
| 1222 | + | } |
|
| 1223 | + | ||
| 1224 | + | /* Gather four words from a validated, non-overlapping register group. */ |
|
| 1225 | + | static void emit_vector_gather(struct emitter *e, u32 d, u32 a, u32 b) { |
|
| 1226 | + | emit_vector_base(e); |
|
| 1227 | + | for (u32 i = 0; i < 4; i++) { |
|
| 1228 | + | emit_u8(e, 0x8b); |
|
| 1229 | + | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1230 | + | emit_u32(e, b * VECTOR_BYTES + i * 4); |
|
| 1231 | + | emit_u8(e, 0x31); |
|
| 1232 | + | emit_u8(e, 0xc9); /* xor ecx, ecx */ |
|
| 1233 | + | emit_u8(e, 0x83); |
|
| 1234 | + | emit_u8(e, 0xfa); |
|
| 1235 | + | emit_u8(e, 16); |
|
| 1236 | + | u32 outside = e->pos; |
|
| 1237 | + | emit_u8(e, 0x0f); |
|
| 1238 | + | emit_u8(e, 0x83); |
|
| 1239 | + | emit_u32(e, 0); |
|
| 1240 | + | emit_u8(e, 0x8b); |
|
| 1241 | + | emit_u8(e, modrm(2, X_RCX, X_RSP)); |
|
| 1242 | + | emit_u8(e, sib(2, X_RDX, X_RAX)); |
|
| 1243 | + | emit_u32(e, a * VECTOR_BYTES); |
|
| 1244 | + | patch_jcc(e, outside, e->pos); |
|
| 1245 | + | emit_u8(e, 0x89); |
|
| 1246 | + | emit_u8(e, modrm(2, X_RCX, X_RAX)); |
|
| 1247 | + | emit_u32(e, d * VECTOR_BYTES + i * 4); |
|
| 1248 | + | } |
|
| 1249 | + | } |
|
| 1250 | + | ||
| 1251 | + | /* Emit unmasked four-word integer operations under the block's state guard. */ |
|
| 1252 | + | static bool emit_vector(struct emitter *e, u32 ins) { |
|
| 1253 | + | u32 fn = ins >> 26, form = (ins >> 12) & 7; |
|
| 1254 | + | u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31; |
|
| 1255 | + | if (!(ins & (1u << 25)) || (d & 1) || (a & 1)) |
|
| 1256 | + | return false; |
|
| 1257 | + | if (!((form == 0 && !(b & 1) && |
|
| 1258 | + | (fn == 0 || fn == 9 || fn == 11 || |
|
| 1259 | + | (fn == 12 && d != a && d != b))) || |
|
| 1260 | + | (form == 3 && |
|
| 1261 | + | (fn == 0 || fn == 9 || fn == 11 || fn == 20 || fn == 21)))) |
|
| 1262 | + | return false; |
|
| 1263 | + | e->vector = true; |
|
| 1264 | + | if (form == 0 && fn == 12) { |
|
| 1265 | + | emit_vector_gather(e, d, a, b); |
|
| 1266 | + | return true; |
|
| 1267 | + | } |
|
| 1268 | + | emit_vector_base(e); |
|
| 1269 | + | emit_xmm_memory(e, 0, a, false); |
|
| 1270 | + | if (form == 3 && (fn == 20 || fn == 21)) { |
|
| 1271 | + | emit_xmm(e, 0x6f, 1, 0); |
|
| 1272 | + | emit_xmm(e, 0x72, 2, 0); |
|
| 1273 | + | emit_u8(e, b); |
|
| 1274 | + | emit_xmm(e, 0x72, 6, 1); |
|
| 1275 | + | emit_u8(e, 32 - b); |
|
| 1276 | + | emit_xmm(e, 0xeb, 0, 1); |
|
| 1277 | + | } else { |
|
| 1278 | + | if (form == 0) { |
|
| 1279 | + | emit_xmm_memory(e, 1, b, false); |
|
| 1280 | + | } else { |
|
| 1281 | + | emit_u8(e, 0xb9); |
|
| 1282 | + | emit_u32(e, (u32)((i32)(b << 27) >> 27)); |
|
| 1283 | + | emit_xmm(e, 0x6e, 1, X_RCX); |
|
| 1284 | + | emit_xmm(e, 0x70, 1, 1); |
|
| 1285 | + | emit_u8(e, 0); |
|
| 1286 | + | } |
|
| 1287 | + | emit_xmm(e, fn == 0 ? 0xfe : fn == 9 ? 0xdb : 0xef, 0, 1); |
|
| 1288 | + | } |
|
| 1289 | + | emit_xmm_memory(e, 0, d, true); |
|
| 1290 | + | return true; |
|
| 1291 | + | } |
|
| 1292 | + | ||
| 1293 | + | /* Guard the vector shape before any instruction in the block has side effects. |
|
| 1294 | + | */ |
|
| 1295 | + | static void emit_vector_guard(struct emitter *e, u32 pc) { |
|
| 1296 | + | emit_u8(e, 0x49); |
|
| 1297 | + | emit_u8(e, 0x8b); |
|
| 1298 | + | emit_u8(e, modrm(1, X_RAX, HIO)); |
|
| 1299 | + | emit_u8(e, offsetof(struct jit_io, vector)); |
|
| 1300 | + | u32 bad[3]; |
|
| 1301 | + | const u32 offsets[3] = { offsetof(struct vector_state, vtype), |
|
| 1302 | + | offsetof(struct vector_state, vl), |
|
| 1303 | + | offsetof(struct vector_state, vstart) }; |
|
| 1304 | + | const u32 values[3] = { 0x11, 4, 0 }; |
|
| 1305 | + | for (u32 i = 0; i < 3; i++) { |
|
| 1306 | + | if (i == 0) { |
|
| 1307 | + | /* Tail and mask policies do not change the four active lanes. */ |
|
| 1308 | + | emit_u8(e, 0x48); |
|
| 1309 | + | emit_u8(e, 0x8b); |
|
| 1310 | + | emit_u8(e, modrm(2, X_RCX, X_RAX)); |
|
| 1311 | + | emit_u32(e, offsets[i]); |
|
| 1312 | + | emit_u8(e, 0x48); |
|
| 1313 | + | emit_u8(e, 0x81); |
|
| 1314 | + | emit_u8(e, 0xe1); |
|
| 1315 | + | emit_u32(e, ~0xc0u); |
|
| 1316 | + | emit_u8(e, 0x48); |
|
| 1317 | + | emit_u8(e, 0x81); |
|
| 1318 | + | emit_u8(e, 0xf9); |
|
| 1319 | + | } else { |
|
| 1320 | + | emit_u8(e, 0x48); |
|
| 1321 | + | emit_u8(e, 0x81); |
|
| 1322 | + | emit_u8(e, modrm(2, 7, X_RAX)); |
|
| 1323 | + | emit_u32(e, offsets[i]); |
|
| 1324 | + | } |
|
| 1325 | + | emit_u32(e, values[i]); |
|
| 1326 | + | bad[i] = emit_jne_placeholder(e); |
|
| 1327 | + | } |
|
| 1328 | + | u32 ok = emit_jmp_placeholder(e); |
|
| 1329 | + | for (u32 i = 0; i < 3; i++) |
|
| 1330 | + | patch_jcc(e, bad[i], e->pos); |
|
| 1331 | + | emit_u8(e, 0x49); |
|
| 1332 | + | emit_u8(e, 0x83); |
|
| 1333 | + | emit_u8(e, modrm(3, 5, HBUDGET)); |
|
| 1334 | + | emit_u8(e, 1); |
|
| 1335 | + | emit_block_exit(e, JIT_EXIT_FAULT, pc); |
|
| 1336 | + | patch_jmp(e, ok, e->pos); |
|
| 1337 | + | } |
|
| 1338 | + | ||
| 1080 | 1339 | /* ---------- Translate one RV64I instruction ----------------------------- */ |
|
| 1081 | 1340 | ||
| 1082 | 1341 | /* |
|
| 1083 | 1342 | * Returns: true if the instruction ends the basic block (branch/jump/ecall), |
|
| 1084 | 1343 | * false if execution should continue to the next instruction. |
| 1091 | 1350 | if (bm != BM_NONE) { |
|
| 1092 | 1351 | emit_bitmanip(e, ins, bm); |
|
| 1093 | 1352 | return false; |
|
| 1094 | 1353 | } |
|
| 1095 | 1354 | ||
| 1355 | + | if ((opcode == 7 || opcode == 39) && emit_vector_memory(e, ins.raw, pc)) |
|
| 1356 | + | return false; |
|
| 1357 | + | if (opcode == 0x57 && emit_vector(e, ins.raw)) |
|
| 1358 | + | return false; |
|
| 1359 | + | ||
| 1096 | 1360 | switch (opcode) { |
|
| 1097 | 1361 | ||
| 1098 | 1362 | case OP_LUI: |
|
| 1099 | 1363 | if (ins.u.rd != 0) { |
|
| 1100 | 1364 | emit_mov_imm32_sx(e, HTMP1, (i32)(ins.u.imm_31_12 << 12)); |
| 1561 | 1825 | emit_u8(&em, 0x4D); |
|
| 1562 | 1826 | emit_u8(&em, 0x85); |
|
| 1563 | 1827 | emit_u8(&em, modrm(3, HBUDGET, HBUDGET)); /* test r15, r15 */ |
|
| 1564 | 1828 | u32 jle_pos = emit_jle_placeholder(&em); |
|
| 1565 | 1829 | ||
| 1830 | + | u32 skip_vector = emit_jmp_placeholder(&em); |
|
| 1831 | + | u32 vector_guard = em.pos; |
|
| 1832 | + | emit_vector_guard(&em, guest_pc); |
|
| 1833 | + | u32 vector_body = em.pos; |
|
| 1834 | + | ||
| 1566 | 1835 | /* sub r15, insn_count -- imm32 is patched once the count is known. */ |
|
| 1567 | 1836 | emit_u8(&em, 0x49); |
|
| 1568 | 1837 | emit_u8(&em, 0x81); |
|
| 1569 | 1838 | emit_u8(&em, modrm(3, 5, HBUDGET)); |
|
| 1570 | 1839 | u32 count_pos = em.pos; |
| 1580 | 1849 | break; |
|
| 1581 | 1850 | } |
|
| 1582 | 1851 | instr_t ins; |
|
| 1583 | 1852 | memcpy(&ins, &mem[pc], sizeof(instr_t)); |
|
| 1584 | 1853 | insn_count++; |
|
| 1854 | + | em.insns = insn_count; |
|
| 1585 | 1855 | ||
| 1586 | 1856 | bool ends_block = translate_insn(&em, ins, pc); |
|
| 1587 | 1857 | pc += INSTR_SIZE; |
|
| 1588 | 1858 | ||
| 1589 | 1859 | if (ends_block) |
| 1601 | 1871 | ||
| 1602 | 1872 | /* .exhausted: out of budget, leave before running anything. */ |
|
| 1603 | 1873 | patch_jcc(&em, jle_pos, em.pos); |
|
| 1604 | 1874 | emit_block_exit(&em, JIT_EXIT_BRANCH, guest_pc); |
|
| 1605 | 1875 | ||
| 1876 | + | patch_jmp(&em, skip_vector, em.vector ? vector_guard : vector_body); |
|
| 1606 | 1877 | if (em.overflow) |
|
| 1607 | 1878 | return NULL; |
|
| 1608 | 1879 | for (u32 i = 0; i < 4; i++) |
|
| 1609 | 1880 | em.buf[count_pos + i] = (u8)(insn_count >> (8 * i)); |
|
| 1610 | 1881 | ||
| 1882 | + | for (u32 i = 0; i < em.refunds_len; i++) { |
|
| 1883 | + | u32 refund = insn_count - em.consumed[i]; |
|
| 1884 | + | for (u32 j = 0; j < 4; j++) |
|
| 1885 | + | em.buf[em.refunds[i] + j] = (u8)(refund >> (8 * j)); |
|
| 1886 | + | } |
|
| 1611 | 1887 | u32 code_size = em.pos; |
|
| 1612 | 1888 | if (jit->code_cache_used + code_size > JIT_CODE_CACHE_SIZE) |
|
| 1613 | 1889 | return NULL; |
|
| 1614 | 1890 | ||
| 1615 | 1891 | u8 *dest = jit->code_cache + jit->code_cache_used; |
jit.h
+7 -4
| 14 | 14 | #include "riscv.h" |
|
| 15 | 15 | #include "types.h" |
|
| 16 | 16 | ||
| 17 | 17 | /* Forward declarations. */ |
|
| 18 | 18 | struct cpu; |
|
| 19 | + | struct vector_state; |
|
| 19 | 20 | ||
| 20 | 21 | /* --------------------------------------------------------------------------- |
|
| 21 | 22 | * Tuning constants |
|
| 22 | 23 | * ------------------------------------------------------------------------- */ |
|
| 23 | 24 | ||
| 24 | 25 | /* Size of the JIT code cache (16 MB). */ |
|
| 25 | 26 | #define JIT_CODE_CACHE_SIZE (16 * 1024 * 1024) |
|
| 26 | 27 | ||
| 27 | 28 | /* Maximum number of x86-64 bytes emitted per guest instruction. |
|
| 28 | 29 | * Conservative upper bound -- most instructions compile to <30 bytes. */ |
|
| 29 | - | #define JIT_MAX_INSN_BYTES 128 |
|
| 30 | + | #define JIT_MAX_INSN_BYTES 1024 |
|
| 30 | 31 | ||
| 31 | 32 | /* Maximum guest instructions in a single translated block. */ |
|
| 32 | 33 | #define JIT_MAX_BLOCK_INSNS 256 |
|
| 33 | 34 | ||
| 34 | 35 | /* Number of hash-table buckets for block lookup (must be power of 2). */ |
| 62 | 63 | }; |
|
| 63 | 64 | ||
| 64 | 65 | /* Exchange area between generated code and C. Generated code keeps a pointer |
|
| 65 | 66 | * to it in r14. */ |
|
| 66 | 67 | struct jit_io { |
|
| 67 | - | u32 pc; /* Next guest PC on exit. */ |
|
| 68 | - | i64 budget; /* Remaining guest instructions; in on entry, out on exit. */ |
|
| 69 | - | u8 *patch; /* On JIT_EXIT_CHAIN, the rel32 field of the jmp to patch. */ |
|
| 68 | + | u32 pc; /* Next guest PC on exit. */ |
|
| 69 | + | i64 budget; /* Remaining guest instructions. */ |
|
| 70 | + | u8 *patch; /* Chain jump rel32 field. */ |
|
| 71 | + | struct vector_state *vector; /* Architectural vector registers. */ |
|
| 72 | + | u64 memory_size; /* Accessible hosted memory bytes. */ |
|
| 70 | 73 | }; |
|
| 71 | 74 | ||
| 72 | 75 | /* One entry of the indirect jump cache. Generated code indexes the cache |
|
| 73 | 76 | * assuming 16-byte entries. */ |
|
| 74 | 77 | struct jit_jump_entry { |
test/jit_vector.c
added
+173 -0
| 1 | + | /* Differential checks of hosted vector translation and interpreter fallback. */ |
|
| 2 | + | #include <assert.h> |
|
| 3 | + | #include <stdio.h> |
|
| 4 | + | #include <stdlib.h> |
|
| 5 | + | ||
| 6 | + | #include "jit.h" |
|
| 7 | + | #include "vector.h" |
|
| 8 | + | ||
| 9 | + | /* Test code lies outside the accessible data region. */ |
|
| 10 | + | #define DATA 4096 |
|
| 11 | + | #define CODE 8192 |
|
| 12 | + | ||
| 13 | + | /* Transfer one interpreter element with the hosted bounds contract. */ |
|
| 14 | + | static bool transfer(void *ctx, u64 addr, u32 size, bool store, u64 *value) { |
|
| 15 | + | u8 *mem = ctx; |
|
| 16 | + | if (addr > DATA || size > DATA - addr) |
|
| 17 | + | return false; |
|
| 18 | + | if (!store) |
|
| 19 | + | *value = 0; |
|
| 20 | + | for (u32 i = 0; i < size; i++) { |
|
| 21 | + | if (store) |
|
| 22 | + | mem[addr + i] = *value >> (i * 8); |
|
| 23 | + | else |
|
| 24 | + | *value |= (u64)mem[addr + i] << (i * 8); |
|
| 25 | + | } |
|
| 26 | + | return true; |
|
| 27 | + | } |
|
| 28 | + | ||
| 29 | + | /* A reproducible stream of operands and initial register bytes. */ |
|
| 30 | + | static u32 random_word(void) { |
|
| 31 | + | static u32 state = 0x25ab64d1; |
|
| 32 | + | state ^= state << 13; |
|
| 33 | + | state ^= state >> 17; |
|
| 34 | + | state ^= state << 5; |
|
| 35 | + | return state; |
|
| 36 | + | } |
|
| 37 | + | ||
| 38 | + | /* Encode one unmasked vector operation. */ |
|
| 39 | + | static u32 alu(u32 fn, u32 form, u32 d, u32 a, u32 b) { |
|
| 40 | + | return fn << 26 | 1u << 25 | a << 20 | b << 15 | form << 12 | d << 7 | 0x57; |
|
| 41 | + | } |
|
| 42 | + | ||
| 43 | + | /* Execute the scalar address update used before vector memory operations. */ |
|
| 44 | + | static enum vector_result step( |
|
| 45 | + | struct vector_state *v, u64 x[32], u32 ins, u8 *mem |
|
| 46 | + | ) { |
|
| 47 | + | if ((ins & 0x707f) == 0x13) { |
|
| 48 | + | u32 rd = (ins >> 7) & 31, rs = (ins >> 15) & 31; |
|
| 49 | + | if (rd) |
|
| 50 | + | x[rd] = x[rs] + (i64)((i32)ins >> 20); |
|
| 51 | + | return VECTOR_OK; |
|
| 52 | + | } |
|
| 53 | + | return vector_execute(v, x, ins, transfer, mem); |
|
| 54 | + | } |
|
| 55 | + | ||
| 56 | + | /* Compare all state, memory, faults and instruction accounting for one block. |
|
| 57 | + | */ |
|
| 58 | + | static void check( |
|
| 59 | + | const u32 *code, u32 count, u32 type, u32 vl, u32 start, u64 base |
|
| 60 | + | ) { |
|
| 61 | + | static struct jit_state jit; |
|
| 62 | + | u8 actual[16384] = { 0 }, expected[16384] = { 0 }; |
|
| 63 | + | struct vector_state a = { 0 }, b; |
|
| 64 | + | u64 x[32] = { 0 }, y[32] = { 0 }; |
|
| 65 | + | for (u32 i = 0; i < DATA; i++) |
|
| 66 | + | actual[i] = expected[i] = random_word(); |
|
| 67 | + | for (u32 i = 0; i < sizeof(a.reg); i++) |
|
| 68 | + | ((u8 *)a.reg)[i] = random_word(); |
|
| 69 | + | /* Gather indexes include the last valid index and out-of-range indexes. */ |
|
| 70 | + | u32 indexes[8] = { 3, 15, 16, 0xffffffff, 2, 1, 0, 4 }; |
|
| 71 | + | memcpy(a.reg[24], indexes, sizeof(indexes)); |
|
| 72 | + | u32 offsets[8] = { 0, 4, 8, 12, 16, 20, 24, 28 }; |
|
| 73 | + | memcpy(a.reg[26], offsets, sizeof(offsets)); |
|
| 74 | + | a.vtype = type; |
|
| 75 | + | a.vl = vl; |
|
| 76 | + | a.vstart = start; |
|
| 77 | + | b = a; |
|
| 78 | + | x[10] = y[10] = base; |
|
| 79 | + | memcpy(actual + CODE, code, count * 4); |
|
| 80 | + | u32 ret = 0x00008067; |
|
| 81 | + | memcpy(actual + CODE + count * 4, &ret, 4); |
|
| 82 | + | assert(jit_init(&jit)); |
|
| 83 | + | jit.io.vector = &a; |
|
| 84 | + | jit.io.memory_size = DATA; |
|
| 85 | + | u32 pc = CODE; |
|
| 86 | + | i64 budget = 100000; |
|
| 87 | + | bool failed = false; |
|
| 88 | + | for (u32 i = 0; i < count + 2; i++) { |
|
| 89 | + | struct jit_block *block = |
|
| 90 | + | jit_get_block(&jit, pc, actual, CODE, (count + 1) * 4); |
|
| 91 | + | assert(block); |
|
| 92 | + | int reason = jit_run(&jit, block, x, actual, budget); |
|
| 93 | + | budget = jit.io.budget; |
|
| 94 | + | pc = jit.io.pc; |
|
| 95 | + | if (reason == JIT_EXIT_RET) |
|
| 96 | + | break; |
|
| 97 | + | if (reason == JIT_EXIT_FAULT) { |
|
| 98 | + | u32 ins; |
|
| 99 | + | memcpy(&ins, actual + pc, 4); |
|
| 100 | + | enum vector_result result = step(&a, x, ins, actual); |
|
| 101 | + | if (result != VECTOR_OK) { |
|
| 102 | + | failed = true; |
|
| 103 | + | break; |
|
| 104 | + | } |
|
| 105 | + | pc += 4; |
|
| 106 | + | } |
|
| 107 | + | } |
|
| 108 | + | bool ref_failed = false; |
|
| 109 | + | u32 used = 0; |
|
| 110 | + | for (; used < count; used++) { |
|
| 111 | + | if (step(&b, y, code[used], expected) != VECTOR_OK) { |
|
| 112 | + | ref_failed = true; |
|
| 113 | + | used++; |
|
| 114 | + | break; |
|
| 115 | + | } |
|
| 116 | + | } |
|
| 117 | + | if (!ref_failed) |
|
| 118 | + | used++; |
|
| 119 | + | assert(failed == ref_failed); |
|
| 120 | + | assert(budget == 100000 - used); |
|
| 121 | + | assert(memcmp(&a, &b, sizeof(a)) == 0); |
|
| 122 | + | assert(memcmp(x, y, sizeof(x)) == 0); |
|
| 123 | + | assert(memcmp(actual, expected, DATA) == 0); |
|
| 124 | + | if ((count == 8 || (count == 7 && base == 128)) && |
|
| 125 | + | (type & ~0xc0u) == 0x11 && vl == 4 && start == 0) |
|
| 126 | + | assert(jit.block_count == 1); |
|
| 127 | + | jit_destroy(&jit); |
|
| 128 | + | } |
|
| 129 | + | ||
| 130 | + | int main(void) { |
|
| 131 | + | u32 ops[] = { |
|
| 132 | + | alu(0, 0, 8, 10, 12), alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0), |
|
| 133 | + | alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31), alu(9, 3, 8, 8, 3), |
|
| 134 | + | alu(12, 0, 16, 14, 24), alu(0, 0, 8, 8, 16), |
|
| 135 | + | }; |
|
| 136 | + | u32 load = 1u << 25 | 10 << 15 | 6 << 12 | 8 << 7 | 7; |
|
| 137 | + | u32 store = (load & ~127u) | 39; |
|
| 138 | + | u32 gather = load | 1u << 26 | 26 << 20; |
|
| 139 | + | u32 mem[] = { ops[0], load, ops[1], gather, ops[2], store, ops[3] }; |
|
| 140 | + | for (u32 i = 0; i < 200; i++) { |
|
| 141 | + | check(ops, 8, 0xd1, 4, 0, 128); |
|
| 142 | + | check(mem, 7, 0xd1, 4, 0, 128); |
|
| 143 | + | } |
|
| 144 | + | for (u32 vl = 0; vl <= 16; vl++) { |
|
| 145 | + | check(ops, 8, 0xd1, vl, 0, 128); |
|
| 146 | + | check(mem, 7, 0xd1, vl, 0, 128); |
|
| 147 | + | } |
|
| 148 | + | for (u32 start = 1; start <= 4; start++) |
|
| 149 | + | check(mem, 7, 0xd1, 4, start, 128); |
|
| 150 | + | for (u32 type = 0x11; type <= 0xd1; type += 0x40) |
|
| 151 | + | check(mem, 7, type, 4, 0, 128); |
|
| 152 | + | u32 mixed[8] = { 0x00450513 }; |
|
| 153 | + | memcpy(mixed + 1, mem, sizeof(mem)); |
|
| 154 | + | check(mixed, 8, 0xd1, 4, 0, 128); |
|
| 155 | + | check(mixed, 8, 0xd1, 4, 1, 128); |
|
| 156 | + | check(mixed, 8, 0x10, 4, 0, 128); |
|
| 157 | + | check(mixed, 8, 0x12, 4, 0, 128); |
|
| 158 | + | check(mixed, 8, 0x80000000, 4, 0, 128); |
|
| 159 | + | u64 bounds[] = { DATA - 16, DATA - 15, DATA - 4, DATA - 3, |
|
| 160 | + | DATA, DATA + 1, ~0ULL - 7 }; |
|
| 161 | + | for (u32 i = 0; i < sizeof(bounds) / sizeof(bounds[0]); i++) { |
|
| 162 | + | check(mem, 7, 0xd1, 4, 0, bounds[i]); |
|
| 163 | + | check(&gather, 1, 0xd1, 4, 0, bounds[i]); |
|
| 164 | + | check(&store, 1, 0xd1, 4, 0, bounds[i]); |
|
| 165 | + | } |
|
| 166 | + | u32 invalid[] = { alu(0, 0, 9, 10, 12), |
|
| 167 | + | alu(12, 0, 8, 8, 24), |
|
| 168 | + | ops[0] & ~(1u << 25) }; |
|
| 169 | + | for (u32 i = 0; i < 3; i++) |
|
| 170 | + | check(invalid + i, 1, 0xd1, 4, 0, 128); |
|
| 171 | + | puts("JIT vector differential checks passed"); |
|
| 172 | + | return 0; |
|
| 173 | + | } |