jit: Translate bulk and bitset vector operations
edb99b5386968a77f4c53d82d5e025c4699adb7dbb0e703ddaa8e03abc52cecd
Translate byte fills, contiguous byte and word transfers, strided byte stores, word bitset operations, mask comparisons and population counts, and lane indexes. Select static vector configurations in native code. Check group alignment, active lengths and restart state before access. Handle exceptional memory ranges through the interpreter with precise guest instruction accounting. Change-Id: If1a15e6d0b50c9840c5ad1eaed644c0e6a4f7759
1 parent
137a6e3c
jit.c
+503 -88
| 11 | 11 | * r12 = ®s[0] (guest register file) |
|
| 12 | 12 | * r13 = memory base |
|
| 13 | 13 | * r14 = &jit->io (exit information; pc is its first field) |
|
| 14 | 14 | * r15 = budget (guest instructions left, signed) |
|
| 15 | 15 | * - Guest registers are loaded/stored on demand from/to the array. |
|
| 16 | - | * - Vector blocks guard the four-word e32,m2 shape before any side effect. |
|
| 17 | - | * Other shapes return to the interpreter. |
|
| 16 | + | * - Vector operations check their element width, group size, active length |
|
| 17 | + | * and restart index before access. |
|
| 18 | 18 | * - Each block starts with "if budget <= 0 exit at this PC; budget -= |
|
| 19 | 19 | * insn_count". Every exit is the block's last instruction, so a block |
|
| 20 | 20 | * always accounts exactly the instructions it executes. |
|
| 21 | 21 | * - Direct jumps (jal, both sides of a branch, block split) are a jmp |
|
| 22 | 22 | * rel32 that first falls into a stub returning JIT_EXIT_CHAIN. C then |
| 765 | 765 | emit_rexw(e); |
|
| 766 | 766 | emit_u8(e, 0x89); |
|
| 767 | 767 | emit_u8(e, modrm(0, X_RCX, X_RAX)); |
|
| 768 | 768 | } |
|
| 769 | 769 | ||
| 770 | + | /* Count the low 32 or 64 bits in rax and return the result in rax. */ |
|
| 771 | + | static void emit_popcount(struct emitter *e, bool wide) { |
|
| 772 | + | if (__builtin_cpu_supports("popcnt")) { |
|
| 773 | + | emit_u8(e, 0xf3); |
|
| 774 | + | if (wide) |
|
| 775 | + | emit_rexw(e); |
|
| 776 | + | emit_u8(e, 0x0f); |
|
| 777 | + | emit_u8(e, 0xb8); |
|
| 778 | + | emit_u8(e, 0xc0); |
|
| 779 | + | } else { |
|
| 780 | + | if (!wide) { |
|
| 781 | + | emit_u8(e, 0x89); |
|
| 782 | + | emit_u8(e, 0xc0); |
|
| 783 | + | } |
|
| 784 | + | /* Clear one set bit per iteration and count in RCX. */ |
|
| 785 | + | emit_u8(e, 0x31); |
|
| 786 | + | emit_u8(e, 0xc9); |
|
| 787 | + | emit_rexw(e); |
|
| 788 | + | emit_u8(e, 0x85); |
|
| 789 | + | emit_u8(e, 0xc0); |
|
| 790 | + | emit_u8(e, 0x74); |
|
| 791 | + | emit_u8(e, 12); |
|
| 792 | + | emit_rexw(e); |
|
| 793 | + | emit_u8(e, 0x8d); |
|
| 794 | + | emit_u8(e, 0x50); |
|
| 795 | + | emit_u8(e, 0xff); |
|
| 796 | + | emit_rexw(e); |
|
| 797 | + | emit_u8(e, 0xff); |
|
| 798 | + | emit_u8(e, 0xc1); |
|
| 799 | + | emit_rexw(e); |
|
| 800 | + | emit_u8(e, 0x21); |
|
| 801 | + | emit_u8(e, 0xd0); |
|
| 802 | + | emit_u8(e, 0x75); |
|
| 803 | + | emit_u8(e, (u8)-12); |
|
| 804 | + | emit_rexw(e); |
|
| 805 | + | emit_u8(e, 0x89); |
|
| 806 | + | emit_u8(e, 0xc8); |
|
| 807 | + | } |
|
| 808 | + | } |
|
| 809 | + | ||
| 770 | 810 | /* Emit a required bit manipulation instruction using baseline x86-64. */ |
|
| 771 | 811 | static void emit_bitmanip(struct emitter *e, instr_t ins, enum bitmanip_op op) { |
|
| 772 | 812 | if (ins.r.rd == ZERO) |
|
| 773 | 813 | return; |
|
| 774 | 814 | if (op == BM_SEXTB || op == BM_SEXTH || op == BM_ZEXTH) { |
| 1038 | 1078 | } |
|
| 1039 | 1079 | break; |
|
| 1040 | 1080 | } |
|
| 1041 | 1081 | case BM_CPOP: |
|
| 1042 | 1082 | case BM_CPOPW: { |
|
| 1043 | - | if (__builtin_cpu_supports("popcnt")) { |
|
| 1044 | - | emit_u8(e, 0xf3); |
|
| 1045 | - | if (op == BM_CPOP) |
|
| 1046 | - | emit_rexw(e); |
|
| 1047 | - | emit_u8(e, 0x0f); |
|
| 1048 | - | emit_u8(e, 0xb8); |
|
| 1049 | - | emit_u8(e, 0xc0); |
|
| 1050 | - | } else { |
|
| 1051 | - | if (op == BM_CPOPW) { |
|
| 1052 | - | emit_u8(e, 0x89); |
|
| 1053 | - | emit_u8(e, 0xc0); |
|
| 1054 | - | } |
|
| 1055 | - | /* Clear one set bit per iteration and count in RCX. */ |
|
| 1056 | - | emit_u8(e, 0x31); |
|
| 1057 | - | emit_u8(e, 0xc9); |
|
| 1058 | - | emit_rexw(e); |
|
| 1059 | - | emit_u8(e, 0x85); |
|
| 1060 | - | emit_u8(e, 0xc0); |
|
| 1061 | - | emit_u8(e, 0x74); |
|
| 1062 | - | emit_u8(e, 12); |
|
| 1063 | - | emit_rexw(e); |
|
| 1064 | - | emit_u8(e, 0x8d); |
|
| 1065 | - | emit_u8(e, 0x50); |
|
| 1066 | - | emit_u8(e, 0xff); |
|
| 1067 | - | emit_rexw(e); |
|
| 1068 | - | emit_u8(e, 0xff); |
|
| 1069 | - | emit_u8(e, 0xc1); |
|
| 1070 | - | emit_rexw(e); |
|
| 1071 | - | emit_u8(e, 0x21); |
|
| 1072 | - | emit_u8(e, 0xd0); |
|
| 1073 | - | emit_u8(e, 0x75); |
|
| 1074 | - | emit_u8(e, (u8)-12); |
|
| 1075 | - | emit_rexw(e); |
|
| 1076 | - | emit_u8(e, 0x89); |
|
| 1077 | - | emit_u8(e, 0xc8); |
|
| 1078 | - | } |
|
| 1083 | + | emit_popcount(e, op == BM_CPOP); |
|
| 1079 | 1084 | break; |
|
| 1080 | 1085 | } |
|
| 1081 | 1086 | case BM_ORCB: |
|
| 1082 | 1087 | break; |
|
| 1083 | 1088 | case BM_NONE: |
| 1113 | 1118 | emit_u8(e, 0x8b); |
|
| 1114 | 1119 | emit_u8(e, modrm(1, X_RAX, HIO)); |
|
| 1115 | 1120 | emit_u8(e, offsetof(struct jit_io, vector)); |
|
| 1116 | 1121 | } |
|
| 1117 | 1122 | ||
| 1123 | + | /* Guard integral groups at one element width and leave vl in rcx. |
|
| 1124 | + | * The register mask limits LMUL to the alignment of every operand group. */ |
|
| 1125 | + | static void emit_group_guard(struct emitter *e, u32 pc, u32 width, u32 regs) { |
|
| 1126 | + | u32 lm = 0; |
|
| 1127 | + | while (lm < 3 && !(regs & (1u << lm))) |
|
| 1128 | + | lm++; |
|
| 1129 | + | emit_vector_base(e); |
|
| 1130 | + | emit_rexw(e); |
|
| 1131 | + | emit_u8(e, 0x8b); |
|
| 1132 | + | emit_u8(e, modrm(2, X_RCX, X_RAX)); |
|
| 1133 | + | emit_u32(e, offsetof(struct vector_state, vtype)); |
|
| 1134 | + | emit_rexw(e); |
|
| 1135 | + | emit_u8(e, 0x89); |
|
| 1136 | + | emit_u8(e, 0xca); /* mov rdx, rcx */ |
|
| 1137 | + | emit_rexw(e); |
|
| 1138 | + | emit_u8(e, 0x81); |
|
| 1139 | + | emit_u8(e, 0xe2); |
|
| 1140 | + | emit_u32(e, ~0xc7u); |
|
| 1141 | + | emit_rexw(e); |
|
| 1142 | + | emit_u8(e, 0x83); |
|
| 1143 | + | emit_u8(e, 0xfa); |
|
| 1144 | + | emit_u8(e, width << 3); |
|
| 1145 | + | emit_vector_fault(e, pc, 0x85); |
|
| 1146 | + | emit_rexw(e); |
|
| 1147 | + | emit_u8(e, 0x83); |
|
| 1148 | + | emit_u8(e, 0xe1); |
|
| 1149 | + | emit_u8(e, 7); |
|
| 1150 | + | emit_rexw(e); |
|
| 1151 | + | emit_u8(e, 0x83); |
|
| 1152 | + | emit_u8(e, 0xf9); |
|
| 1153 | + | emit_u8(e, lm); |
|
| 1154 | + | emit_vector_fault(e, pc, 0x87); |
|
| 1155 | + | emit_mov_imm32_sx(e, X_RDX, VECTOR_BYTES >> width); |
|
| 1156 | + | emit_rexw(e); |
|
| 1157 | + | emit_u8(e, 0xd3); |
|
| 1158 | + | emit_u8(e, 0xe2); /* shl rdx, cl */ |
|
| 1159 | + | emit_rexw(e); |
|
| 1160 | + | emit_u8(e, 0x83); |
|
| 1161 | + | emit_u8(e, modrm(2, 7, X_RAX)); |
|
| 1162 | + | emit_u32(e, offsetof(struct vector_state, vstart)); |
|
| 1163 | + | emit_u8(e, 0); |
|
| 1164 | + | emit_vector_fault(e, pc, 0x85); |
|
| 1165 | + | emit_rexw(e); |
|
| 1166 | + | emit_u8(e, 0x8b); |
|
| 1167 | + | emit_u8(e, modrm(2, X_RCX, X_RAX)); |
|
| 1168 | + | emit_u32(e, offsetof(struct vector_state, vl)); |
|
| 1169 | + | emit_rexw(e); |
|
| 1170 | + | emit_u8(e, 0x39); |
|
| 1171 | + | emit_u8(e, 0xd1); /* cmp rcx, rdx */ |
|
| 1172 | + | emit_vector_fault(e, pc, 0x87); |
|
| 1173 | + | } |
|
| 1174 | + | ||
| 1175 | + | /* Form a pointer into one architectural register group. */ |
|
| 1176 | + | static void emit_group_pointer(struct emitter *e, enum x86reg dst, u32 reg) { |
|
| 1177 | + | emit_rexw(e); |
|
| 1178 | + | emit_u8(e, 0x8d); |
|
| 1179 | + | emit_u8(e, modrm(2, dst, X_RAX)); |
|
| 1180 | + | emit_u32(e, reg * VECTOR_BYTES); |
|
| 1181 | + | } |
|
| 1182 | + | ||
| 1183 | + | /* Select a static vector type and end the block after its state change. */ |
|
| 1184 | + | static bool emit_vector_config(struct emitter *e, u32 ins, u32 pc) { |
|
| 1185 | + | bool immediate = (ins >> 30) == 3; |
|
| 1186 | + | if ((!immediate && (ins >> 31)) || ((ins >> 12) & 7) != 7) |
|
| 1187 | + | return false; |
|
| 1188 | + | u32 type = (ins >> 20) & (immediate ? 1023 : 2047); |
|
| 1189 | + | u32 sew = 8u << ((type >> 3) & 7), lm = type & 7; |
|
| 1190 | + | u32 src = (ins >> 15) & 31, dst = (ins >> 7) & 31; |
|
| 1191 | + | if ((type & ~255u) || sew > 64 || lm == 4 || (!immediate && !src && !dst)) |
|
| 1192 | + | return false; |
|
| 1193 | + | u32 bytes = lm < 4 ? VECTOR_BYTES << lm : VECTOR_BYTES >> (8 - lm); |
|
| 1194 | + | if (bytes * 8 < sew) |
|
| 1195 | + | return false; |
|
| 1196 | + | u32 max = bytes * 8 / sew; |
|
| 1197 | + | if (immediate) { |
|
| 1198 | + | emit_mov_imm32_sx(e, X_RAX, src < max ? src : max); |
|
| 1199 | + | } else if (!src) { |
|
| 1200 | + | emit_mov_imm32_sx(e, X_RAX, max); |
|
| 1201 | + | } else { |
|
| 1202 | + | emit_load_guest(e, X_RAX, src); |
|
| 1203 | + | emit_mov_imm32_sx(e, X_RCX, max); |
|
| 1204 | + | emit_alu_rax_rcx(e, 0x39); |
|
| 1205 | + | emit_rexw(e); |
|
| 1206 | + | emit_u8(e, 0x0f); |
|
| 1207 | + | emit_u8(e, 0x47); |
|
| 1208 | + | emit_u8(e, 0xc1); /* cmova rax, rcx */ |
|
| 1209 | + | } |
|
| 1210 | + | emit_store_guest(e, dst, X_RAX); |
|
| 1211 | + | emit_rexw(e); |
|
| 1212 | + | emit_u8(e, 0x89); |
|
| 1213 | + | emit_u8(e, 0xc2); |
|
| 1214 | + | emit_vector_base(e); |
|
| 1215 | + | emit_rexw(e); |
|
| 1216 | + | emit_u8(e, 0x89); |
|
| 1217 | + | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1218 | + | emit_u32(e, offsetof(struct vector_state, vl)); |
|
| 1219 | + | emit_rexw(e); |
|
| 1220 | + | emit_u8(e, 0xc7); |
|
| 1221 | + | emit_u8(e, modrm(2, 0, X_RAX)); |
|
| 1222 | + | emit_u32(e, offsetof(struct vector_state, vtype)); |
|
| 1223 | + | emit_u32(e, type); |
|
| 1224 | + | emit_rexw(e); |
|
| 1225 | + | emit_u8(e, 0xc7); |
|
| 1226 | + | emit_u8(e, modrm(2, 0, X_RAX)); |
|
| 1227 | + | emit_u32(e, offsetof(struct vector_state, vstart)); |
|
| 1228 | + | emit_u32(e, 0); |
|
| 1229 | + | emit_chain_exit(e, pc + INSTR_SIZE); |
|
| 1230 | + | return true; |
|
| 1231 | + | } |
|
| 1232 | + | ||
| 1233 | + | /* Fill active bytes of a register group from a scalar or signed immediate. */ |
|
| 1234 | + | static bool emit_byte_splat(struct emitter *e, u32 ins, u32 pc) { |
|
| 1235 | + | u32 form = (ins >> 12) & 7, reg = (ins >> 7) & 31; |
|
| 1236 | + | if ((ins & 0xfff0007f) != 0x5e000057 || (form != 3 && form != 4)) |
|
| 1237 | + | return false; |
|
| 1238 | + | emit_group_guard(e, pc, 0, reg); |
|
| 1239 | + | emit_group_pointer(e, X_RDI, reg); |
|
| 1240 | + | u32 src = (ins >> 15) & 31; |
|
| 1241 | + | if (form == 4) |
|
| 1242 | + | emit_load_guest(e, X_RAX, src); |
|
| 1243 | + | else |
|
| 1244 | + | emit_mov_imm32_sx(e, X_RAX, (i32)(src << 27) >> 27); |
|
| 1245 | + | emit_u8(e, 0xf3); |
|
| 1246 | + | emit_u8(e, 0xaa); /* rep stosb */ |
|
| 1247 | + | return true; |
|
| 1248 | + | } |
|
| 1249 | + | ||
| 1118 | 1250 | /* Form one indexed guest address, with overflow checked before any transfer. */ |
|
| 1119 | 1251 | static void emit_vector_address(struct emitter *e, u32 ins, u32 lane, u32 pc) { |
|
| 1120 | 1252 | u32 base = (ins >> 15) & 31, index = (ins >> 20) & 31; |
|
| 1121 | 1253 | emit_vector_base(e); |
|
| 1122 | 1254 | emit_u8(e, 0x8b); |
| 1144 | 1276 | emit_u8(e, 0x39); |
|
| 1145 | 1277 | emit_u8(e, 0xc8); |
|
| 1146 | 1278 | emit_vector_fault(e, pc, 0x87); /* ja */ |
|
| 1147 | 1279 | } |
|
| 1148 | 1280 | ||
| 1281 | + | /* Transfer active bytes with complete bounds checks before any guest write. */ |
|
| 1282 | + | static bool emit_group_memory(struct emitter *e, u32 ins, u32 pc) { |
|
| 1283 | + | u32 op = ins & 127, reg = (ins >> 7) & 31; |
|
| 1284 | + | u32 mode = (ins >> 26) & 3, index = (ins >> 20) & 31; |
|
| 1285 | + | u32 field = (ins >> 12) & 7, width = field == 6 ? 2 : 0; |
|
| 1286 | + | bool store = op == 39; |
|
| 1287 | + | if ((op != 7 && !store) || (field != 0 && field != 6) || |
|
| 1288 | + | !(ins & (1u << 25)) || (ins >> 29) || (ins & (1u << 28)) || |
|
| 1289 | + | !((mode == 0 && index == 0) || (mode == 2 && store && width == 0))) |
|
| 1290 | + | return false; |
|
| 1291 | + | emit_group_guard(e, pc, width, reg); |
|
| 1292 | + | if (width) { |
|
| 1293 | + | emit_rexw(e); |
|
| 1294 | + | emit_u8(e, 0xc1); |
|
| 1295 | + | emit_u8(e, 0xe1); |
|
| 1296 | + | emit_u8(e, width); |
|
| 1297 | + | } |
|
| 1298 | + | emit_rexw(e); |
|
| 1299 | + | emit_u8(e, 0x85); |
|
| 1300 | + | emit_u8(e, 0xc9); |
|
| 1301 | + | u32 empty = e->pos; |
|
| 1302 | + | emit_u8(e, 0x0f); |
|
| 1303 | + | emit_u8(e, 0x84); |
|
| 1304 | + | emit_u32(e, 0); |
|
| 1305 | + | if (mode == 0) { |
|
| 1306 | + | emit_group_pointer(e, store ? X_RSI : X_RDI, reg); |
|
| 1307 | + | emit_load_guest(e, X_RAX, (ins >> 15) & 31); |
|
| 1308 | + | emit_u8(e, 0x49); |
|
| 1309 | + | emit_u8(e, 0x8b); |
|
| 1310 | + | emit_u8(e, modrm(1, X_RDX, HIO)); |
|
| 1311 | + | emit_u8(e, offsetof(struct jit_io, memory_size)); |
|
| 1312 | + | emit_rexw(e); |
|
| 1313 | + | emit_u8(e, 0x29); |
|
| 1314 | + | emit_u8(e, 0xca); /* sub rdx, rcx */ |
|
| 1315 | + | emit_vector_fault(e, pc, 0x82); |
|
| 1316 | + | emit_rexw(e); |
|
| 1317 | + | emit_u8(e, 0x39); |
|
| 1318 | + | emit_u8(e, 0xd0); /* cmp rax, rdx */ |
|
| 1319 | + | emit_vector_fault(e, pc, 0x87); |
|
| 1320 | + | emit_add_rax_r13(e); |
|
| 1321 | + | emit_rexw(e); |
|
| 1322 | + | emit_u8(e, 0x89); |
|
| 1323 | + | emit_u8(e, store ? 0xc7 : 0xc6); |
|
| 1324 | + | emit_u8(e, 0xf3); |
|
| 1325 | + | emit_u8(e, 0xa4); /* rep movsb */ |
|
| 1326 | + | } else { |
|
| 1327 | + | emit_load_guest(e, X_RDI, (ins >> 15) & 31); |
|
| 1328 | + | emit_load_guest(e, X_RSI, index); |
|
| 1329 | + | emit_rexw(e); |
|
| 1330 | + | emit_u8(e, 0x89); |
|
| 1331 | + | emit_u8(e, 0xc8); |
|
| 1332 | + | emit_rexw(e); |
|
| 1333 | + | emit_u8(e, 0xff); |
|
| 1334 | + | emit_u8(e, 0xc8); /* dec rax */ |
|
| 1335 | + | emit_rexw(e); |
|
| 1336 | + | emit_u8(e, 0xf7); |
|
| 1337 | + | emit_u8(e, 0xe6); /* mul rsi */ |
|
| 1338 | + | emit_rexw(e); |
|
| 1339 | + | emit_u8(e, 0x85); |
|
| 1340 | + | emit_u8(e, 0xd2); |
|
| 1341 | + | emit_vector_fault(e, pc, 0x85); |
|
| 1342 | + | emit_rexw(e); |
|
| 1343 | + | emit_u8(e, 0x01); |
|
| 1344 | + | emit_u8(e, 0xf8); /* add rax, rdi */ |
|
| 1345 | + | emit_vector_fault(e, pc, 0x82); |
|
| 1346 | + | emit_vector_bound(e, 1, pc); |
|
| 1347 | + | emit_rexw(e); |
|
| 1348 | + | emit_u8(e, 0x89); |
|
| 1349 | + | emit_u8(e, 0xf2); /* mov rdx, rsi */ |
|
| 1350 | + | emit_vector_base(e); |
|
| 1351 | + | emit_rexw(e); |
|
| 1352 | + | emit_u8(e, 0x8b); |
|
| 1353 | + | emit_u8(e, modrm(2, X_RCX, X_RAX)); |
|
| 1354 | + | emit_u32(e, offsetof(struct vector_state, vl)); |
|
| 1355 | + | emit_group_pointer(e, X_RSI, reg); |
|
| 1356 | + | emit_u8(e, 0x4c); |
|
| 1357 | + | emit_u8(e, 0x01); |
|
| 1358 | + | emit_u8(e, 0xef); /* add rdi, r13 */ |
|
| 1359 | + | u32 loop = e->pos; |
|
| 1360 | + | emit_u8(e, 0xac); /* lodsb */ |
|
| 1361 | + | emit_u8(e, 0x88); |
|
| 1362 | + | emit_u8(e, 0x07); |
|
| 1363 | + | emit_rexw(e); |
|
| 1364 | + | emit_u8(e, 0x01); |
|
| 1365 | + | emit_u8(e, 0xd7); |
|
| 1366 | + | emit_u8(e, 0xe2); |
|
| 1367 | + | emit_u8(e, (u8)(loop - e->pos - 1)); |
|
| 1368 | + | } |
|
| 1369 | + | patch_jcc(e, empty, e->pos); |
|
| 1370 | + | return true; |
|
| 1371 | + | } |
|
| 1372 | + | ||
| 1149 | 1373 | /* Emit an SSE instruction on two XMM registers. */ |
|
| 1150 | 1374 | static void emit_xmm(struct emitter *e, u8 opcode, u32 d, u32 src) { |
|
| 1151 | 1375 | emit_u8(e, 0x66); |
|
| 1152 | 1376 | emit_u8(e, 0x0f); |
|
| 1153 | 1377 | emit_u8(e, opcode); |
| 1162 | 1386 | emit_u8(e, store ? 0x7f : 0x6f); |
|
| 1163 | 1387 | emit_u8(e, modrm(2, xmm, X_RAX)); |
|
| 1164 | 1388 | emit_u32(e, reg * VECTOR_BYTES); |
|
| 1165 | 1389 | } |
|
| 1166 | 1390 | ||
| 1167 | - | /* Emit checked unmasked word loads and stores for one register group. */ |
|
| 1168 | - | static bool emit_vector_memory(struct emitter *e, u32 ins, u32 pc) { |
|
| 1169 | - | u32 op = ins & 127, reg = (ins >> 7) & 31, index = (ins >> 20) & 31; |
|
| 1170 | - | u32 mode = (ins >> 26) & 3; |
|
| 1171 | - | bool store = op == 39; |
|
| 1172 | - | if ((op != 7 && !store) || ((ins >> 12) & 7) != 6 || !(ins & (1u << 25)) || |
|
| 1173 | - | (ins >> 29) || (ins & (1u << 28)) || (reg & 1) || |
|
| 1174 | - | !((mode == 0 && index == 0) || (mode == 1 && !store && !(index & 1)))) |
|
| 1175 | - | return false; |
|
| 1176 | - | if (mode == 1 && !store && reg == index) |
|
| 1391 | + | /* Emit checked unmasked indexed word loads for one register group. */ |
|
| 1392 | + | static bool emit_indexed_words(struct emitter *e, u32 ins, u32 pc) { |
|
| 1393 | + | u32 reg = (ins >> 7) & 31, index = (ins >> 20) & 31; |
|
| 1394 | + | if ((ins & 127) != 7 || ((ins >> 12) & 7) != 6 || !(ins & (1u << 25)) || |
|
| 1395 | + | (ins >> 29) || (ins & (1u << 28)) || (reg & 1) || (index & 1) || |
|
| 1396 | + | ((ins >> 26) & 3) != 1 || reg == index) |
|
| 1177 | 1397 | return false; |
|
| 1178 | 1398 | e->vector = true; |
|
| 1179 | - | if (mode == 0) { |
|
| 1399 | + | /* Preflight every address so an exceptional access has no partial |
|
| 1400 | + | * writes. */ |
|
| 1401 | + | for (u32 i = 0; i < 4; i++) { |
|
| 1402 | + | emit_vector_address(e, ins, i, pc); |
|
| 1403 | + | emit_vector_bound(e, 4, pc); |
|
| 1404 | + | } |
|
| 1405 | + | for (u32 i = 0; i < 4; i++) { |
|
| 1406 | + | emit_vector_base(e); |
|
| 1407 | + | emit_u8(e, 0x8b); |
|
| 1408 | + | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1409 | + | emit_u32(e, index * VECTOR_BYTES + i * 4); |
|
| 1180 | 1410 | emit_load_guest(e, X_RAX, (ins >> 15) & 31); |
|
| 1181 | - | emit_vector_bound(e, 16, pc); |
|
| 1182 | - | if (store) { |
|
| 1183 | - | emit_vector_base(e); |
|
| 1184 | - | emit_xmm_memory(e, 0, reg, false); |
|
| 1185 | - | emit_load_guest(e, X_RAX, (ins >> 15) & 31); |
|
| 1186 | - | } |
|
| 1411 | + | emit_u8(e, 0x48); |
|
| 1412 | + | emit_u8(e, 0x01); |
|
| 1413 | + | emit_u8(e, 0xd0); |
|
| 1187 | 1414 | emit_add_rax_r13(e); |
|
| 1188 | - | emit_u8(e, 0xf3); |
|
| 1189 | - | emit_u8(e, 0x0f); |
|
| 1190 | - | emit_u8(e, store ? 0x7f : 0x6f); |
|
| 1191 | - | emit_u8(e, 0x00); |
|
| 1192 | - | if (!store) { |
|
| 1193 | - | emit_vector_base(e); |
|
| 1194 | - | emit_xmm_memory(e, 0, reg, true); |
|
| 1195 | - | } |
|
| 1196 | - | } else { |
|
| 1197 | - | /* Preflight every address so an exceptional access has no partial |
|
| 1198 | - | * writes. */ |
|
| 1199 | - | for (u32 i = 0; i < 4; i++) { |
|
| 1200 | - | emit_vector_address(e, ins, i, pc); |
|
| 1201 | - | emit_vector_bound(e, 4, pc); |
|
| 1202 | - | } |
|
| 1203 | - | for (u32 i = 0; i < 4; i++) { |
|
| 1204 | - | emit_vector_base(e); |
|
| 1205 | - | emit_u8(e, 0x8b); |
|
| 1206 | - | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1207 | - | emit_u32(e, index * VECTOR_BYTES + i * 4); |
|
| 1208 | - | emit_load_guest(e, X_RAX, (ins >> 15) & 31); |
|
| 1209 | - | emit_u8(e, 0x48); |
|
| 1210 | - | emit_u8(e, 0x01); |
|
| 1211 | - | emit_u8(e, 0xd0); |
|
| 1212 | - | emit_add_rax_r13(e); |
|
| 1213 | - | emit_u8(e, 0x8b); |
|
| 1214 | - | emit_u8(e, 0x10); /* mov edx, [rax] */ |
|
| 1215 | - | emit_vector_base(e); |
|
| 1216 | - | emit_u8(e, 0x89); |
|
| 1217 | - | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1218 | - | emit_u32(e, reg * VECTOR_BYTES + i * 4); |
|
| 1219 | - | } |
|
| 1415 | + | emit_u8(e, 0x8b); |
|
| 1416 | + | emit_u8(e, 0x10); /* mov edx, [rax] */ |
|
| 1417 | + | emit_vector_base(e); |
|
| 1418 | + | emit_u8(e, 0x89); |
|
| 1419 | + | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1420 | + | emit_u32(e, reg * VECTOR_BYTES + i * 4); |
|
| 1220 | 1421 | } |
|
| 1221 | 1422 | return true; |
|
| 1222 | 1423 | } |
|
| 1223 | 1424 | ||
| 1224 | 1425 | /* Gather four words from a validated, non-overlapping register group. */ |
| 1246 | 1447 | emit_u8(e, modrm(2, X_RCX, X_RAX)); |
|
| 1247 | 1448 | emit_u32(e, d * VECTOR_BYTES + i * 4); |
|
| 1248 | 1449 | } |
|
| 1249 | 1450 | } |
|
| 1250 | 1451 | ||
| 1452 | + | /* Emit a relative conditional jump to an existing native position. */ |
|
| 1453 | + | static void emit_jump_back(struct emitter *e, u8 condition, u32 target) { |
|
| 1454 | + | emit_u8(e, 0x0f); |
|
| 1455 | + | emit_u8(e, condition); |
|
| 1456 | + | emit_u32(e, target - e->pos - 4); |
|
| 1457 | + | } |
|
| 1458 | + | ||
| 1459 | + | /* Emit a zero-count branch after testing rcx. */ |
|
| 1460 | + | static u32 emit_count_zero(struct emitter *e) { |
|
| 1461 | + | emit_rexw(e); |
|
| 1462 | + | emit_u8(e, 0x85); |
|
| 1463 | + | emit_u8(e, 0xc9); |
|
| 1464 | + | u32 pos = e->pos; |
|
| 1465 | + | emit_u8(e, 0x0f); |
|
| 1466 | + | emit_u8(e, 0x84); |
|
| 1467 | + | emit_u32(e, 0); |
|
| 1468 | + | return pos; |
|
| 1469 | + | } |
|
| 1470 | + | ||
| 1471 | + | /* Apply word-wise OR or AND-NOT in SIMD chunks, preserving inactive words. */ |
|
| 1472 | + | static void emit_group_bits(struct emitter *e, u32 ins, u32 pc) { |
|
| 1473 | + | u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31; |
|
| 1474 | + | bool invert = (ins >> 26) == 1; |
|
| 1475 | + | emit_group_guard(e, pc, 2, d | a | b); |
|
| 1476 | + | emit_group_pointer(e, X_RSI, a); |
|
| 1477 | + | emit_group_pointer(e, X_RDI, b); |
|
| 1478 | + | emit_group_pointer(e, X_RDX, d); |
|
| 1479 | + | emit_rexw(e); |
|
| 1480 | + | emit_u8(e, 0x83); |
|
| 1481 | + | emit_u8(e, 0xf9); |
|
| 1482 | + | emit_u8(e, 4); |
|
| 1483 | + | u32 tail = e->pos; |
|
| 1484 | + | emit_u8(e, 0x0f); |
|
| 1485 | + | emit_u8(e, 0x82); |
|
| 1486 | + | emit_u32(e, 0); |
|
| 1487 | + | u32 loop = e->pos; |
|
| 1488 | + | const u8 loads[] = { 0xf3, 0x0f, 0x6f, 0x06, 0xf3, 0x0f, 0x6f, 0x0f }; |
|
| 1489 | + | for (u32 i = 0; i < sizeof(loads); i++) |
|
| 1490 | + | emit_u8(e, loads[i]); |
|
| 1491 | + | emit_xmm(e, invert ? 0xdf : 0xeb, invert ? 1 : 0, invert ? 0 : 1); |
|
| 1492 | + | emit_u8(e, 0xf3); |
|
| 1493 | + | emit_u8(e, 0x0f); |
|
| 1494 | + | emit_u8(e, 0x7f); |
|
| 1495 | + | emit_u8(e, modrm(0, invert ? 1 : 0, X_RDX)); |
|
| 1496 | + | const u32 pointers[] = { X_RSI, X_RDI, X_RDX }; |
|
| 1497 | + | for (u32 i = 0; i < 3; i++) { |
|
| 1498 | + | emit_rexw(e); |
|
| 1499 | + | emit_u8(e, 0x83); |
|
| 1500 | + | emit_u8(e, modrm(3, 0, pointers[i])); |
|
| 1501 | + | emit_u8(e, 16); |
|
| 1502 | + | } |
|
| 1503 | + | emit_rexw(e); |
|
| 1504 | + | emit_u8(e, 0x83); |
|
| 1505 | + | emit_u8(e, 0xe9); |
|
| 1506 | + | emit_u8(e, 4); |
|
| 1507 | + | emit_rexw(e); |
|
| 1508 | + | emit_u8(e, 0x83); |
|
| 1509 | + | emit_u8(e, 0xf9); |
|
| 1510 | + | emit_u8(e, 4); |
|
| 1511 | + | emit_jump_back(e, 0x83, loop); |
|
| 1512 | + | patch_jcc(e, tail, e->pos); |
|
| 1513 | + | u32 empty = emit_count_zero(e); |
|
| 1514 | + | loop = e->pos; |
|
| 1515 | + | emit_u8(e, 0x8b); |
|
| 1516 | + | emit_u8(e, invert ? 0x07 : 0x06); |
|
| 1517 | + | if (invert) { |
|
| 1518 | + | emit_u8(e, 0xf7); |
|
| 1519 | + | emit_u8(e, 0xd0); |
|
| 1520 | + | } |
|
| 1521 | + | emit_u8(e, invert ? 0x23 : 0x0b); |
|
| 1522 | + | emit_u8(e, invert ? 0x06 : 0x07); |
|
| 1523 | + | emit_u8(e, 0x89); |
|
| 1524 | + | emit_u8(e, 0x02); |
|
| 1525 | + | for (u32 i = 0; i < 3; i++) { |
|
| 1526 | + | emit_rexw(e); |
|
| 1527 | + | emit_u8(e, 0x83); |
|
| 1528 | + | emit_u8(e, modrm(3, 0, pointers[i])); |
|
| 1529 | + | emit_u8(e, 4); |
|
| 1530 | + | } |
|
| 1531 | + | emit_u8(e, 0xe2); |
|
| 1532 | + | emit_u8(e, (u8)(loop - e->pos - 1)); |
|
| 1533 | + | patch_jcc(e, empty, e->pos); |
|
| 1534 | + | } |
|
| 1535 | + | ||
| 1536 | + | /* Write active word comparisons into a mask after reading both source groups. |
|
| 1537 | + | */ |
|
| 1538 | + | static void emit_group_compare(struct emitter *e, u32 ins, u32 pc) { |
|
| 1539 | + | u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31; |
|
| 1540 | + | emit_group_guard(e, pc, 2, a | b); |
|
| 1541 | + | emit_group_pointer(e, X_RSI, a); |
|
| 1542 | + | emit_group_pointer(e, X_RDI, b); |
|
| 1543 | + | emit_group_pointer(e, X_RDX, d); |
|
| 1544 | + | u32 empty = emit_count_zero(e); |
|
| 1545 | + | const u8 begin[] = { 0x45, 0x31, 0xc0, 0x49, 0x89, 0xc9 }; |
|
| 1546 | + | for (u32 i = 0; i < sizeof(begin); i++) |
|
| 1547 | + | emit_u8(e, begin[i]); |
|
| 1548 | + | u32 loop = e->pos; |
|
| 1549 | + | const u8 compare[] = { 0x8b, 0x44, 0x8e, 0xfc, 0x3b, 0x44, 0x8f, 0xfc, |
|
| 1550 | + | 0x0f, 0x95, 0xc0, 0x0f, 0xb6, 0xc0, 0x49, 0xd1, |
|
| 1551 | + | 0xe0, 0x49, 0x09, 0xc0, 0x48, 0xff, 0xc9 }; |
|
| 1552 | + | for (u32 i = 0; i < sizeof(compare); i++) |
|
| 1553 | + | emit_u8(e, compare[i]); |
|
| 1554 | + | emit_jump_back(e, 0x85, loop); |
|
| 1555 | + | emit_u8(e, 0x4c); |
|
| 1556 | + | emit_u8(e, 0x89); |
|
| 1557 | + | emit_u8(e, 0xc9); /* mov rcx, r9 */ |
|
| 1558 | + | emit_mov_imm32_sx(e, X_RAX, -1); |
|
| 1559 | + | emit_rexw(e); |
|
| 1560 | + | emit_u8(e, 0x83); |
|
| 1561 | + | emit_u8(e, 0xf9); |
|
| 1562 | + | emit_u8(e, 64); |
|
| 1563 | + | u32 full = e->pos; |
|
| 1564 | + | emit_u8(e, 0x0f); |
|
| 1565 | + | emit_u8(e, 0x84); |
|
| 1566 | + | emit_u32(e, 0); |
|
| 1567 | + | emit_rexw(e); |
|
| 1568 | + | emit_u8(e, 0xd3); |
|
| 1569 | + | emit_u8(e, 0xe0); |
|
| 1570 | + | u32 merge = emit_jmp_placeholder(e); |
|
| 1571 | + | patch_jcc(e, full, e->pos); |
|
| 1572 | + | emit_u8(e, 0x31); |
|
| 1573 | + | emit_u8(e, 0xc0); |
|
| 1574 | + | patch_jmp(e, merge, e->pos); |
|
| 1575 | + | const u8 finish[] = { |
|
| 1576 | + | 0x48, 0x23, 0x02, 0x4c, 0x09, 0xc0, 0x48, 0x89, 0x02 |
|
| 1577 | + | }; |
|
| 1578 | + | for (u32 i = 0; i < sizeof(finish); i++) |
|
| 1579 | + | emit_u8(e, finish[i]); |
|
| 1580 | + | patch_jcc(e, empty, e->pos); |
|
| 1581 | + | } |
|
| 1582 | + | ||
| 1583 | + | /* Count the active bits of a mask register at word element width. */ |
|
| 1584 | + | static void emit_mask_population(struct emitter *e, u32 ins, u32 pc) { |
|
| 1585 | + | u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31; |
|
| 1586 | + | emit_group_guard(e, pc, 2, 0); |
|
| 1587 | + | emit_rexw(e); |
|
| 1588 | + | emit_u8(e, 0x8b); |
|
| 1589 | + | emit_u8(e, modrm(2, X_RAX, X_RAX)); |
|
| 1590 | + | emit_u32(e, a * VECTOR_BYTES); |
|
| 1591 | + | emit_rexw(e); |
|
| 1592 | + | emit_u8(e, 0x83); |
|
| 1593 | + | emit_u8(e, 0xf9); |
|
| 1594 | + | emit_u8(e, 64); |
|
| 1595 | + | u32 full = e->pos; |
|
| 1596 | + | emit_u8(e, 0x0f); |
|
| 1597 | + | emit_u8(e, 0x84); |
|
| 1598 | + | emit_u32(e, 0); |
|
| 1599 | + | emit_mov_imm32_sx(e, X_RDX, -1); |
|
| 1600 | + | emit_rexw(e); |
|
| 1601 | + | emit_u8(e, 0xd3); |
|
| 1602 | + | emit_u8(e, 0xe2); |
|
| 1603 | + | emit_rexw(e); |
|
| 1604 | + | emit_u8(e, 0xf7); |
|
| 1605 | + | emit_u8(e, 0xd2); |
|
| 1606 | + | emit_rexw(e); |
|
| 1607 | + | emit_u8(e, 0x21); |
|
| 1608 | + | emit_u8(e, 0xd0); |
|
| 1609 | + | patch_jcc(e, full, e->pos); |
|
| 1610 | + | emit_popcount(e, true); |
|
| 1611 | + | emit_store_guest(e, d, X_RAX); |
|
| 1612 | + | } |
|
| 1613 | + | ||
| 1614 | + | /* Write consecutive word indexes into the active destination elements. */ |
|
| 1615 | + | static void emit_lane_indexes(struct emitter *e, u32 ins, u32 pc) { |
|
| 1616 | + | u32 d = (ins >> 7) & 31; |
|
| 1617 | + | emit_group_guard(e, pc, 2, d); |
|
| 1618 | + | emit_group_pointer(e, X_RDI, d); |
|
| 1619 | + | u32 empty = emit_count_zero(e); |
|
| 1620 | + | emit_u8(e, 0x31); |
|
| 1621 | + | emit_u8(e, 0xd2); |
|
| 1622 | + | u32 loop = e->pos; |
|
| 1623 | + | const u8 store[] = { 0x89, 0x17, 0x48, 0x83, 0xc7, 4, 0xff, 0xc2 }; |
|
| 1624 | + | for (u32 i = 0; i < sizeof(store); i++) |
|
| 1625 | + | emit_u8(e, store[i]); |
|
| 1626 | + | emit_u8(e, 0xe2); |
|
| 1627 | + | emit_u8(e, (u8)(loop - e->pos - 1)); |
|
| 1628 | + | patch_jcc(e, empty, e->pos); |
|
| 1629 | + | } |
|
| 1630 | + | ||
| 1631 | + | /* Emit the word group operations used by bitsets and lane-index setup. */ |
|
| 1632 | + | static bool emit_word_groups(struct emitter *e, u32 ins, u32 pc) { |
|
| 1633 | + | u32 fn = ins >> 26, form = (ins >> 12) & 7; |
|
| 1634 | + | u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31; |
|
| 1635 | + | if (!(ins & (1u << 25))) |
|
| 1636 | + | return false; |
|
| 1637 | + | if (form == 0 && (fn == 1 || fn == 10)) { |
|
| 1638 | + | emit_group_bits(e, ins, pc); |
|
| 1639 | + | return true; |
|
| 1640 | + | } |
|
| 1641 | + | if (form == 0 && fn == 25) { |
|
| 1642 | + | if ((d > a && d < a + 8) || (d > b && d < b + 8)) |
|
| 1643 | + | return false; |
|
| 1644 | + | emit_group_compare(e, ins, pc); |
|
| 1645 | + | return true; |
|
| 1646 | + | } |
|
| 1647 | + | if (form == 2 && fn == 16 && b == 16) { |
|
| 1648 | + | emit_mask_population(e, ins, pc); |
|
| 1649 | + | return true; |
|
| 1650 | + | } |
|
| 1651 | + | if (form == 2 && fn == 20 && b == 17 && a == 0) { |
|
| 1652 | + | emit_lane_indexes(e, ins, pc); |
|
| 1653 | + | return true; |
|
| 1654 | + | } |
|
| 1655 | + | return false; |
|
| 1656 | + | } |
|
| 1657 | + | ||
| 1251 | 1658 | /* Emit unmasked four-word integer operations under the block's state guard. */ |
|
| 1252 | 1659 | static bool emit_vector(struct emitter *e, u32 ins) { |
|
| 1253 | 1660 | u32 fn = ins >> 26, form = (ins >> 12) & 7; |
|
| 1254 | 1661 | u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31; |
|
| 1255 | 1662 | if (!(ins & (1u << 25)) || (d & 1) || (a & 1)) |
| 1350 | 1757 | if (bm != BM_NONE) { |
|
| 1351 | 1758 | emit_bitmanip(e, ins, bm); |
|
| 1352 | 1759 | return false; |
|
| 1353 | 1760 | } |
|
| 1354 | 1761 | ||
| 1355 | - | if ((opcode == 7 || opcode == 39) && emit_vector_memory(e, ins.raw, pc)) |
|
| 1762 | + | if (opcode == 0x57 && emit_vector_config(e, ins.raw, pc)) |
|
| 1763 | + | return true; |
|
| 1764 | + | if (opcode == 0x57 && emit_byte_splat(e, ins.raw, pc)) |
|
| 1765 | + | return false; |
|
| 1766 | + | if ((opcode == 7 || opcode == 39) && emit_group_memory(e, ins.raw, pc)) |
|
| 1767 | + | return false; |
|
| 1768 | + | if ((opcode == 7 || opcode == 39) && emit_indexed_words(e, ins.raw, pc)) |
|
| 1769 | + | return false; |
|
| 1770 | + | if (opcode == 0x57 && emit_word_groups(e, ins.raw, pc)) |
|
| 1356 | 1771 | return false; |
|
| 1357 | 1772 | if (opcode == 0x57 && emit_vector(e, ins.raw)) |
|
| 1358 | 1773 | return false; |
|
| 1359 | 1774 | ||
| 1360 | 1775 | switch (opcode) { |
test/jit_vector.c
+148 -0
| 51 | 51 | return VECTOR_OK; |
|
| 52 | 52 | } |
|
| 53 | 53 | return vector_execute(v, x, ins, transfer, mem); |
|
| 54 | 54 | } |
|
| 55 | 55 | ||
| 56 | + | /* Native cases must complete without interpreter fallback. */ |
|
| 57 | + | static bool require_native; |
|
| 58 | + | /* Byte stride supplied to strided-store checks. */ |
|
| 59 | + | static u64 byte_stride = 3; |
|
| 60 | + | ||
| 56 | 61 | /* Compare all state, memory, faults and instruction accounting for one block. |
|
| 57 | 62 | */ |
|
| 58 | 63 | static void check( |
|
| 59 | 64 | const u32 *code, u32 count, u32 type, u32 vl, u32 start, u64 base |
|
| 60 | 65 | ) { |
| 74 | 79 | a.vtype = type; |
|
| 75 | 80 | a.vl = vl; |
|
| 76 | 81 | a.vstart = start; |
|
| 77 | 82 | b = a; |
|
| 78 | 83 | x[10] = y[10] = base; |
|
| 84 | + | x[11] = y[11] = byte_stride; |
|
| 85 | + | x[12] = y[12] = 0xa5; |
|
| 79 | 86 | memcpy(actual + CODE, code, count * 4); |
|
| 80 | 87 | u32 ret = 0x00008067; |
|
| 81 | 88 | memcpy(actual + CODE + count * 4, &ret, 4); |
|
| 82 | 89 | assert(jit_init(&jit)); |
|
| 83 | 90 | jit.io.vector = &a; |
| 93 | 100 | budget = jit.io.budget; |
|
| 94 | 101 | pc = jit.io.pc; |
|
| 95 | 102 | if (reason == JIT_EXIT_RET) |
|
| 96 | 103 | break; |
|
| 97 | 104 | if (reason == JIT_EXIT_FAULT) { |
|
| 105 | + | assert(!require_native); |
|
| 98 | 106 | u32 ins; |
|
| 99 | 107 | memcpy(&ins, actual + pc, 4); |
|
| 100 | 108 | enum vector_result result = step(&a, x, ins, actual); |
|
| 101 | 109 | if (result != VECTOR_OK) { |
|
| 102 | 110 | failed = true; |
| 125 | 133 | (type & ~0xc0u) == 0x11 && vl == 4 && start == 0) |
|
| 126 | 134 | assert(jit.block_count == 1); |
|
| 127 | 135 | jit_destroy(&jit); |
|
| 128 | 136 | } |
|
| 129 | 137 | ||
| 138 | + | /* Check byte transfers, fills, configuration, and fault boundaries. */ |
|
| 139 | + | static void check_bulk(void) { |
|
| 140 | + | u32 load = 1u << 25 | 10 << 15 | 8 << 7 | 7; |
|
| 141 | + | u32 store = (load & ~127u) | 39; |
|
| 142 | + | u32 stride = store | 2u << 26 | 11 << 20; |
|
| 143 | + | u32 fill = alu(23, 4, 8, 0, 12); |
|
| 144 | + | u32 zero = alu(23, 3, 8, 0, 0); |
|
| 145 | + | u32 copies[] = { load, 0x10050513, store }; |
|
| 146 | + | u32 fills[] = { fill, store, zero, stride }; |
|
| 147 | + | u32 lengths[] = { 0, 1, 3, 15, 16, 17, 31, 32, |
|
| 148 | + | 33, 63, 64, 65, 127, 128, 255, 256 }; |
|
| 149 | + | require_native = true; |
|
| 150 | + | for (u32 i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) { |
|
| 151 | + | check(copies, 3, 0xc3, lengths[i], 0, 128); |
|
| 152 | + | check(fills, 4, 0xc3, lengths[i], 0, 128); |
|
| 153 | + | } |
|
| 154 | + | u32 config = 0xc3 << 20 | 10 << 15 | 7 << 12 | 5 << 7 | 0x57; |
|
| 155 | + | u32 configured[] = { config, fill, store }; |
|
| 156 | + | check(configured, 3, 0xd1, 4, 3, 128); |
|
| 157 | + | for (u32 group = 0; group < 32; group += 8) { |
|
| 158 | + | u32 ops[] = { (load & ~(31 << 7)) | group << 7, |
|
| 159 | + | (store & ~(31 << 7)) | group << 7 }; |
|
| 160 | + | check(ops, 2, 0xc3, 256, 0, DATA - 256); |
|
| 161 | + | } |
|
| 162 | + | for (u32 policy = 0; policy <= 0xc0; policy += 0x40) { |
|
| 163 | + | check(copies, 3, policy | 3, 17, 0, 128); |
|
| 164 | + | check(fills, 4, policy | 3, 17, 0, 128); |
|
| 165 | + | } |
|
| 166 | + | u32 negative = alu(23, 3, 8, 0, 31); |
|
| 167 | + | check(&negative, 1, 0xc3, 255, 0, 128); |
|
| 168 | + | for (u32 i = 0; i < 128; i++) { |
|
| 169 | + | check(copies, 3, 0xc3, random_word() % 257, 0, 128); |
|
| 170 | + | check(fills, 4, 0xc3, random_word() % 257, 0, 128); |
|
| 171 | + | } |
|
| 172 | + | u32 block[JIT_MAX_BLOCK_INSNS]; |
|
| 173 | + | for (u32 i = 0; i < JIT_MAX_BLOCK_INSNS; i++) |
|
| 174 | + | block[i] = fill; |
|
| 175 | + | check(block, JIT_MAX_BLOCK_INSNS, 0xc3, 17, 0, 128); |
|
| 176 | + | require_native = false; |
|
| 177 | + | for (u32 vl = 0; vl <= 256; vl += 16) { |
|
| 178 | + | check(&load, 1, 0xc3, vl, 1, DATA - 64); |
|
| 179 | + | check(&store, 1, 0xc3, vl, 1, DATA - 64); |
|
| 180 | + | check(&stride, 1, 0xc3, vl, 1, DATA - 64); |
|
| 181 | + | } |
|
| 182 | + | u64 bounds[] = { |
|
| 183 | + | DATA - 256, DATA - 255, DATA - 1, DATA, DATA + 1, ~0ULL - 7 |
|
| 184 | + | }; |
|
| 185 | + | for (u32 i = 0; i < sizeof(bounds) / sizeof(bounds[0]); i++) { |
|
| 186 | + | check(&load, 1, 0xc3, 256, 0, bounds[i]); |
|
| 187 | + | check(&store, 1, 0xc3, 256, 0, bounds[i]); |
|
| 188 | + | check(&stride, 1, 0xc3, 256, 0, bounds[i]); |
|
| 189 | + | require_native = true; |
|
| 190 | + | check(&store, 1, 0xc3, 0, 0, bounds[i]); |
|
| 191 | + | require_native = false; |
|
| 192 | + | } |
|
| 193 | + | u64 strides[] = { 0, 1, 2, 8, ~0ULL, ~7ULL, 1ULL << 63 }; |
|
| 194 | + | for (u32 i = 0; i < sizeof(strides) / sizeof(strides[0]); i++) { |
|
| 195 | + | byte_stride = strides[i]; |
|
| 196 | + | require_native = byte_stride <= 8; |
|
| 197 | + | check(&stride, 1, 0xc3, 16, 0, 128); |
|
| 198 | + | require_native = true; |
|
| 199 | + | check(&stride, 1, 0xc3, 1, 0, DATA - 1); |
|
| 200 | + | require_native = false; |
|
| 201 | + | check(&stride, 1, 0xc3, 16, 0, DATA - 4); |
|
| 202 | + | } |
|
| 203 | + | byte_stride = 3; |
|
| 204 | + | u32 types[] = { 0, 1, 2, 3, 5, 6, 7, 0x10, 0x11, 0x13, 0x18, 0x1b }; |
|
| 205 | + | for (u32 i = 0; i < sizeof(types) / sizeof(types[0]); i++) { |
|
| 206 | + | u32 set = (config & ~(0x7ffu << 20)) | types[i] << 20; |
|
| 207 | + | require_native = true; |
|
| 208 | + | check(&set, 1, 0xd1, 4, 2, 0); |
|
| 209 | + | check(&set, 1, 0xd1, 4, 2, ~0ULL); |
|
| 210 | + | set = (set & ~(31u << 7)) | 10 << 7; |
|
| 211 | + | check(&set, 1, 0xd1, 4, 2, 128); |
|
| 212 | + | set &= ~(31u << 15); |
|
| 213 | + | check(&set, 1, 0xd1, 4, 2, 128); |
|
| 214 | + | for (u32 avl = 0; avl <= 31; avl++) { |
|
| 215 | + | u32 immediate = (set & ~(31u << 15)) | 3u << 30 | avl << 15; |
|
| 216 | + | check(&immediate, 1, 0xd1, 4, 2, 128); |
|
| 217 | + | } |
|
| 218 | + | } |
|
| 219 | + | require_native = false; |
|
| 220 | + | u32 keep = config & ~(31u << 15 | 31u << 7); |
|
| 221 | + | check(&keep, 1, 0xc3, 16, 0, 128); |
|
| 222 | + | check(&keep, 1, 0xd1, 4, 0, 128); |
|
| 223 | + | u32 bad_types[] = { 4, 0x20, 0x100, 0x3f }; |
|
| 224 | + | for (u32 i = 0; i < 4; i++) { |
|
| 225 | + | u32 set = (config & ~(0x7ffu << 20)) | bad_types[i] << 20; |
|
| 226 | + | check(&set, 1, 0xd1, 4, 0, 128); |
|
| 227 | + | } |
|
| 228 | + | u32 invalid[] = { load & ~(1u << 25), store | 1 << 7, fill | 1 << 7 }; |
|
| 229 | + | for (u32 i = 0; i < 3; i++) |
|
| 230 | + | check(invalid + i, 1, 0xc3, 16, 0, 128); |
|
| 231 | + | check(copies, 3, 0xd1, 4, 0, 128); |
|
| 232 | + | check(fills, 4, 0x80000003, 4, 0, 128); |
|
| 233 | + | u32 mixed[] = { 0x00450513, store }; |
|
| 234 | + | check(mixed, 2, 0xc3, 17, 0, DATA - 16); |
|
| 235 | + | check(mixed, 2, 0xc3, 256, 0, ~0ULL - 7); |
|
| 236 | + | } |
|
| 237 | + | ||
| 238 | + | /* Check every vector operation in the bitset and BLAKE3 assembly helpers. */ |
|
| 239 | + | static void check_word_groups(void) { |
|
| 240 | + | u32 load = 1u << 25 | 10 << 15 | 6 << 12 | 8 << 7 | 7; |
|
| 241 | + | u32 store = (load & ~127u) | 39; |
|
| 242 | + | u32 compare = alu(25, 0, 0, 8, 16); |
|
| 243 | + | u32 population = alu(16, 2, 5, 0, 16); |
|
| 244 | + | u32 ops[] = { load, alu(10, 0, 8, 8, 16), alu(1, 0, 8, 8, 16), |
|
| 245 | + | compare, population, store }; |
|
| 246 | + | require_native = true; |
|
| 247 | + | for (u32 lm = 2; lm <= 3; lm++) { |
|
| 248 | + | for (u32 vl = 0; vl <= ((u32)VECTOR_BYTES << lm) / 4; vl++) |
|
| 249 | + | check(ops, 6, 0xd0 | lm, vl, 0, 128); |
|
| 250 | + | } |
|
| 251 | + | u32 equal[] = { alu(25, 0, 0, 8, 8), population }; |
|
| 252 | + | u32 clear[] = { |
|
| 253 | + | alu(1, 0, 8, 8, 8), alu(10, 0, 8, 8, 8), compare, population |
|
| 254 | + | }; |
|
| 255 | + | for (u32 vl = 0; vl <= 64; vl++) { |
|
| 256 | + | check(equal, 2, 0xd3, vl, 0, 128); |
|
| 257 | + | check(clear, 4, 0xd3, vl, 0, 128); |
|
| 258 | + | } |
|
| 259 | + | u32 id = alu(20, 2, 2, 0, 17); |
|
| 260 | + | check(&id, 1, 0xd1, 4, 0, 128); |
|
| 261 | + | require_native = false; |
|
| 262 | + | check(ops, 6, 0xd3, 17, 1, 128); |
|
| 263 | + | check(ops, 6, 0xd3, 64, 0, DATA - 63); |
|
| 264 | + | check(ops, 6, 0xd3, 64, 0, ~0ULL - 7); |
|
| 265 | + | u32 alias[] = { alu(1, 0, 8, 16, 8), |
|
| 266 | + | alu(10, 0, 16, 16, 8), |
|
| 267 | + | alu(25, 0, 8, 8, 16), |
|
| 268 | + | alu(16, 2, 5, 8, 16) }; |
|
| 269 | + | require_native = true; |
|
| 270 | + | check(alias, 4, 0xd3, 33, 0, 128); |
|
| 271 | + | require_native = false; |
|
| 272 | + | u32 invalid = alu(25, 0, 9, 8, 16); |
|
| 273 | + | check(&invalid, 1, 0xd3, 33, 0, 128); |
|
| 274 | + | } |
|
| 275 | + | ||
| 130 | 276 | int main(void) { |
|
| 277 | + | check_bulk(); |
|
| 278 | + | check_word_groups(); |
|
| 131 | 279 | u32 ops[] = { |
|
| 132 | 280 | alu(0, 0, 8, 10, 12), alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0), |
|
| 133 | 281 | alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31), alu(9, 3, 8, 8, 3), |
|
| 134 | 282 | alu(12, 0, 16, 14, 24), alu(0, 0, 8, 8, 16), |
|
| 135 | 283 | }; |