jit: Translate bulk and bitset vector operations

edb99b5386968a77f4c53d82d5e025c4699adb7dbb0e703ddaa8e03abc52cecd
Translate byte fills, contiguous byte and word transfers, strided byte
stores, word bitset operations, mask comparisons and population counts,
and lane indexes. Select static vector configurations in native code.

Check group alignment, active lengths and restart state before access.
Handle exceptional memory ranges through the interpreter with precise
guest instruction accounting.

Change-Id: If1a15e6d0b50c9840c5ad1eaed644c0e6a4f7759
Alexis Sellier committed ago 1 parent 137a6e3c
jit.c +503 -88
11 11
 *       r12 = &regs[0]   (guest register file)
12 12
 *       r13 = memory base
13 13
 *       r14 = &jit->io   (exit information; pc is its first field)
14 14
 *       r15 = budget     (guest instructions left, signed)
15 15
 *   - Guest registers are loaded/stored on demand from/to the array.
16 -
 *   - Vector blocks guard the four-word e32,m2 shape before any side effect.
17 -
 *     Other shapes return to the interpreter.
16 +
 *   - Vector operations check their element width, group size, active length
17 +
 *     and restart index before access.
18 18
 *   - Each block starts with "if budget <= 0 exit at this PC; budget -=
19 19
 *     insn_count".  Every exit is the block's last instruction, so a block
20 20
 *     always accounts exactly the instructions it executes.
21 21
 *   - Direct jumps (jal, both sides of a branch, block split) are a jmp
22 22
 *     rel32 that first falls into a stub returning JIT_EXIT_CHAIN.  C then
765 765
    emit_rexw(e);
766 766
    emit_u8(e, 0x89);
767 767
    emit_u8(e, modrm(0, X_RCX, X_RAX));
768 768
}
769 769
770 +
/* Count the low 32 or 64 bits in rax and return the result in rax. */
771 +
static void emit_popcount(struct emitter *e, bool wide) {
772 +
    if (__builtin_cpu_supports("popcnt")) {
773 +
        emit_u8(e, 0xf3);
774 +
        if (wide)
775 +
            emit_rexw(e);
776 +
        emit_u8(e, 0x0f);
777 +
        emit_u8(e, 0xb8);
778 +
        emit_u8(e, 0xc0);
779 +
    } else {
780 +
        if (!wide) {
781 +
            emit_u8(e, 0x89);
782 +
            emit_u8(e, 0xc0);
783 +
        }
784 +
        /* Clear one set bit per iteration and count in RCX. */
785 +
        emit_u8(e, 0x31);
786 +
        emit_u8(e, 0xc9);
787 +
        emit_rexw(e);
788 +
        emit_u8(e, 0x85);
789 +
        emit_u8(e, 0xc0);
790 +
        emit_u8(e, 0x74);
791 +
        emit_u8(e, 12);
792 +
        emit_rexw(e);
793 +
        emit_u8(e, 0x8d);
794 +
        emit_u8(e, 0x50);
795 +
        emit_u8(e, 0xff);
796 +
        emit_rexw(e);
797 +
        emit_u8(e, 0xff);
798 +
        emit_u8(e, 0xc1);
799 +
        emit_rexw(e);
800 +
        emit_u8(e, 0x21);
801 +
        emit_u8(e, 0xd0);
802 +
        emit_u8(e, 0x75);
803 +
        emit_u8(e, (u8)-12);
804 +
        emit_rexw(e);
805 +
        emit_u8(e, 0x89);
806 +
        emit_u8(e, 0xc8);
807 +
    }
808 +
}
809 +
770 810
/* Emit a required bit manipulation instruction using baseline x86-64. */
771 811
static void emit_bitmanip(struct emitter *e, instr_t ins, enum bitmanip_op op) {
772 812
    if (ins.r.rd == ZERO)
773 813
        return;
774 814
    if (op == BM_SEXTB || op == BM_SEXTH || op == BM_ZEXTH) {
1038 1078
        }
1039 1079
        break;
1040 1080
    }
1041 1081
    case BM_CPOP:
1042 1082
    case BM_CPOPW: {
1043 -
        if (__builtin_cpu_supports("popcnt")) {
1044 -
            emit_u8(e, 0xf3);
1045 -
            if (op == BM_CPOP)
1046 -
                emit_rexw(e);
1047 -
            emit_u8(e, 0x0f);
1048 -
            emit_u8(e, 0xb8);
1049 -
            emit_u8(e, 0xc0);
1050 -
        } else {
1051 -
            if (op == BM_CPOPW) {
1052 -
                emit_u8(e, 0x89);
1053 -
                emit_u8(e, 0xc0);
1054 -
            }
1055 -
            /* Clear one set bit per iteration and count in RCX. */
1056 -
            emit_u8(e, 0x31);
1057 -
            emit_u8(e, 0xc9);
1058 -
            emit_rexw(e);
1059 -
            emit_u8(e, 0x85);
1060 -
            emit_u8(e, 0xc0);
1061 -
            emit_u8(e, 0x74);
1062 -
            emit_u8(e, 12);
1063 -
            emit_rexw(e);
1064 -
            emit_u8(e, 0x8d);
1065 -
            emit_u8(e, 0x50);
1066 -
            emit_u8(e, 0xff);
1067 -
            emit_rexw(e);
1068 -
            emit_u8(e, 0xff);
1069 -
            emit_u8(e, 0xc1);
1070 -
            emit_rexw(e);
1071 -
            emit_u8(e, 0x21);
1072 -
            emit_u8(e, 0xd0);
1073 -
            emit_u8(e, 0x75);
1074 -
            emit_u8(e, (u8)-12);
1075 -
            emit_rexw(e);
1076 -
            emit_u8(e, 0x89);
1077 -
            emit_u8(e, 0xc8);
1078 -
        }
1083 +
        emit_popcount(e, op == BM_CPOP);
1079 1084
        break;
1080 1085
    }
1081 1086
    case BM_ORCB:
1082 1087
        break;
1083 1088
    case BM_NONE:
1113 1118
    emit_u8(e, 0x8b);
1114 1119
    emit_u8(e, modrm(1, X_RAX, HIO));
1115 1120
    emit_u8(e, offsetof(struct jit_io, vector));
1116 1121
}
1117 1122
1123 +
/* Guard integral groups at one element width and leave vl in rcx.
1124 +
 * The register mask limits LMUL to the alignment of every operand group. */
1125 +
static void emit_group_guard(struct emitter *e, u32 pc, u32 width, u32 regs) {
1126 +
    u32 lm = 0;
1127 +
    while (lm < 3 && !(regs & (1u << lm)))
1128 +
        lm++;
1129 +
    emit_vector_base(e);
1130 +
    emit_rexw(e);
1131 +
    emit_u8(e, 0x8b);
1132 +
    emit_u8(e, modrm(2, X_RCX, X_RAX));
1133 +
    emit_u32(e, offsetof(struct vector_state, vtype));
1134 +
    emit_rexw(e);
1135 +
    emit_u8(e, 0x89);
1136 +
    emit_u8(e, 0xca); /* mov rdx, rcx */
1137 +
    emit_rexw(e);
1138 +
    emit_u8(e, 0x81);
1139 +
    emit_u8(e, 0xe2);
1140 +
    emit_u32(e, ~0xc7u);
1141 +
    emit_rexw(e);
1142 +
    emit_u8(e, 0x83);
1143 +
    emit_u8(e, 0xfa);
1144 +
    emit_u8(e, width << 3);
1145 +
    emit_vector_fault(e, pc, 0x85);
1146 +
    emit_rexw(e);
1147 +
    emit_u8(e, 0x83);
1148 +
    emit_u8(e, 0xe1);
1149 +
    emit_u8(e, 7);
1150 +
    emit_rexw(e);
1151 +
    emit_u8(e, 0x83);
1152 +
    emit_u8(e, 0xf9);
1153 +
    emit_u8(e, lm);
1154 +
    emit_vector_fault(e, pc, 0x87);
1155 +
    emit_mov_imm32_sx(e, X_RDX, VECTOR_BYTES >> width);
1156 +
    emit_rexw(e);
1157 +
    emit_u8(e, 0xd3);
1158 +
    emit_u8(e, 0xe2); /* shl rdx, cl */
1159 +
    emit_rexw(e);
1160 +
    emit_u8(e, 0x83);
1161 +
    emit_u8(e, modrm(2, 7, X_RAX));
1162 +
    emit_u32(e, offsetof(struct vector_state, vstart));
1163 +
    emit_u8(e, 0);
1164 +
    emit_vector_fault(e, pc, 0x85);
1165 +
    emit_rexw(e);
1166 +
    emit_u8(e, 0x8b);
1167 +
    emit_u8(e, modrm(2, X_RCX, X_RAX));
1168 +
    emit_u32(e, offsetof(struct vector_state, vl));
1169 +
    emit_rexw(e);
1170 +
    emit_u8(e, 0x39);
1171 +
    emit_u8(e, 0xd1); /* cmp rcx, rdx */
1172 +
    emit_vector_fault(e, pc, 0x87);
1173 +
}
1174 +
1175 +
/* Form a pointer into one architectural register group. */
1176 +
static void emit_group_pointer(struct emitter *e, enum x86reg dst, u32 reg) {
1177 +
    emit_rexw(e);
1178 +
    emit_u8(e, 0x8d);
1179 +
    emit_u8(e, modrm(2, dst, X_RAX));
1180 +
    emit_u32(e, reg * VECTOR_BYTES);
1181 +
}
1182 +
1183 +
/* Select a static vector type and end the block after its state change. */
1184 +
static bool emit_vector_config(struct emitter *e, u32 ins, u32 pc) {
1185 +
    bool immediate = (ins >> 30) == 3;
1186 +
    if ((!immediate && (ins >> 31)) || ((ins >> 12) & 7) != 7)
1187 +
        return false;
1188 +
    u32 type = (ins >> 20) & (immediate ? 1023 : 2047);
1189 +
    u32 sew = 8u << ((type >> 3) & 7), lm = type & 7;
1190 +
    u32 src = (ins >> 15) & 31, dst = (ins >> 7) & 31;
1191 +
    if ((type & ~255u) || sew > 64 || lm == 4 || (!immediate && !src && !dst))
1192 +
        return false;
1193 +
    u32 bytes = lm < 4 ? VECTOR_BYTES << lm : VECTOR_BYTES >> (8 - lm);
1194 +
    if (bytes * 8 < sew)
1195 +
        return false;
1196 +
    u32 max = bytes * 8 / sew;
1197 +
    if (immediate) {
1198 +
        emit_mov_imm32_sx(e, X_RAX, src < max ? src : max);
1199 +
    } else if (!src) {
1200 +
        emit_mov_imm32_sx(e, X_RAX, max);
1201 +
    } else {
1202 +
        emit_load_guest(e, X_RAX, src);
1203 +
        emit_mov_imm32_sx(e, X_RCX, max);
1204 +
        emit_alu_rax_rcx(e, 0x39);
1205 +
        emit_rexw(e);
1206 +
        emit_u8(e, 0x0f);
1207 +
        emit_u8(e, 0x47);
1208 +
        emit_u8(e, 0xc1); /* cmova rax, rcx */
1209 +
    }
1210 +
    emit_store_guest(e, dst, X_RAX);
1211 +
    emit_rexw(e);
1212 +
    emit_u8(e, 0x89);
1213 +
    emit_u8(e, 0xc2);
1214 +
    emit_vector_base(e);
1215 +
    emit_rexw(e);
1216 +
    emit_u8(e, 0x89);
1217 +
    emit_u8(e, modrm(2, X_RDX, X_RAX));
1218 +
    emit_u32(e, offsetof(struct vector_state, vl));
1219 +
    emit_rexw(e);
1220 +
    emit_u8(e, 0xc7);
1221 +
    emit_u8(e, modrm(2, 0, X_RAX));
1222 +
    emit_u32(e, offsetof(struct vector_state, vtype));
1223 +
    emit_u32(e, type);
1224 +
    emit_rexw(e);
1225 +
    emit_u8(e, 0xc7);
1226 +
    emit_u8(e, modrm(2, 0, X_RAX));
1227 +
    emit_u32(e, offsetof(struct vector_state, vstart));
1228 +
    emit_u32(e, 0);
1229 +
    emit_chain_exit(e, pc + INSTR_SIZE);
1230 +
    return true;
1231 +
}
1232 +
1233 +
/* Fill active bytes of a register group from a scalar or signed immediate. */
1234 +
static bool emit_byte_splat(struct emitter *e, u32 ins, u32 pc) {
1235 +
    u32 form = (ins >> 12) & 7, reg = (ins >> 7) & 31;
1236 +
    if ((ins & 0xfff0007f) != 0x5e000057 || (form != 3 && form != 4))
1237 +
        return false;
1238 +
    emit_group_guard(e, pc, 0, reg);
1239 +
    emit_group_pointer(e, X_RDI, reg);
1240 +
    u32 src = (ins >> 15) & 31;
1241 +
    if (form == 4)
1242 +
        emit_load_guest(e, X_RAX, src);
1243 +
    else
1244 +
        emit_mov_imm32_sx(e, X_RAX, (i32)(src << 27) >> 27);
1245 +
    emit_u8(e, 0xf3);
1246 +
    emit_u8(e, 0xaa); /* rep stosb */
1247 +
    return true;
1248 +
}
1249 +
1118 1250
/* Form one indexed guest address, with overflow checked before any transfer. */
1119 1251
static void emit_vector_address(struct emitter *e, u32 ins, u32 lane, u32 pc) {
1120 1252
    u32 base = (ins >> 15) & 31, index = (ins >> 20) & 31;
1121 1253
    emit_vector_base(e);
1122 1254
    emit_u8(e, 0x8b);
1144 1276
    emit_u8(e, 0x39);
1145 1277
    emit_u8(e, 0xc8);
1146 1278
    emit_vector_fault(e, pc, 0x87); /* ja */
1147 1279
}
1148 1280
1281 +
/* Transfer active bytes with complete bounds checks before any guest write. */
1282 +
static bool emit_group_memory(struct emitter *e, u32 ins, u32 pc) {
1283 +
    u32  op = ins & 127, reg = (ins >> 7) & 31;
1284 +
    u32  mode = (ins >> 26) & 3, index = (ins >> 20) & 31;
1285 +
    u32  field = (ins >> 12) & 7, width = field == 6 ? 2 : 0;
1286 +
    bool store = op == 39;
1287 +
    if ((op != 7 && !store) || (field != 0 && field != 6) ||
1288 +
        !(ins & (1u << 25)) || (ins >> 29) || (ins & (1u << 28)) ||
1289 +
        !((mode == 0 && index == 0) || (mode == 2 && store && width == 0)))
1290 +
        return false;
1291 +
    emit_group_guard(e, pc, width, reg);
1292 +
    if (width) {
1293 +
        emit_rexw(e);
1294 +
        emit_u8(e, 0xc1);
1295 +
        emit_u8(e, 0xe1);
1296 +
        emit_u8(e, width);
1297 +
    }
1298 +
    emit_rexw(e);
1299 +
    emit_u8(e, 0x85);
1300 +
    emit_u8(e, 0xc9);
1301 +
    u32 empty = e->pos;
1302 +
    emit_u8(e, 0x0f);
1303 +
    emit_u8(e, 0x84);
1304 +
    emit_u32(e, 0);
1305 +
    if (mode == 0) {
1306 +
        emit_group_pointer(e, store ? X_RSI : X_RDI, reg);
1307 +
        emit_load_guest(e, X_RAX, (ins >> 15) & 31);
1308 +
        emit_u8(e, 0x49);
1309 +
        emit_u8(e, 0x8b);
1310 +
        emit_u8(e, modrm(1, X_RDX, HIO));
1311 +
        emit_u8(e, offsetof(struct jit_io, memory_size));
1312 +
        emit_rexw(e);
1313 +
        emit_u8(e, 0x29);
1314 +
        emit_u8(e, 0xca); /* sub rdx, rcx */
1315 +
        emit_vector_fault(e, pc, 0x82);
1316 +
        emit_rexw(e);
1317 +
        emit_u8(e, 0x39);
1318 +
        emit_u8(e, 0xd0); /* cmp rax, rdx */
1319 +
        emit_vector_fault(e, pc, 0x87);
1320 +
        emit_add_rax_r13(e);
1321 +
        emit_rexw(e);
1322 +
        emit_u8(e, 0x89);
1323 +
        emit_u8(e, store ? 0xc7 : 0xc6);
1324 +
        emit_u8(e, 0xf3);
1325 +
        emit_u8(e, 0xa4); /* rep movsb */
1326 +
    } else {
1327 +
        emit_load_guest(e, X_RDI, (ins >> 15) & 31);
1328 +
        emit_load_guest(e, X_RSI, index);
1329 +
        emit_rexw(e);
1330 +
        emit_u8(e, 0x89);
1331 +
        emit_u8(e, 0xc8);
1332 +
        emit_rexw(e);
1333 +
        emit_u8(e, 0xff);
1334 +
        emit_u8(e, 0xc8); /* dec rax */
1335 +
        emit_rexw(e);
1336 +
        emit_u8(e, 0xf7);
1337 +
        emit_u8(e, 0xe6); /* mul rsi */
1338 +
        emit_rexw(e);
1339 +
        emit_u8(e, 0x85);
1340 +
        emit_u8(e, 0xd2);
1341 +
        emit_vector_fault(e, pc, 0x85);
1342 +
        emit_rexw(e);
1343 +
        emit_u8(e, 0x01);
1344 +
        emit_u8(e, 0xf8); /* add rax, rdi */
1345 +
        emit_vector_fault(e, pc, 0x82);
1346 +
        emit_vector_bound(e, 1, pc);
1347 +
        emit_rexw(e);
1348 +
        emit_u8(e, 0x89);
1349 +
        emit_u8(e, 0xf2); /* mov rdx, rsi */
1350 +
        emit_vector_base(e);
1351 +
        emit_rexw(e);
1352 +
        emit_u8(e, 0x8b);
1353 +
        emit_u8(e, modrm(2, X_RCX, X_RAX));
1354 +
        emit_u32(e, offsetof(struct vector_state, vl));
1355 +
        emit_group_pointer(e, X_RSI, reg);
1356 +
        emit_u8(e, 0x4c);
1357 +
        emit_u8(e, 0x01);
1358 +
        emit_u8(e, 0xef); /* add rdi, r13 */
1359 +
        u32 loop = e->pos;
1360 +
        emit_u8(e, 0xac); /* lodsb */
1361 +
        emit_u8(e, 0x88);
1362 +
        emit_u8(e, 0x07);
1363 +
        emit_rexw(e);
1364 +
        emit_u8(e, 0x01);
1365 +
        emit_u8(e, 0xd7);
1366 +
        emit_u8(e, 0xe2);
1367 +
        emit_u8(e, (u8)(loop - e->pos - 1));
1368 +
    }
1369 +
    patch_jcc(e, empty, e->pos);
1370 +
    return true;
1371 +
}
1372 +
1149 1373
/* Emit an SSE instruction on two XMM registers. */
1150 1374
static void emit_xmm(struct emitter *e, u8 opcode, u32 d, u32 src) {
1151 1375
    emit_u8(e, 0x66);
1152 1376
    emit_u8(e, 0x0f);
1153 1377
    emit_u8(e, opcode);
1162 1386
    emit_u8(e, store ? 0x7f : 0x6f);
1163 1387
    emit_u8(e, modrm(2, xmm, X_RAX));
1164 1388
    emit_u32(e, reg * VECTOR_BYTES);
1165 1389
}
1166 1390
1167 -
/* Emit checked unmasked word loads and stores for one register group. */
1168 -
static bool emit_vector_memory(struct emitter *e, u32 ins, u32 pc) {
1169 -
    u32  op = ins & 127, reg = (ins >> 7) & 31, index = (ins >> 20) & 31;
1170 -
    u32  mode  = (ins >> 26) & 3;
1171 -
    bool store = op == 39;
1172 -
    if ((op != 7 && !store) || ((ins >> 12) & 7) != 6 || !(ins & (1u << 25)) ||
1173 -
        (ins >> 29) || (ins & (1u << 28)) || (reg & 1) ||
1174 -
        !((mode == 0 && index == 0) || (mode == 1 && !store && !(index & 1))))
1175 -
        return false;
1176 -
    if (mode == 1 && !store && reg == index)
1391 +
/* Emit checked unmasked indexed word loads for one register group. */
1392 +
static bool emit_indexed_words(struct emitter *e, u32 ins, u32 pc) {
1393 +
    u32 reg = (ins >> 7) & 31, index = (ins >> 20) & 31;
1394 +
    if ((ins & 127) != 7 || ((ins >> 12) & 7) != 6 || !(ins & (1u << 25)) ||
1395 +
        (ins >> 29) || (ins & (1u << 28)) || (reg & 1) || (index & 1) ||
1396 +
        ((ins >> 26) & 3) != 1 || reg == index)
1177 1397
        return false;
1178 1398
    e->vector = true;
1179 -
    if (mode == 0) {
1399 +
    /* Preflight every address so an exceptional access has no partial
1400 +
     * writes. */
1401 +
    for (u32 i = 0; i < 4; i++) {
1402 +
        emit_vector_address(e, ins, i, pc);
1403 +
        emit_vector_bound(e, 4, pc);
1404 +
    }
1405 +
    for (u32 i = 0; i < 4; i++) {
1406 +
        emit_vector_base(e);
1407 +
        emit_u8(e, 0x8b);
1408 +
        emit_u8(e, modrm(2, X_RDX, X_RAX));
1409 +
        emit_u32(e, index * VECTOR_BYTES + i * 4);
1180 1410
        emit_load_guest(e, X_RAX, (ins >> 15) & 31);
1181 -
        emit_vector_bound(e, 16, pc);
1182 -
        if (store) {
1183 -
            emit_vector_base(e);
1184 -
            emit_xmm_memory(e, 0, reg, false);
1185 -
            emit_load_guest(e, X_RAX, (ins >> 15) & 31);
1186 -
        }
1411 +
        emit_u8(e, 0x48);
1412 +
        emit_u8(e, 0x01);
1413 +
        emit_u8(e, 0xd0);
1187 1414
        emit_add_rax_r13(e);
1188 -
        emit_u8(e, 0xf3);
1189 -
        emit_u8(e, 0x0f);
1190 -
        emit_u8(e, store ? 0x7f : 0x6f);
1191 -
        emit_u8(e, 0x00);
1192 -
        if (!store) {
1193 -
            emit_vector_base(e);
1194 -
            emit_xmm_memory(e, 0, reg, true);
1195 -
        }
1196 -
    } else {
1197 -
        /* Preflight every address so an exceptional access has no partial
1198 -
         * writes. */
1199 -
        for (u32 i = 0; i < 4; i++) {
1200 -
            emit_vector_address(e, ins, i, pc);
1201 -
            emit_vector_bound(e, 4, pc);
1202 -
        }
1203 -
        for (u32 i = 0; i < 4; i++) {
1204 -
            emit_vector_base(e);
1205 -
            emit_u8(e, 0x8b);
1206 -
            emit_u8(e, modrm(2, X_RDX, X_RAX));
1207 -
            emit_u32(e, index * VECTOR_BYTES + i * 4);
1208 -
            emit_load_guest(e, X_RAX, (ins >> 15) & 31);
1209 -
            emit_u8(e, 0x48);
1210 -
            emit_u8(e, 0x01);
1211 -
            emit_u8(e, 0xd0);
1212 -
            emit_add_rax_r13(e);
1213 -
            emit_u8(e, 0x8b);
1214 -
            emit_u8(e, 0x10); /* mov edx, [rax] */
1215 -
            emit_vector_base(e);
1216 -
            emit_u8(e, 0x89);
1217 -
            emit_u8(e, modrm(2, X_RDX, X_RAX));
1218 -
            emit_u32(e, reg * VECTOR_BYTES + i * 4);
1219 -
        }
1415 +
        emit_u8(e, 0x8b);
1416 +
        emit_u8(e, 0x10); /* mov edx, [rax] */
1417 +
        emit_vector_base(e);
1418 +
        emit_u8(e, 0x89);
1419 +
        emit_u8(e, modrm(2, X_RDX, X_RAX));
1420 +
        emit_u32(e, reg * VECTOR_BYTES + i * 4);
1220 1421
    }
1221 1422
    return true;
1222 1423
}
1223 1424
1224 1425
/* Gather four words from a validated, non-overlapping register group. */
1246 1447
        emit_u8(e, modrm(2, X_RCX, X_RAX));
1247 1448
        emit_u32(e, d * VECTOR_BYTES + i * 4);
1248 1449
    }
1249 1450
}
1250 1451
1452 +
/* Emit a relative conditional jump to an existing native position. */
1453 +
static void emit_jump_back(struct emitter *e, u8 condition, u32 target) {
1454 +
    emit_u8(e, 0x0f);
1455 +
    emit_u8(e, condition);
1456 +
    emit_u32(e, target - e->pos - 4);
1457 +
}
1458 +
1459 +
/* Emit a zero-count branch after testing rcx. */
1460 +
static u32 emit_count_zero(struct emitter *e) {
1461 +
    emit_rexw(e);
1462 +
    emit_u8(e, 0x85);
1463 +
    emit_u8(e, 0xc9);
1464 +
    u32 pos = e->pos;
1465 +
    emit_u8(e, 0x0f);
1466 +
    emit_u8(e, 0x84);
1467 +
    emit_u32(e, 0);
1468 +
    return pos;
1469 +
}
1470 +
1471 +
/* Apply word-wise OR or AND-NOT in SIMD chunks, preserving inactive words. */
1472 +
static void emit_group_bits(struct emitter *e, u32 ins, u32 pc) {
1473 +
    u32  d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31;
1474 +
    bool invert = (ins >> 26) == 1;
1475 +
    emit_group_guard(e, pc, 2, d | a | b);
1476 +
    emit_group_pointer(e, X_RSI, a);
1477 +
    emit_group_pointer(e, X_RDI, b);
1478 +
    emit_group_pointer(e, X_RDX, d);
1479 +
    emit_rexw(e);
1480 +
    emit_u8(e, 0x83);
1481 +
    emit_u8(e, 0xf9);
1482 +
    emit_u8(e, 4);
1483 +
    u32 tail = e->pos;
1484 +
    emit_u8(e, 0x0f);
1485 +
    emit_u8(e, 0x82);
1486 +
    emit_u32(e, 0);
1487 +
    u32      loop    = e->pos;
1488 +
    const u8 loads[] = { 0xf3, 0x0f, 0x6f, 0x06, 0xf3, 0x0f, 0x6f, 0x0f };
1489 +
    for (u32 i = 0; i < sizeof(loads); i++)
1490 +
        emit_u8(e, loads[i]);
1491 +
    emit_xmm(e, invert ? 0xdf : 0xeb, invert ? 1 : 0, invert ? 0 : 1);
1492 +
    emit_u8(e, 0xf3);
1493 +
    emit_u8(e, 0x0f);
1494 +
    emit_u8(e, 0x7f);
1495 +
    emit_u8(e, modrm(0, invert ? 1 : 0, X_RDX));
1496 +
    const u32 pointers[] = { X_RSI, X_RDI, X_RDX };
1497 +
    for (u32 i = 0; i < 3; i++) {
1498 +
        emit_rexw(e);
1499 +
        emit_u8(e, 0x83);
1500 +
        emit_u8(e, modrm(3, 0, pointers[i]));
1501 +
        emit_u8(e, 16);
1502 +
    }
1503 +
    emit_rexw(e);
1504 +
    emit_u8(e, 0x83);
1505 +
    emit_u8(e, 0xe9);
1506 +
    emit_u8(e, 4);
1507 +
    emit_rexw(e);
1508 +
    emit_u8(e, 0x83);
1509 +
    emit_u8(e, 0xf9);
1510 +
    emit_u8(e, 4);
1511 +
    emit_jump_back(e, 0x83, loop);
1512 +
    patch_jcc(e, tail, e->pos);
1513 +
    u32 empty = emit_count_zero(e);
1514 +
    loop      = e->pos;
1515 +
    emit_u8(e, 0x8b);
1516 +
    emit_u8(e, invert ? 0x07 : 0x06);
1517 +
    if (invert) {
1518 +
        emit_u8(e, 0xf7);
1519 +
        emit_u8(e, 0xd0);
1520 +
    }
1521 +
    emit_u8(e, invert ? 0x23 : 0x0b);
1522 +
    emit_u8(e, invert ? 0x06 : 0x07);
1523 +
    emit_u8(e, 0x89);
1524 +
    emit_u8(e, 0x02);
1525 +
    for (u32 i = 0; i < 3; i++) {
1526 +
        emit_rexw(e);
1527 +
        emit_u8(e, 0x83);
1528 +
        emit_u8(e, modrm(3, 0, pointers[i]));
1529 +
        emit_u8(e, 4);
1530 +
    }
1531 +
    emit_u8(e, 0xe2);
1532 +
    emit_u8(e, (u8)(loop - e->pos - 1));
1533 +
    patch_jcc(e, empty, e->pos);
1534 +
}
1535 +
1536 +
/* Write active word comparisons into a mask after reading both source groups.
1537 +
 */
1538 +
static void emit_group_compare(struct emitter *e, u32 ins, u32 pc) {
1539 +
    u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31;
1540 +
    emit_group_guard(e, pc, 2, a | b);
1541 +
    emit_group_pointer(e, X_RSI, a);
1542 +
    emit_group_pointer(e, X_RDI, b);
1543 +
    emit_group_pointer(e, X_RDX, d);
1544 +
    u32      empty   = emit_count_zero(e);
1545 +
    const u8 begin[] = { 0x45, 0x31, 0xc0, 0x49, 0x89, 0xc9 };
1546 +
    for (u32 i = 0; i < sizeof(begin); i++)
1547 +
        emit_u8(e, begin[i]);
1548 +
    u32      loop      = e->pos;
1549 +
    const u8 compare[] = { 0x8b, 0x44, 0x8e, 0xfc, 0x3b, 0x44, 0x8f, 0xfc,
1550 +
                           0x0f, 0x95, 0xc0, 0x0f, 0xb6, 0xc0, 0x49, 0xd1,
1551 +
                           0xe0, 0x49, 0x09, 0xc0, 0x48, 0xff, 0xc9 };
1552 +
    for (u32 i = 0; i < sizeof(compare); i++)
1553 +
        emit_u8(e, compare[i]);
1554 +
    emit_jump_back(e, 0x85, loop);
1555 +
    emit_u8(e, 0x4c);
1556 +
    emit_u8(e, 0x89);
1557 +
    emit_u8(e, 0xc9); /* mov rcx, r9 */
1558 +
    emit_mov_imm32_sx(e, X_RAX, -1);
1559 +
    emit_rexw(e);
1560 +
    emit_u8(e, 0x83);
1561 +
    emit_u8(e, 0xf9);
1562 +
    emit_u8(e, 64);
1563 +
    u32 full = e->pos;
1564 +
    emit_u8(e, 0x0f);
1565 +
    emit_u8(e, 0x84);
1566 +
    emit_u32(e, 0);
1567 +
    emit_rexw(e);
1568 +
    emit_u8(e, 0xd3);
1569 +
    emit_u8(e, 0xe0);
1570 +
    u32 merge = emit_jmp_placeholder(e);
1571 +
    patch_jcc(e, full, e->pos);
1572 +
    emit_u8(e, 0x31);
1573 +
    emit_u8(e, 0xc0);
1574 +
    patch_jmp(e, merge, e->pos);
1575 +
    const u8 finish[] = {
1576 +
        0x48, 0x23, 0x02, 0x4c, 0x09, 0xc0, 0x48, 0x89, 0x02
1577 +
    };
1578 +
    for (u32 i = 0; i < sizeof(finish); i++)
1579 +
        emit_u8(e, finish[i]);
1580 +
    patch_jcc(e, empty, e->pos);
1581 +
}
1582 +
1583 +
/* Count the active bits of a mask register at word element width. */
1584 +
static void emit_mask_population(struct emitter *e, u32 ins, u32 pc) {
1585 +
    u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31;
1586 +
    emit_group_guard(e, pc, 2, 0);
1587 +
    emit_rexw(e);
1588 +
    emit_u8(e, 0x8b);
1589 +
    emit_u8(e, modrm(2, X_RAX, X_RAX));
1590 +
    emit_u32(e, a * VECTOR_BYTES);
1591 +
    emit_rexw(e);
1592 +
    emit_u8(e, 0x83);
1593 +
    emit_u8(e, 0xf9);
1594 +
    emit_u8(e, 64);
1595 +
    u32 full = e->pos;
1596 +
    emit_u8(e, 0x0f);
1597 +
    emit_u8(e, 0x84);
1598 +
    emit_u32(e, 0);
1599 +
    emit_mov_imm32_sx(e, X_RDX, -1);
1600 +
    emit_rexw(e);
1601 +
    emit_u8(e, 0xd3);
1602 +
    emit_u8(e, 0xe2);
1603 +
    emit_rexw(e);
1604 +
    emit_u8(e, 0xf7);
1605 +
    emit_u8(e, 0xd2);
1606 +
    emit_rexw(e);
1607 +
    emit_u8(e, 0x21);
1608 +
    emit_u8(e, 0xd0);
1609 +
    patch_jcc(e, full, e->pos);
1610 +
    emit_popcount(e, true);
1611 +
    emit_store_guest(e, d, X_RAX);
1612 +
}
1613 +
1614 +
/* Write consecutive word indexes into the active destination elements. */
1615 +
static void emit_lane_indexes(struct emitter *e, u32 ins, u32 pc) {
1616 +
    u32 d = (ins >> 7) & 31;
1617 +
    emit_group_guard(e, pc, 2, d);
1618 +
    emit_group_pointer(e, X_RDI, d);
1619 +
    u32 empty = emit_count_zero(e);
1620 +
    emit_u8(e, 0x31);
1621 +
    emit_u8(e, 0xd2);
1622 +
    u32      loop    = e->pos;
1623 +
    const u8 store[] = { 0x89, 0x17, 0x48, 0x83, 0xc7, 4, 0xff, 0xc2 };
1624 +
    for (u32 i = 0; i < sizeof(store); i++)
1625 +
        emit_u8(e, store[i]);
1626 +
    emit_u8(e, 0xe2);
1627 +
    emit_u8(e, (u8)(loop - e->pos - 1));
1628 +
    patch_jcc(e, empty, e->pos);
1629 +
}
1630 +
1631 +
/* Emit the word group operations used by bitsets and lane-index setup. */
1632 +
static bool emit_word_groups(struct emitter *e, u32 ins, u32 pc) {
1633 +
    u32 fn = ins >> 26, form = (ins >> 12) & 7;
1634 +
    u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31;
1635 +
    if (!(ins & (1u << 25)))
1636 +
        return false;
1637 +
    if (form == 0 && (fn == 1 || fn == 10)) {
1638 +
        emit_group_bits(e, ins, pc);
1639 +
        return true;
1640 +
    }
1641 +
    if (form == 0 && fn == 25) {
1642 +
        if ((d > a && d < a + 8) || (d > b && d < b + 8))
1643 +
            return false;
1644 +
        emit_group_compare(e, ins, pc);
1645 +
        return true;
1646 +
    }
1647 +
    if (form == 2 && fn == 16 && b == 16) {
1648 +
        emit_mask_population(e, ins, pc);
1649 +
        return true;
1650 +
    }
1651 +
    if (form == 2 && fn == 20 && b == 17 && a == 0) {
1652 +
        emit_lane_indexes(e, ins, pc);
1653 +
        return true;
1654 +
    }
1655 +
    return false;
1656 +
}
1657 +
1251 1658
/* Emit unmasked four-word integer operations under the block's state guard. */
1252 1659
static bool emit_vector(struct emitter *e, u32 ins) {
1253 1660
    u32 fn = ins >> 26, form = (ins >> 12) & 7;
1254 1661
    u32 d = (ins >> 7) & 31, a = (ins >> 20) & 31, b = (ins >> 15) & 31;
1255 1662
    if (!(ins & (1u << 25)) || (d & 1) || (a & 1))
1350 1757
    if (bm != BM_NONE) {
1351 1758
        emit_bitmanip(e, ins, bm);
1352 1759
        return false;
1353 1760
    }
1354 1761
1355 -
    if ((opcode == 7 || opcode == 39) && emit_vector_memory(e, ins.raw, pc))
1762 +
    if (opcode == 0x57 && emit_vector_config(e, ins.raw, pc))
1763 +
        return true;
1764 +
    if (opcode == 0x57 && emit_byte_splat(e, ins.raw, pc))
1765 +
        return false;
1766 +
    if ((opcode == 7 || opcode == 39) && emit_group_memory(e, ins.raw, pc))
1767 +
        return false;
1768 +
    if ((opcode == 7 || opcode == 39) && emit_indexed_words(e, ins.raw, pc))
1769 +
        return false;
1770 +
    if (opcode == 0x57 && emit_word_groups(e, ins.raw, pc))
1356 1771
        return false;
1357 1772
    if (opcode == 0x57 && emit_vector(e, ins.raw))
1358 1773
        return false;
1359 1774
1360 1775
    switch (opcode) {
test/jit_vector.c +148 -0
51 51
        return VECTOR_OK;
52 52
    }
53 53
    return vector_execute(v, x, ins, transfer, mem);
54 54
}
55 55
56 +
/* Native cases must complete without interpreter fallback. */
57 +
static bool require_native;
58 +
/* Byte stride supplied to strided-store checks. */
59 +
static u64 byte_stride = 3;
60 +
56 61
/* Compare all state, memory, faults and instruction accounting for one block.
57 62
 */
58 63
static void check(
59 64
    const u32 *code, u32 count, u32 type, u32 vl, u32 start, u64 base
60 65
) {
74 79
    a.vtype  = type;
75 80
    a.vl     = vl;
76 81
    a.vstart = start;
77 82
    b        = a;
78 83
    x[10] = y[10] = base;
84 +
    x[11] = y[11] = byte_stride;
85 +
    x[12] = y[12] = 0xa5;
79 86
    memcpy(actual + CODE, code, count * 4);
80 87
    u32 ret = 0x00008067;
81 88
    memcpy(actual + CODE + count * 4, &ret, 4);
82 89
    assert(jit_init(&jit));
83 90
    jit.io.vector      = &a;
93 100
        budget     = jit.io.budget;
94 101
        pc         = jit.io.pc;
95 102
        if (reason == JIT_EXIT_RET)
96 103
            break;
97 104
        if (reason == JIT_EXIT_FAULT) {
105 +
            assert(!require_native);
98 106
            u32 ins;
99 107
            memcpy(&ins, actual + pc, 4);
100 108
            enum vector_result result = step(&a, x, ins, actual);
101 109
            if (result != VECTOR_OK) {
102 110
                failed = true;
125 133
        (type & ~0xc0u) == 0x11 && vl == 4 && start == 0)
126 134
        assert(jit.block_count == 1);
127 135
    jit_destroy(&jit);
128 136
}
129 137
138 +
/* Check byte transfers, fills, configuration, and fault boundaries. */
139 +
static void check_bulk(void) {
140 +
    u32 load       = 1u << 25 | 10 << 15 | 8 << 7 | 7;
141 +
    u32 store      = (load & ~127u) | 39;
142 +
    u32 stride     = store | 2u << 26 | 11 << 20;
143 +
    u32 fill       = alu(23, 4, 8, 0, 12);
144 +
    u32 zero       = alu(23, 3, 8, 0, 0);
145 +
    u32 copies[]   = { load, 0x10050513, store };
146 +
    u32 fills[]    = { fill, store, zero, stride };
147 +
    u32 lengths[]  = { 0,  1,  3,  15, 16,  17,  31,  32,
148 +
                       33, 63, 64, 65, 127, 128, 255, 256 };
149 +
    require_native = true;
150 +
    for (u32 i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
151 +
        check(copies, 3, 0xc3, lengths[i], 0, 128);
152 +
        check(fills, 4, 0xc3, lengths[i], 0, 128);
153 +
    }
154 +
    u32 config       = 0xc3 << 20 | 10 << 15 | 7 << 12 | 5 << 7 | 0x57;
155 +
    u32 configured[] = { config, fill, store };
156 +
    check(configured, 3, 0xd1, 4, 3, 128);
157 +
    for (u32 group = 0; group < 32; group += 8) {
158 +
        u32 ops[] = { (load & ~(31 << 7)) | group << 7,
159 +
                      (store & ~(31 << 7)) | group << 7 };
160 +
        check(ops, 2, 0xc3, 256, 0, DATA - 256);
161 +
    }
162 +
    for (u32 policy = 0; policy <= 0xc0; policy += 0x40) {
163 +
        check(copies, 3, policy | 3, 17, 0, 128);
164 +
        check(fills, 4, policy | 3, 17, 0, 128);
165 +
    }
166 +
    u32 negative = alu(23, 3, 8, 0, 31);
167 +
    check(&negative, 1, 0xc3, 255, 0, 128);
168 +
    for (u32 i = 0; i < 128; i++) {
169 +
        check(copies, 3, 0xc3, random_word() % 257, 0, 128);
170 +
        check(fills, 4, 0xc3, random_word() % 257, 0, 128);
171 +
    }
172 +
    u32 block[JIT_MAX_BLOCK_INSNS];
173 +
    for (u32 i = 0; i < JIT_MAX_BLOCK_INSNS; i++)
174 +
        block[i] = fill;
175 +
    check(block, JIT_MAX_BLOCK_INSNS, 0xc3, 17, 0, 128);
176 +
    require_native = false;
177 +
    for (u32 vl = 0; vl <= 256; vl += 16) {
178 +
        check(&load, 1, 0xc3, vl, 1, DATA - 64);
179 +
        check(&store, 1, 0xc3, vl, 1, DATA - 64);
180 +
        check(&stride, 1, 0xc3, vl, 1, DATA - 64);
181 +
    }
182 +
    u64 bounds[] = {
183 +
        DATA - 256, DATA - 255, DATA - 1, DATA, DATA + 1, ~0ULL - 7
184 +
    };
185 +
    for (u32 i = 0; i < sizeof(bounds) / sizeof(bounds[0]); i++) {
186 +
        check(&load, 1, 0xc3, 256, 0, bounds[i]);
187 +
        check(&store, 1, 0xc3, 256, 0, bounds[i]);
188 +
        check(&stride, 1, 0xc3, 256, 0, bounds[i]);
189 +
        require_native = true;
190 +
        check(&store, 1, 0xc3, 0, 0, bounds[i]);
191 +
        require_native = false;
192 +
    }
193 +
    u64 strides[] = { 0, 1, 2, 8, ~0ULL, ~7ULL, 1ULL << 63 };
194 +
    for (u32 i = 0; i < sizeof(strides) / sizeof(strides[0]); i++) {
195 +
        byte_stride    = strides[i];
196 +
        require_native = byte_stride <= 8;
197 +
        check(&stride, 1, 0xc3, 16, 0, 128);
198 +
        require_native = true;
199 +
        check(&stride, 1, 0xc3, 1, 0, DATA - 1);
200 +
        require_native = false;
201 +
        check(&stride, 1, 0xc3, 16, 0, DATA - 4);
202 +
    }
203 +
    byte_stride = 3;
204 +
    u32 types[] = { 0, 1, 2, 3, 5, 6, 7, 0x10, 0x11, 0x13, 0x18, 0x1b };
205 +
    for (u32 i = 0; i < sizeof(types) / sizeof(types[0]); i++) {
206 +
        u32 set        = (config & ~(0x7ffu << 20)) | types[i] << 20;
207 +
        require_native = true;
208 +
        check(&set, 1, 0xd1, 4, 2, 0);
209 +
        check(&set, 1, 0xd1, 4, 2, ~0ULL);
210 +
        set = (set & ~(31u << 7)) | 10 << 7;
211 +
        check(&set, 1, 0xd1, 4, 2, 128);
212 +
        set &= ~(31u << 15);
213 +
        check(&set, 1, 0xd1, 4, 2, 128);
214 +
        for (u32 avl = 0; avl <= 31; avl++) {
215 +
            u32 immediate = (set & ~(31u << 15)) | 3u << 30 | avl << 15;
216 +
            check(&immediate, 1, 0xd1, 4, 2, 128);
217 +
        }
218 +
    }
219 +
    require_native = false;
220 +
    u32 keep       = config & ~(31u << 15 | 31u << 7);
221 +
    check(&keep, 1, 0xc3, 16, 0, 128);
222 +
    check(&keep, 1, 0xd1, 4, 0, 128);
223 +
    u32 bad_types[] = { 4, 0x20, 0x100, 0x3f };
224 +
    for (u32 i = 0; i < 4; i++) {
225 +
        u32 set = (config & ~(0x7ffu << 20)) | bad_types[i] << 20;
226 +
        check(&set, 1, 0xd1, 4, 0, 128);
227 +
    }
228 +
    u32 invalid[] = { load & ~(1u << 25), store | 1 << 7, fill | 1 << 7 };
229 +
    for (u32 i = 0; i < 3; i++)
230 +
        check(invalid + i, 1, 0xc3, 16, 0, 128);
231 +
    check(copies, 3, 0xd1, 4, 0, 128);
232 +
    check(fills, 4, 0x80000003, 4, 0, 128);
233 +
    u32 mixed[] = { 0x00450513, store };
234 +
    check(mixed, 2, 0xc3, 17, 0, DATA - 16);
235 +
    check(mixed, 2, 0xc3, 256, 0, ~0ULL - 7);
236 +
}
237 +
238 +
/* Check every vector operation in the bitset and BLAKE3 assembly helpers. */
239 +
static void check_word_groups(void) {
240 +
    u32 load       = 1u << 25 | 10 << 15 | 6 << 12 | 8 << 7 | 7;
241 +
    u32 store      = (load & ~127u) | 39;
242 +
    u32 compare    = alu(25, 0, 0, 8, 16);
243 +
    u32 population = alu(16, 2, 5, 0, 16);
244 +
    u32 ops[]      = { load,    alu(10, 0, 8, 8, 16), alu(1, 0, 8, 8, 16),
245 +
                       compare, population,           store };
246 +
    require_native = true;
247 +
    for (u32 lm = 2; lm <= 3; lm++) {
248 +
        for (u32 vl = 0; vl <= ((u32)VECTOR_BYTES << lm) / 4; vl++)
249 +
            check(ops, 6, 0xd0 | lm, vl, 0, 128);
250 +
    }
251 +
    u32 equal[] = { alu(25, 0, 0, 8, 8), population };
252 +
    u32 clear[] = {
253 +
        alu(1, 0, 8, 8, 8), alu(10, 0, 8, 8, 8), compare, population
254 +
    };
255 +
    for (u32 vl = 0; vl <= 64; vl++) {
256 +
        check(equal, 2, 0xd3, vl, 0, 128);
257 +
        check(clear, 4, 0xd3, vl, 0, 128);
258 +
    }
259 +
    u32 id = alu(20, 2, 2, 0, 17);
260 +
    check(&id, 1, 0xd1, 4, 0, 128);
261 +
    require_native = false;
262 +
    check(ops, 6, 0xd3, 17, 1, 128);
263 +
    check(ops, 6, 0xd3, 64, 0, DATA - 63);
264 +
    check(ops, 6, 0xd3, 64, 0, ~0ULL - 7);
265 +
    u32 alias[]    = { alu(1, 0, 8, 16, 8),
266 +
                       alu(10, 0, 16, 16, 8),
267 +
                       alu(25, 0, 8, 8, 16),
268 +
                       alu(16, 2, 5, 8, 16) };
269 +
    require_native = true;
270 +
    check(alias, 4, 0xd3, 33, 0, 128);
271 +
    require_native = false;
272 +
    u32 invalid    = alu(25, 0, 9, 8, 16);
273 +
    check(&invalid, 1, 0xd3, 33, 0, 128);
274 +
}
275 +
130 276
int main(void) {
277 +
    check_bulk();
278 +
    check_word_groups();
131 279
    u32 ops[] = {
132 280
        alu(0, 0, 8, 10, 12),   alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0),
133 281
        alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31),   alu(9, 3, 8, 8, 3),
134 282
        alu(12, 0, 16, 14, 24), alu(0, 0, 8, 8, 16),
135 283
    };