/* Differential checks of hosted vector translation and interpreter fallback. */ #include #include #include #include "jit.h" #include "profile.h" #include "vector.h" /* Test code lies outside the accessible data region. */ #define DATA 4096 #define CODE 8192 /* Transfer one interpreter element with the hosted bounds contract. */ static bool transfer(void *ctx, u64 addr, u32 size, bool store, u64 *value) { u8 *mem = ctx; if (addr > DATA || size > DATA - addr) return false; if (!store) *value = 0; for (u32 i = 0; i < size; i++) { if (store) mem[addr + i] = *value >> (i * 8); else *value |= (u64)mem[addr + i] << (i * 8); } return true; } /* A reproducible stream of operands and initial register bytes. */ static u32 random_word(void) { static u32 state = 0x25ab64d1; state ^= state << 13; state ^= state >> 17; state ^= state << 5; return state; } /* Encode one unmasked vector operation. */ static u32 alu(u32 fn, u32 form, u32 d, u32 a, u32 b) { return fn << 26 | 1u << 25 | a << 20 | b << 15 | form << 12 | d << 7 | 0x57; } /* Execute the scalar address update used before vector memory operations. */ static enum vector_result step( struct vector_state *v, u64 x[32], u32 ins, u8 *mem ) { if ((ins & 0x707f) == 0x13) { u32 rd = (ins >> 7) & 31, rs = (ins >> 15) & 31; if (rd) x[rd] = x[rs] + (i64)((i32)ins >> 20); return VECTOR_OK; } return vector_execute(v, x, ins, transfer, mem); } /* Native cases must complete without interpreter fallback. */ static bool require_native; /* A short vector copy stays in the same translated block as its setup. */ static bool require_single_block; /* Byte stride supplied to strided-store checks. */ static u64 byte_stride = 3; /* Compare all state, memory, faults and instruction accounting for one block. */ static void check( const u32 *code, u32 count, u32 type, u32 vl, u32 start, u64 base ) { static struct jit_state jit; u8 actual[16384] = { 0 }, expected[16384] = { 0 }; struct vector_state a = { 0 }, b; u64 x[32] = { 0 }, y[32] = { 0 }; for (u32 i = 0; i < DATA; i++) actual[i] = expected[i] = random_word(); for (u32 i = 0; i < sizeof(a.reg); i++) ((u8 *)a.reg)[i] = random_word(); /* Gather indexes include the last valid index and out-of-range indexes. */ u32 indexes[8] = { 3, 15, 16, 0xffffffff, 2, 1, 0, 4 }; memcpy(a.reg[24], indexes, sizeof(indexes)); u32 offsets[8] = { 0, 4, 8, 12, 16, 20, 24, 28 }; memcpy(a.reg[26], offsets, sizeof(offsets)); a.vtype = type; a.vl = vl; a.vstart = start; b = a; x[10] = y[10] = base; x[11] = y[11] = byte_stride; x[12] = y[12] = 0xa5; memcpy(actual + CODE, code, count * 4); u32 ret = 0x00008067; memcpy(actual + CODE + count * 4, &ret, 4); assert(jit_init(&jit)); u32 ranges[JIT_MAX_BLOCK_INSNS + 2]; jit.io.profile_next = ranges; jit.io.vector = &a; jit.io.memory_size = DATA; u32 pc = CODE; i64 budget = 100000; bool failed = false; for (u32 i = 0; i < count + 2; i++) { struct jit_block *block = jit_get_block(&jit, pc, actual, CODE, (count + 1) * 4); assert(block); int reason = jit_run(&jit, block, x, actual, budget); budget = jit.io.budget; pc = jit.io.pc; if (reason == JIT_EXIT_RET) break; if (reason == JIT_EXIT_FAULT) { assert(!require_native); u32 ins; memcpy(&ins, actual + pc, 4); enum vector_result result = step(&a, x, ins, actual); if (result != VECTOR_OK) { failed = true; break; } pc += 4; } } bool ref_failed = false; u32 used = 0; for (; used < count; used++) { if (step(&b, y, code[used], expected) != VECTOR_OK) { ref_failed = true; used++; break; } } if (!ref_failed) used++; assert(failed == ref_failed); assert(budget == 100000 - used); u32 traced = 0; for (const u32 *range = ranges; range < jit.io.profile_next; range++) { assert((*range & PROFILE_PC_MASK) == CODE + traced * 4); traced += (*range >> PROFILE_PC_BITS) + 1; } assert(traced == used); assert(memcmp(&a, &b, sizeof(a)) == 0); assert(memcmp(x, y, sizeof(x)) == 0); assert(memcmp(actual, expected, DATA) == 0); if ((count == 8 || (count == 7 && base == 128)) && (type & ~0xc0u) == 0x11 && vl == 4 && start == 0) assert(jit.block_count == 1); if (require_single_block) assert(jit.block_count == 1); jit_destroy(&jit); } /* Short copies with a known length keep scalar and vector state precise. */ static void check_short_copy(void) { u32 load = 0x02000007 | 10 << 15 | 8 << 7; u32 store = 0x02000027 | 11 << 15 | 8 << 7; require_native = require_single_block = true; for (u32 size = 1; size <= 64; size++) { require_single_block = (size & 3) != 3; u32 config = 0xc0007057 | 0xc3 << 20 | size << 15; u32 immediate[] = { config, load, store }; if (size < 32) check(immediate, 3, 0xd1, 4, 3, 0); u32 reg[] = { size << 20 | 5 << 7 | 0x13, 0xc3 << 20 | 5 << 15 | 0x7057, load, store }; check(reg, 4, 0xd1, 4, 3, 0); } require_single_block = true; for (u32 width = 1; width <= 3; width++) { u32 field = (width + 4) << 12; for (u32 count = 2; count < 32 && count <= (64u >> width); count += 2) { u32 setup = 0xcc307057 | width << 23 | count << 15; u32 copy[] = { setup, load | field, store | field }; check(copy, 3, 0xd1, 4, 3, 128); } } u32 config = 0xcc387057; u32 other[] = { config, load, store + (8 << 7) }; check(other, 3, 0, 0, 0, 128); u32 repeated[] = { config, load, config | 12 << 7, store }; check(repeated, 4, 0, 0, 0, 128); require_native = require_single_block = false; u32 fault[] = { config, load, store }; check(fault, 3, 0, 0, 0, DATA - 8); byte_stride = DATA - 8; check(fault, 3, 0, 0, 0, 128); u32 wide_fault[] = { 0xcd917057, load | (7 << 12), store | (7 << 12) }; check(wide_fault, 3, 0, 0, 0, 128); byte_stride = 3; check(wide_fault, 3, 0, 0, 0, DATA - 8); } /* Check byte transfers, fills, configuration, and fault boundaries. */ static void check_bulk(void) { u32 load = 1u << 25 | 10 << 15 | 8 << 7 | 7; u32 store = (load & ~127u) | 39; u32 stride = store | 2u << 26 | 11 << 20; u32 fill = alu(23, 4, 8, 0, 12); u32 zero = alu(23, 3, 8, 0, 0); u32 copies[] = { load, 0x10050513, store }; u32 fills[] = { fill, store, zero, stride }; u32 lengths[] = { 0, 1, 3, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 255, 256 }; require_native = true; for (u32 i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) { check(copies, 3, 0xc3, lengths[i], 0, 128); check(fills, 4, 0xc3, lengths[i], 0, 128); } u32 config = 0xc3 << 20 | 10 << 15 | 7 << 12 | 5 << 7 | 0x57; u32 configured[] = { config, fill, store }; check(configured, 3, 0xd1, 4, 3, 128); for (u32 group = 0; group < 32; group += 8) { u32 ops[] = { (load & ~(31 << 7)) | group << 7, (store & ~(31 << 7)) | group << 7 }; check(ops, 2, 0xc3, 256, 0, DATA - 256); } for (u32 policy = 0; policy <= 0xc0; policy += 0x40) { check(copies, 3, policy | 3, 17, 0, 128); check(fills, 4, policy | 3, 17, 0, 128); } u32 negative = alu(23, 3, 8, 0, 31); check(&negative, 1, 0xc3, 255, 0, 128); for (u32 i = 0; i < 128; i++) { check(copies, 3, 0xc3, random_word() % 257, 0, 128); check(fills, 4, 0xc3, random_word() % 257, 0, 128); } u32 block[JIT_MAX_BLOCK_INSNS]; for (u32 i = 0; i < JIT_MAX_BLOCK_INSNS; i++) block[i] = fill; check(block, JIT_MAX_BLOCK_INSNS, 0xc3, 17, 0, 128); require_native = false; for (u32 vl = 0; vl <= 256; vl += 16) { check(&load, 1, 0xc3, vl, 1, DATA - 64); check(&store, 1, 0xc3, vl, 1, DATA - 64); check(&stride, 1, 0xc3, vl, 1, DATA - 64); } u64 bounds[] = { DATA - 256, DATA - 255, DATA - 1, DATA, DATA + 1, ~0ULL - 7 }; for (u32 i = 0; i < sizeof(bounds) / sizeof(bounds[0]); i++) { check(&load, 1, 0xc3, 256, 0, bounds[i]); check(&store, 1, 0xc3, 256, 0, bounds[i]); check(&stride, 1, 0xc3, 256, 0, bounds[i]); require_native = true; check(&store, 1, 0xc3, 0, 0, bounds[i]); require_native = false; } u64 strides[] = { 0, 1, 2, 8, ~0ULL, ~7ULL, 1ULL << 63 }; for (u32 i = 0; i < sizeof(strides) / sizeof(strides[0]); i++) { byte_stride = strides[i]; require_native = byte_stride <= 8; check(&stride, 1, 0xc3, 16, 0, 128); require_native = true; check(&stride, 1, 0xc3, 1, 0, DATA - 1); require_native = false; check(&stride, 1, 0xc3, 16, 0, DATA - 4); } byte_stride = 3; u32 types[] = { 0, 1, 2, 3, 5, 6, 7, 0x10, 0x11, 0x13, 0x18, 0x1b }; for (u32 i = 0; i < sizeof(types) / sizeof(types[0]); i++) { u32 set = (config & ~(0x7ffu << 20)) | types[i] << 20; require_native = true; check(&set, 1, 0xd1, 4, 2, 0); check(&set, 1, 0xd1, 4, 2, ~0ULL); set = (set & ~(31u << 7)) | 10 << 7; check(&set, 1, 0xd1, 4, 2, 128); set &= ~(31u << 15); check(&set, 1, 0xd1, 4, 2, 128); for (u32 avl = 0; avl <= 31; avl++) { u32 immediate = (set & ~(31u << 15)) | 3u << 30 | avl << 15; check(&immediate, 1, 0xd1, 4, 2, 128); } } require_native = false; u32 keep = config & ~(31u << 15 | 31u << 7); check(&keep, 1, 0xc3, 16, 0, 128); check(&keep, 1, 0xd1, 4, 0, 128); u32 bad_types[] = { 4, 0x20, 0x100, 0x3f }; for (u32 i = 0; i < 4; i++) { u32 set = (config & ~(0x7ffu << 20)) | bad_types[i] << 20; check(&set, 1, 0xd1, 4, 0, 128); } u32 invalid[] = { load & ~(1u << 25), store | 1 << 7, fill | 1 << 7 }; for (u32 i = 0; i < 3; i++) check(invalid + i, 1, 0xc3, 16, 0, 128); check(copies, 3, 0xd1, 4, 0, 128); check(fills, 4, 0x80000003, 4, 0, 128); u32 mixed[] = { 0x00450513, store }; check(mixed, 2, 0xc3, 17, 0, DATA - 16); check(mixed, 2, 0xc3, 256, 0, ~0ULL - 7); } /* Check every vector operation in the bitset and BLAKE3 assembly helpers. */ static void check_word_groups(void) { u32 load = 1u << 25 | 10 << 15 | 6 << 12 | 8 << 7 | 7; u32 store = (load & ~127u) | 39; u32 compare = alu(25, 0, 0, 8, 16); u32 population = alu(16, 2, 5, 0, 16); u32 ops[] = { load, alu(10, 0, 8, 8, 16), alu(1, 0, 8, 8, 16), compare, population, store }; require_native = true; for (u32 lm = 2; lm <= 3; lm++) { for (u32 vl = 0; vl <= ((u32)VECTOR_BYTES << lm) / 4; vl++) check(ops, 6, 0xd0 | lm, vl, 0, 128); } u32 equal[] = { alu(25, 0, 0, 8, 8), population }; u32 clear[] = { alu(1, 0, 8, 8, 8), alu(10, 0, 8, 8, 8), compare, population }; for (u32 vl = 0; vl <= 64; vl++) { check(equal, 2, 0xd3, vl, 0, 128); check(clear, 4, 0xd3, vl, 0, 128); } u32 id = alu(20, 2, 2, 0, 17); check(&id, 1, 0xd1, 4, 0, 128); require_native = false; check(ops, 6, 0xd3, 17, 1, 128); check(ops, 6, 0xd3, 64, 0, DATA - 63); check(ops, 6, 0xd3, 64, 0, ~0ULL - 7); u32 alias[] = { alu(1, 0, 8, 16, 8), alu(10, 0, 16, 16, 8), alu(25, 0, 8, 8, 16), alu(16, 2, 5, 8, 16) }; require_native = true; check(alias, 4, 0xd3, 33, 0, 128); require_native = false; u32 invalid = alu(25, 0, 9, 8, 16); check(&invalid, 1, 0xd3, 33, 0, 128); } /* Check the budget at entry and across linked and indirect block exits. */ static void check_budget(void) { static struct jit_state jit; u8 mem[CODE + 24] = { 0 }; u64 x[32] = { 0 }; const u32 code[] = { 0x00000063, /* beq zero, zero, 0 */ 0x00100513, /* addi a0, zero, 1 */ 0x00008067, /* ret */ 0x00028067, /* jalr zero, t0, 0 */ 0x00200513, /* addi a0, zero, 2 */ 0x00008067, /* ret */ }; memcpy(mem + CODE, code, sizeof(code)); assert(jit_init(&jit)); struct jit_block *loop = jit_get_block(&jit, CODE, mem, CODE, sizeof(code)); assert(loop); assert(jit_run(&jit, loop, x, mem, -1) == JIT_EXIT_BRANCH); assert(jit.io.pc == CODE && jit.io.budget == -1 && x[10] == 0); assert(jit_run(&jit, loop, x, mem, 5) == JIT_EXIT_CHAIN); assert(jit.io.pc == CODE && jit.io.budget == 4 && x[10] == 0); jit_patch_chain(jit.io.patch, loop); assert(jit_run(&jit, loop, x, mem, 5) == JIT_EXIT_BRANCH); assert(jit.io.pc == CODE && jit.io.budget == 0 && x[10] == 0); u32 target = CODE + 16; struct jit_block *jump = jit_get_block(&jit, CODE + 12, mem, CODE, sizeof(code)); struct jit_block *dest = jit_get_block(&jit, target, mem, CODE, sizeof(code)); assert(jump && dest); x[5] = target; assert(jit_run(&jit, jump, x, mem, 2) == JIT_EXIT_BRANCH); assert(jit.io.pc == target && jit.io.budget == 1 && x[10] == 0); jit_cache_jump(&jit, dest); assert(jit_run(&jit, jump, x, mem, 1) == JIT_EXIT_BRANCH); assert(jit.io.pc == target && jit.io.budget == 0 && x[10] == 0); assert(jit_run(&jit, jump, x, mem, 2) == JIT_EXIT_RET); assert(jit.io.budget == -1 && x[10] == 2); jit_destroy(&jit); } int main(void) { check_budget(); check_short_copy(); check_bulk(); check_word_groups(); u32 ops[] = { alu(0, 0, 8, 10, 12), alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0), alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31), alu(9, 3, 8, 8, 3), alu(12, 0, 16, 14, 24), alu(0, 0, 8, 8, 16), }; u32 load = 1u << 25 | 10 << 15 | 6 << 12 | 8 << 7 | 7; u32 store = (load & ~127u) | 39; u32 gather = load | 1u << 26 | 26 << 20; u32 mem[] = { ops[0], load, ops[1], gather, ops[2], store, ops[3] }; for (u32 i = 0; i < 200; i++) { check(ops, 8, 0xd1, 4, 0, 128); check(mem, 7, 0xd1, 4, 0, 128); } for (u32 vl = 0; vl <= 16; vl++) { check(ops, 8, 0xd1, vl, 0, 128); check(mem, 7, 0xd1, vl, 0, 128); } for (u32 start = 1; start <= 4; start++) check(mem, 7, 0xd1, 4, start, 128); for (u32 type = 0x11; type <= 0xd1; type += 0x40) check(mem, 7, type, 4, 0, 128); u32 mixed[8] = { 0x00450513 }; memcpy(mixed + 1, mem, sizeof(mem)); check(mixed, 8, 0xd1, 4, 0, 128); check(mixed, 8, 0xd1, 4, 1, 128); check(mixed, 8, 0x10, 4, 0, 128); check(mixed, 8, 0x12, 4, 0, 128); check(mixed, 8, 0x80000000, 4, 0, 128); u64 bounds[] = { DATA - 16, DATA - 15, DATA - 4, DATA - 3, DATA, DATA + 1, ~0ULL - 7 }; for (u32 i = 0; i < sizeof(bounds) / sizeof(bounds[0]); i++) { check(mem, 7, 0xd1, 4, 0, bounds[i]); check(&gather, 1, 0xd1, 4, 0, bounds[i]); check(&store, 1, 0xd1, 4, 0, bounds[i]); } u32 invalid[] = { alu(0, 0, 9, 10, 12), alu(12, 0, 8, 8, 24), ops[0] & ~(1u << 25) }; for (u32 i = 0; i < 3; i++) check(invalid + i, 1, 0xd1, 4, 0, 128); puts("JIT vector differential checks passed"); return 0; }