test/jit_vector.c 15.8 KiB raw
1
/* Differential checks of hosted vector translation and interpreter fallback. */
2
#include <assert.h>
3
#include <stdio.h>
4
#include <stdlib.h>
5
6
#include "jit.h"
7
#include "profile.h"
8
#include "vector.h"
9
10
/* Test code lies outside the accessible data region. */
11
#define DATA 4096
12
#define CODE 8192
13
14
/* Transfer one interpreter element with the hosted bounds contract. */
15
static bool transfer(void *ctx, u64 addr, u32 size, bool store, u64 *value) {
16
    u8 *mem = ctx;
17
    if (addr > DATA || size > DATA - addr)
18
        return false;
19
    if (!store)
20
        *value = 0;
21
    for (u32 i = 0; i < size; i++) {
22
        if (store)
23
            mem[addr + i] = *value >> (i * 8);
24
        else
25
            *value |= (u64)mem[addr + i] << (i * 8);
26
    }
27
    return true;
28
}
29
30
/* A reproducible stream of operands and initial register bytes. */
31
static u32 random_word(void) {
32
    static u32 state  = 0x25ab64d1;
33
    state            ^= state << 13;
34
    state            ^= state >> 17;
35
    state            ^= state << 5;
36
    return state;
37
}
38
39
/* Encode one unmasked vector operation. */
40
static u32 alu(u32 fn, u32 form, u32 d, u32 a, u32 b) {
41
    return fn << 26 | 1u << 25 | a << 20 | b << 15 | form << 12 | d << 7 | 0x57;
42
}
43
44
/* Execute the scalar address update used before vector memory operations. */
45
static enum vector_result step(
46
    struct vector_state *v, u64 x[32], u32 ins, u8 *mem
47
) {
48
    if ((ins & 0x707f) == 0x13) {
49
        u32 rd = (ins >> 7) & 31, rs = (ins >> 15) & 31;
50
        if (rd)
51
            x[rd] = x[rs] + (i64)((i32)ins >> 20);
52
        return VECTOR_OK;
53
    }
54
    return vector_execute(v, x, ins, transfer, mem);
55
}
56
57
/* Native cases must complete without interpreter fallback. */
58
static bool require_native;
59
/* A short vector copy stays in the same translated block as its setup. */
60
static bool require_single_block;
61
/* Byte stride supplied to strided-store checks. */
62
static u64 byte_stride = 3;
63
64
/* Compare all state, memory, faults and instruction accounting for one block.
65
 */
66
static void check(
67
    const u32 *code, u32 count, u32 type, u32 vl, u32 start, u64 base
68
) {
69
    static struct jit_state jit;
70
    u8                      actual[16384] = { 0 }, expected[16384] = { 0 };
71
    struct vector_state     a     = { 0 }, b;
72
    u64                     x[32] = { 0 }, y[32] = { 0 };
73
    for (u32 i = 0; i < DATA; i++)
74
        actual[i] = expected[i] = random_word();
75
    for (u32 i = 0; i < sizeof(a.reg); i++)
76
        ((u8 *)a.reg)[i] = random_word();
77
    /* Gather indexes include the last valid index and out-of-range indexes. */
78
    u32 indexes[8] = { 3, 15, 16, 0xffffffff, 2, 1, 0, 4 };
79
    memcpy(a.reg[24], indexes, sizeof(indexes));
80
    u32 offsets[8] = { 0, 4, 8, 12, 16, 20, 24, 28 };
81
    memcpy(a.reg[26], offsets, sizeof(offsets));
82
    a.vtype  = type;
83
    a.vl     = vl;
84
    a.vstart = start;
85
    b        = a;
86
    x[10] = y[10] = base;
87
    x[11] = y[11] = byte_stride;
88
    x[12] = y[12] = 0xa5;
89
    memcpy(actual + CODE, code, count * 4);
90
    u32 ret = 0x00008067;
91
    memcpy(actual + CODE + count * 4, &ret, 4);
92
    assert(jit_init(&jit));
93
    u32 ranges[JIT_MAX_BLOCK_INSNS + 2];
94
    jit.io.profile_next = ranges;
95
    jit.io.vector       = &a;
96
    jit.io.memory_size  = DATA;
97
    u32  pc             = CODE;
98
    i64  budget         = 100000;
99
    bool failed         = false;
100
    for (u32 i = 0; i < count + 2; i++) {
101
        struct jit_block *block =
102
            jit_get_block(&jit, pc, actual, CODE, (count + 1) * 4);
103
        assert(block);
104
        int reason = jit_run(&jit, block, x, actual, budget);
105
        budget     = jit.io.budget;
106
        pc         = jit.io.pc;
107
        if (reason == JIT_EXIT_RET)
108
            break;
109
        if (reason == JIT_EXIT_FAULT) {
110
            assert(!require_native);
111
            u32 ins;
112
            memcpy(&ins, actual + pc, 4);
113
            enum vector_result result = step(&a, x, ins, actual);
114
            if (result != VECTOR_OK) {
115
                failed = true;
116
                break;
117
            }
118
            pc += 4;
119
        }
120
    }
121
    bool ref_failed = false;
122
    u32  used       = 0;
123
    for (; used < count; used++) {
124
        if (step(&b, y, code[used], expected) != VECTOR_OK) {
125
            ref_failed = true;
126
            used++;
127
            break;
128
        }
129
    }
130
    if (!ref_failed)
131
        used++;
132
    assert(failed == ref_failed);
133
    assert(budget == 100000 - used);
134
    u32 traced = 0;
135
    for (const u32 *range = ranges; range < jit.io.profile_next; range++) {
136
        assert((*range & PROFILE_PC_MASK) == CODE + traced * 4);
137
        traced += (*range >> PROFILE_PC_BITS) + 1;
138
    }
139
    assert(traced == used);
140
    assert(memcmp(&a, &b, sizeof(a)) == 0);
141
    assert(memcmp(x, y, sizeof(x)) == 0);
142
    assert(memcmp(actual, expected, DATA) == 0);
143
    if ((count == 8 || (count == 7 && base == 128)) &&
144
        (type & ~0xc0u) == 0x11 && vl == 4 && start == 0)
145
        assert(jit.block_count == 1);
146
    if (require_single_block)
147
        assert(jit.block_count == 1);
148
    jit_destroy(&jit);
149
}
150
151
/* Short copies with a known length keep scalar and vector state precise. */
152
static void check_short_copy(void) {
153
    u32 load       = 0x02000007 | 10 << 15 | 8 << 7;
154
    u32 store      = 0x02000027 | 11 << 15 | 8 << 7;
155
    require_native = require_single_block = true;
156
    for (u32 size = 1; size <= 64; size++) {
157
        require_single_block = (size & 3) != 3;
158
        u32 config           = 0xc0007057 | 0xc3 << 20 | size << 15;
159
        u32 immediate[]      = { config, load, store };
160
        if (size < 32)
161
            check(immediate, 3, 0xd1, 4, 3, 0);
162
        u32 reg[] = { size << 20 | 5 << 7 | 0x13,
163
                      0xc3 << 20 | 5 << 15 | 0x7057,
164
                      load,
165
                      store };
166
        check(reg, 4, 0xd1, 4, 3, 0);
167
    }
168
    require_single_block = true;
169
    for (u32 width = 1; width <= 3; width++) {
170
        u32 field = (width + 4) << 12;
171
        for (u32 count = 2; count < 32 && count <= (64u >> width); count += 2) {
172
            u32 setup  = 0xcc307057 | width << 23 | count << 15;
173
            u32 copy[] = { setup, load | field, store | field };
174
            check(copy, 3, 0xd1, 4, 3, 128);
175
        }
176
    }
177
    u32 config  = 0xcc387057;
178
    u32 other[] = { config, load, store + (8 << 7) };
179
    check(other, 3, 0, 0, 0, 128);
180
    u32 repeated[] = { config, load, config | 12 << 7, store };
181
    check(repeated, 4, 0, 0, 0, 128);
182
    require_native = require_single_block = false;
183
    u32 fault[]                           = { config, load, store };
184
    check(fault, 3, 0, 0, 0, DATA - 8);
185
    byte_stride = DATA - 8;
186
    check(fault, 3, 0, 0, 0, 128);
187
    u32 wide_fault[] = { 0xcd917057, load | (7 << 12), store | (7 << 12) };
188
    check(wide_fault, 3, 0, 0, 0, 128);
189
    byte_stride = 3;
190
    check(wide_fault, 3, 0, 0, 0, DATA - 8);
191
}
192
193
/* Check byte transfers, fills, configuration, and fault boundaries. */
194
static void check_bulk(void) {
195
    u32 load       = 1u << 25 | 10 << 15 | 8 << 7 | 7;
196
    u32 store      = (load & ~127u) | 39;
197
    u32 stride     = store | 2u << 26 | 11 << 20;
198
    u32 fill       = alu(23, 4, 8, 0, 12);
199
    u32 zero       = alu(23, 3, 8, 0, 0);
200
    u32 copies[]   = { load, 0x10050513, store };
201
    u32 fills[]    = { fill, store, zero, stride };
202
    u32 lengths[]  = { 0,  1,  3,  15, 16,  17,  31,  32,
203
                       33, 63, 64, 65, 127, 128, 255, 256 };
204
    require_native = true;
205
    for (u32 i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
206
        check(copies, 3, 0xc3, lengths[i], 0, 128);
207
        check(fills, 4, 0xc3, lengths[i], 0, 128);
208
    }
209
    u32 config       = 0xc3 << 20 | 10 << 15 | 7 << 12 | 5 << 7 | 0x57;
210
    u32 configured[] = { config, fill, store };
211
    check(configured, 3, 0xd1, 4, 3, 128);
212
    for (u32 group = 0; group < 32; group += 8) {
213
        u32 ops[] = { (load & ~(31 << 7)) | group << 7,
214
                      (store & ~(31 << 7)) | group << 7 };
215
        check(ops, 2, 0xc3, 256, 0, DATA - 256);
216
    }
217
    for (u32 policy = 0; policy <= 0xc0; policy += 0x40) {
218
        check(copies, 3, policy | 3, 17, 0, 128);
219
        check(fills, 4, policy | 3, 17, 0, 128);
220
    }
221
    u32 negative = alu(23, 3, 8, 0, 31);
222
    check(&negative, 1, 0xc3, 255, 0, 128);
223
    for (u32 i = 0; i < 128; i++) {
224
        check(copies, 3, 0xc3, random_word() % 257, 0, 128);
225
        check(fills, 4, 0xc3, random_word() % 257, 0, 128);
226
    }
227
    u32 block[JIT_MAX_BLOCK_INSNS];
228
    for (u32 i = 0; i < JIT_MAX_BLOCK_INSNS; i++)
229
        block[i] = fill;
230
    check(block, JIT_MAX_BLOCK_INSNS, 0xc3, 17, 0, 128);
231
    require_native = false;
232
    for (u32 vl = 0; vl <= 256; vl += 16) {
233
        check(&load, 1, 0xc3, vl, 1, DATA - 64);
234
        check(&store, 1, 0xc3, vl, 1, DATA - 64);
235
        check(&stride, 1, 0xc3, vl, 1, DATA - 64);
236
    }
237
    u64 bounds[] = {
238
        DATA - 256, DATA - 255, DATA - 1, DATA, DATA + 1, ~0ULL - 7
239
    };
240
    for (u32 i = 0; i < sizeof(bounds) / sizeof(bounds[0]); i++) {
241
        check(&load, 1, 0xc3, 256, 0, bounds[i]);
242
        check(&store, 1, 0xc3, 256, 0, bounds[i]);
243
        check(&stride, 1, 0xc3, 256, 0, bounds[i]);
244
        require_native = true;
245
        check(&store, 1, 0xc3, 0, 0, bounds[i]);
246
        require_native = false;
247
    }
248
    u64 strides[] = { 0, 1, 2, 8, ~0ULL, ~7ULL, 1ULL << 63 };
249
    for (u32 i = 0; i < sizeof(strides) / sizeof(strides[0]); i++) {
250
        byte_stride    = strides[i];
251
        require_native = byte_stride <= 8;
252
        check(&stride, 1, 0xc3, 16, 0, 128);
253
        require_native = true;
254
        check(&stride, 1, 0xc3, 1, 0, DATA - 1);
255
        require_native = false;
256
        check(&stride, 1, 0xc3, 16, 0, DATA - 4);
257
    }
258
    byte_stride = 3;
259
    u32 types[] = { 0, 1, 2, 3, 5, 6, 7, 0x10, 0x11, 0x13, 0x18, 0x1b };
260
    for (u32 i = 0; i < sizeof(types) / sizeof(types[0]); i++) {
261
        u32 set        = (config & ~(0x7ffu << 20)) | types[i] << 20;
262
        require_native = true;
263
        check(&set, 1, 0xd1, 4, 2, 0);
264
        check(&set, 1, 0xd1, 4, 2, ~0ULL);
265
        set = (set & ~(31u << 7)) | 10 << 7;
266
        check(&set, 1, 0xd1, 4, 2, 128);
267
        set &= ~(31u << 15);
268
        check(&set, 1, 0xd1, 4, 2, 128);
269
        for (u32 avl = 0; avl <= 31; avl++) {
270
            u32 immediate = (set & ~(31u << 15)) | 3u << 30 | avl << 15;
271
            check(&immediate, 1, 0xd1, 4, 2, 128);
272
        }
273
    }
274
    require_native = false;
275
    u32 keep       = config & ~(31u << 15 | 31u << 7);
276
    check(&keep, 1, 0xc3, 16, 0, 128);
277
    check(&keep, 1, 0xd1, 4, 0, 128);
278
    u32 bad_types[] = { 4, 0x20, 0x100, 0x3f };
279
    for (u32 i = 0; i < 4; i++) {
280
        u32 set = (config & ~(0x7ffu << 20)) | bad_types[i] << 20;
281
        check(&set, 1, 0xd1, 4, 0, 128);
282
    }
283
    u32 invalid[] = { load & ~(1u << 25), store | 1 << 7, fill | 1 << 7 };
284
    for (u32 i = 0; i < 3; i++)
285
        check(invalid + i, 1, 0xc3, 16, 0, 128);
286
    check(copies, 3, 0xd1, 4, 0, 128);
287
    check(fills, 4, 0x80000003, 4, 0, 128);
288
    u32 mixed[] = { 0x00450513, store };
289
    check(mixed, 2, 0xc3, 17, 0, DATA - 16);
290
    check(mixed, 2, 0xc3, 256, 0, ~0ULL - 7);
291
}
292
293
/* Check every vector operation in the bitset and BLAKE3 assembly helpers. */
294
static void check_word_groups(void) {
295
    u32 load       = 1u << 25 | 10 << 15 | 6 << 12 | 8 << 7 | 7;
296
    u32 store      = (load & ~127u) | 39;
297
    u32 compare    = alu(25, 0, 0, 8, 16);
298
    u32 population = alu(16, 2, 5, 0, 16);
299
    u32 ops[]      = { load,    alu(10, 0, 8, 8, 16), alu(1, 0, 8, 8, 16),
300
                       compare, population,           store };
301
    require_native = true;
302
    for (u32 lm = 2; lm <= 3; lm++) {
303
        for (u32 vl = 0; vl <= ((u32)VECTOR_BYTES << lm) / 4; vl++)
304
            check(ops, 6, 0xd0 | lm, vl, 0, 128);
305
    }
306
    u32 equal[] = { alu(25, 0, 0, 8, 8), population };
307
    u32 clear[] = {
308
        alu(1, 0, 8, 8, 8), alu(10, 0, 8, 8, 8), compare, population
309
    };
310
    for (u32 vl = 0; vl <= 64; vl++) {
311
        check(equal, 2, 0xd3, vl, 0, 128);
312
        check(clear, 4, 0xd3, vl, 0, 128);
313
    }
314
    u32 id = alu(20, 2, 2, 0, 17);
315
    check(&id, 1, 0xd1, 4, 0, 128);
316
    require_native = false;
317
    check(ops, 6, 0xd3, 17, 1, 128);
318
    check(ops, 6, 0xd3, 64, 0, DATA - 63);
319
    check(ops, 6, 0xd3, 64, 0, ~0ULL - 7);
320
    u32 alias[]    = { alu(1, 0, 8, 16, 8),
321
                       alu(10, 0, 16, 16, 8),
322
                       alu(25, 0, 8, 8, 16),
323
                       alu(16, 2, 5, 8, 16) };
324
    require_native = true;
325
    check(alias, 4, 0xd3, 33, 0, 128);
326
    require_native = false;
327
    u32 invalid    = alu(25, 0, 9, 8, 16);
328
    check(&invalid, 1, 0xd3, 33, 0, 128);
329
}
330
331
/* Check the budget at entry and across linked and indirect block exits. */
332
static void check_budget(void) {
333
    static struct jit_state jit;
334
    u8                      mem[CODE + 24] = { 0 };
335
    u64                     x[32]          = { 0 };
336
    const u32               code[]         = {
337
        0x00000063, /* beq zero, zero, 0 */
338
        0x00100513, /* addi a0, zero, 1 */
339
        0x00008067, /* ret */
340
        0x00028067, /* jalr zero, t0, 0 */
341
        0x00200513, /* addi a0, zero, 2 */
342
        0x00008067, /* ret */
343
    };
344
    memcpy(mem + CODE, code, sizeof(code));
345
    assert(jit_init(&jit));
346
347
    struct jit_block *loop = jit_get_block(&jit, CODE, mem, CODE, sizeof(code));
348
    assert(loop);
349
    assert(jit_run(&jit, loop, x, mem, -1) == JIT_EXIT_BRANCH);
350
    assert(jit.io.pc == CODE && jit.io.budget == -1 && x[10] == 0);
351
    assert(jit_run(&jit, loop, x, mem, 5) == JIT_EXIT_CHAIN);
352
    assert(jit.io.pc == CODE && jit.io.budget == 4 && x[10] == 0);
353
    jit_patch_chain(jit.io.patch, loop);
354
    assert(jit_run(&jit, loop, x, mem, 5) == JIT_EXIT_BRANCH);
355
    assert(jit.io.pc == CODE && jit.io.budget == 0 && x[10] == 0);
356
357
    u32               target = CODE + 16;
358
    struct jit_block *jump =
359
        jit_get_block(&jit, CODE + 12, mem, CODE, sizeof(code));
360
    struct jit_block *dest =
361
        jit_get_block(&jit, target, mem, CODE, sizeof(code));
362
    assert(jump && dest);
363
    x[5] = target;
364
    assert(jit_run(&jit, jump, x, mem, 2) == JIT_EXIT_BRANCH);
365
    assert(jit.io.pc == target && jit.io.budget == 1 && x[10] == 0);
366
    jit_cache_jump(&jit, dest);
367
    assert(jit_run(&jit, jump, x, mem, 1) == JIT_EXIT_BRANCH);
368
    assert(jit.io.pc == target && jit.io.budget == 0 && x[10] == 0);
369
    assert(jit_run(&jit, jump, x, mem, 2) == JIT_EXIT_RET);
370
    assert(jit.io.budget == -1 && x[10] == 2);
371
    jit_destroy(&jit);
372
}
373
374
int main(void) {
375
    check_budget();
376
    check_short_copy();
377
    check_bulk();
378
    check_word_groups();
379
    u32 ops[] = {
380
        alu(0, 0, 8, 10, 12),   alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0),
381
        alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31),   alu(9, 3, 8, 8, 3),
382
        alu(12, 0, 16, 14, 24), alu(0, 0, 8, 8, 16),
383
    };
384
    u32 load   = 1u << 25 | 10 << 15 | 6 << 12 | 8 << 7 | 7;
385
    u32 store  = (load & ~127u) | 39;
386
    u32 gather = load | 1u << 26 | 26 << 20;
387
    u32 mem[]  = { ops[0], load, ops[1], gather, ops[2], store, ops[3] };
388
    for (u32 i = 0; i < 200; i++) {
389
        check(ops, 8, 0xd1, 4, 0, 128);
390
        check(mem, 7, 0xd1, 4, 0, 128);
391
    }
392
    for (u32 vl = 0; vl <= 16; vl++) {
393
        check(ops, 8, 0xd1, vl, 0, 128);
394
        check(mem, 7, 0xd1, vl, 0, 128);
395
    }
396
    for (u32 start = 1; start <= 4; start++)
397
        check(mem, 7, 0xd1, 4, start, 128);
398
    for (u32 type = 0x11; type <= 0xd1; type += 0x40)
399
        check(mem, 7, type, 4, 0, 128);
400
    u32 mixed[8] = { 0x00450513 };
401
    memcpy(mixed + 1, mem, sizeof(mem));
402
    check(mixed, 8, 0xd1, 4, 0, 128);
403
    check(mixed, 8, 0xd1, 4, 1, 128);
404
    check(mixed, 8, 0x10, 4, 0, 128);
405
    check(mixed, 8, 0x12, 4, 0, 128);
406
    check(mixed, 8, 0x80000000, 4, 0, 128);
407
    u64 bounds[] = { DATA - 16, DATA - 15, DATA - 4, DATA - 3,
408
                     DATA,      DATA + 1,  ~0ULL - 7 };
409
    for (u32 i = 0; i < sizeof(bounds) / sizeof(bounds[0]); i++) {
410
        check(mem, 7, 0xd1, 4, 0, bounds[i]);
411
        check(&gather, 1, 0xd1, 4, 0, bounds[i]);
412
        check(&store, 1, 0xd1, 4, 0, bounds[i]);
413
    }
414
    u32 invalid[] = { alu(0, 0, 9, 10, 12),
415
                      alu(12, 0, 8, 8, 24),
416
                      ops[0] & ~(1u << 25) };
417
    for (u32 i = 0; i < 3; i++)
418
        check(invalid + i, 1, 0xd1, 4, 0, 128);
419
    puts("JIT vector differential checks passed");
420
    return 0;
421
}