emulator: Support Zve64x and Zvbb instructions

a9d70099b3c35df3c12335899b4714630c889c04ff0229cf680b0222cbd6e2bc
Run integer vector instructions with 256-bit registers and element
widths up to 64 bits in hosted and machine modes. Share execution
semantics and use checked memory callbacks for precise faults and
restart state.

Use host SIMD for common arithmetic, bitwise, shift, rotate and Zvbb
operations. Preserve masks, tails and restart positions without a
full register-file copy on these paths.

Implement vector CSRs, register groups, masks, fixed-point rounding,
and saturation. Report the extensions in the device tree and test
vector results, memory faults, register boundaries, and VS gating.

Assisted-by: Codex:gpt-6
Change-Id: I0cee53db79f4253f1da9d4c86dee8c4e8e0cab42
Alexis Sellier committed ago 1 parent 2ce71aa6
AGENTS.md added +4 -0
1 +
Run `make fmt` before committing changes.
2 +
Keep commit messages wrapped at 70 columns.
3 +
Do not commit code in a language that isn't already part of the repository's
4 +
checked in source.
Makefile +34 -18
1 1
# Builds the RISC-V emulator.
2 2
3 -
PREFIX  ?= $(HOME)/bin
4 -
CC      := clang
5 -
CFLAGS  := -fvisibility=hidden -std=c99 -O3 -g \
6 -
           -Wall -Wextra -Wpedantic \
7 -
           -Wformat=2 -Wformat-security \
8 -
           -Wnull-dereference \
9 -
           -Wno-format-nonliteral \
10 -
           -Wcast-align \
11 -
           -Wunused -Wuninitialized \
12 -
           -Wmissing-field-initializers \
13 -
           -fno-common -fstack-protector-all \
14 -
           -mcmodel=medium \
15 -
           -march=native -flto
16 -
LDFLAGS := -fuse-ld=lld -Wl,-z,stack-size=33554432
17 -
EMULATOR_SRCS  := emulator.c abi_host.c abi_linux.c image_native.c image_rad0.c \
3 +
PREFIX        ?= $(HOME)/bin
4 +
CC            := clang
5 +
CPPFLAGS      := -iquote .
6 +
CFLAGS        := -fvisibility=hidden -std=c99 -O3 -g \
7 +
                 -Wall -Wextra -Wpedantic \
8 +
                 -Wformat=2 -Wformat-security \
9 +
                 -Wnull-dereference \
10 +
                 -Wno-format-nonliteral \
11 +
                 -Wcast-align \
12 +
                 -Wunused -Wuninitialized \
13 +
                 -Wmissing-field-initializers \
14 +
                 -fno-common -fstack-protector-all \
15 +
                 -mcmodel=medium \
16 +
                 -march=native -flto
17 +
LDFLAGS       := -fuse-ld=lld -Wl,-z,stack-size=33554432
18 +
EMULATOR_SRCS := emulator.c abi_host.c abi_linux.c image_native.c image_rad0.c \
18 19
                 image_raw.c host_posix.c jit.c riscv/debug.c io.c riscv.c \
19 -
                 machine.c machine_cpu.c machine_csr.c machine_devices.c machine_fdt.c
20 +
                 vector.c machine.c machine_cpu.c machine_csr.c machine_devices.c machine_fdt.c
20 21
21 22
default: bin/emulator
22 23
23 -
bin/emulator: $(EMULATOR_SRCS) jit.h bitmanip.h muldiv.h color.h types.h io.h host.h hosted.h riscv.h riscv/debug.h machine.h machine_internal.h
24 +
bin/emulator: $(EMULATOR_SRCS) jit.h bitmanip.h vector.h muldiv.h color.h types.h io.h host.h hosted.h riscv.h riscv/debug.h machine.h machine_internal.h
24 25
	@echo "cc    emulator => $@"
25 26
	@mkdir -p bin
26 -
	@$(CC) $(CFLAGS) $(LDFLAGS) $(EMULATOR_SRCS) -o $@
27 +
	@$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(EMULATOR_SRCS) -o $@
27 28
28 29
install: bin/emulator
29 30
	@echo "copy  bin/emulator => $(PREFIX)/emulator"
30 31
	@mkdir -p $(PREFIX)
31 32
	@cp bin/emulator $(PREFIX)/emulator
38 39
39 40
.PHONY: default clean install
40 41
.SUFFIXES:
41 42
.DELETE_ON_ERROR:
42 43
.SILENT:
44 +
45 +
# Check integer vector execution with host memory diagnostics.
46 +
bin/vector-test: test/vector.c vector.c vector.h types.h
47 +
	@mkdir -p bin
48 +
	@$(CC) $(CPPFLAGS) -std=c99 -Wall -Wextra -g -fsanitize=address,undefined test/vector.c vector.c -o $@
49 +
50 +
bin/vector-machine-test: test/vector_machine.c vector.c vector.h machine_cpu.c machine_csr.c machine_devices.c machine.c machine_fdt.c machine_internal.h machine.h types.h bitmanip.h muldiv.h
51 +
	@mkdir -p bin
52 +
	@$(CC) $(CPPFLAGS) -std=c99 -Wall -Wextra -g -fsanitize=address,undefined test/vector_machine.c vector.c machine_cpu.c machine_csr.c machine_devices.c machine.c machine_fdt.c -o $@
53 +
54 +
test-vector: bin/vector-test bin/vector-machine-test
55 +
	@ASAN_OPTIONS=detect_leaks=0 timeout 30 bin/vector-test
56 +
	@ASAN_OPTIONS=detect_leaks=0 timeout 30 bin/vector-machine-test
57 +
58 +
.PHONY: test-vector
STYLE added +13 -0
1 +
* Separate function definitions with a blank line. If a function has a
2 +
comment, put the blank line before the comment.
3 +
* Groups of function prototypes do not need blank lines between them or
4 +
before their comments.
5 +
* Put standard and system headers first. Put third-party headers next,
6 +
  followed by project headers. Separate these groups with a blank line.
7 +
* Include project headers with quotes. Use paths relative to the source
8 +
  directory or the project root. Do not use parent directory components.
9 +
* Use short, clear variable and field names. Avoid names that make
10 +
declarations difficult to align.
11 +
* Keep struct fields and their comments aligned. Keep trailing comments
12 +
short. Put longer comments above the fields they describe. Make sure struct
13 +
fields align after formatting.
bitmanip.h +2 -1
1 1
/* RV64 Zba, Zbb, and Zbs instruction decoding and execution. */
2 2
#ifndef BITMANIP_H
3 3
#define BITMANIP_H
4 4
5 -
#include "types.h"
6 5
#include <emmintrin.h>
7 6
7 +
#include "types.h"
8 +
8 9
/* Required bit manipulation operations. Zero denotes another instruction. */
9 10
enum bitmanip_op {
10 11
    BM_NONE,
11 12
    BM_ADDUW,
12 13
    BM_SH1ADD,
emulator.c +33 -0
187 187
static void ui_render_registers(
188 188
    struct cpu *, enum display, int col, int height
189 189
);
190 190
static void ui_render_stack(struct cpu *, enum display, int col, int height);
191 191
static void ui_render(struct cpu *, enum display);
192 +
193 +
/* Check and transfer one vector element in hosted guest memory. */
194 +
static bool hosted_vector_memory(
195 +
    void *context, u64 address, u32 size, bool store, u64 *value
196 +
) {
197 +
    (void)context;
198 +
    u32 limit = guest_sizes().memory;
199 +
    if (address > limit || size > limit - address)
200 +
        return false;
201 +
    if (store)
202 +
        encode_le(&memory[address], *value, size);
203 +
    else {
204 +
        *value = 0;
205 +
        for (u32 j = 0; j < size; j++)
206 +
            *value |= (u64)memory[address + j] << (j * 8);
207 +
    }
208 +
    return true;
209 +
}
210 +
192 211
static void cpu_execute(struct cpu *, enum display, bool headless);
193 212
static void emit_fault_diagnostics(struct cpu *, u32 pc);
194 213
195 214
/* Emulator runtime options, populated from CLI flags. */
196 215
struct emulator_options {
1234 1253
1235 1254
/* Reset CPU state (keeping program loaded). */
1236 1255
static void cpu_reset(struct cpu *cpu) {
1237 1256
    trace_reset();
1238 1257
    memset(cpu->regs, 0, sizeof(cpu->regs));
1258 +
    memset(&cpu->vector, 0, sizeof(cpu->vector));
1259 +
    cpu->vector.vtype = 1ULL << 63;
1239 1260
1240 1261
    /* Set SP to the top of the usable stack, aligned to 16 bytes
1241 1262
     * as required by the RISC-V ABI. */
1242 1263
    cpu->regs[SP]    = stack_usable_top() & ~0xF;
1243 1264
    cpu->pc          = program_base;
1575 1596
            );
1576 1597
            cpu->modified = (reg_t)ins.r.rd;
1577 1598
        }
1578 1599
        goto retire;
1579 1600
    }
1601 +
    if (opcode == 0x57 || opcode == 7 || opcode == 39 ||
1602 +
        (opcode == OP_SYSTEM && ins.r.funct3)) {
1603 +
        enum vector_result vr = vector_execute(
1604 +
            &cpu->vector, cpu->regs, ins.raw, hosted_vector_memory, cpu
1605 +
        );
1606 +
        if (vr != VECTOR_OK) {
1607 +
            cpu->running = false;
1608 +
            cpu->faulted = true;
1609 +
            emit_fault_diagnostics(cpu, executed_pc);
1610 +
        }
1611 +
        goto retire;
1612 +
    }
1580 1613
    switch (opcode) {
1581 1614
    case OP_LUI:
1582 1615
        if (ins.u.rd != 0) {
1583 1616
            u32 lui_val = ins.u.imm_31_12 << 12;
1584 1617
            cpu->regs[ins.u.rd] =
hosted.h +5 -5
7 7
#ifndef HOSTED_H
8 8
#define HOSTED_H
9 9
10 10
#include <stdio.h>
11 11
12 -
#include "types.h"
13 -
/* After types.h, so that `bool` is the same type as in emulator.c, which
14 -
 * includes it through jit.h. */
15 -
#include <stdbool.h>
16 -
17 12
#include "machine.h"
18 13
#include "riscv.h"
14 +
#include "types.h"
15 +
#include "vector.h"
19 16
20 17
/* Maximum physical memory size (384MB). The runtime can choose any size up to
21 18
 * this value with `-memory-size=...`. */
22 19
#define MEMORY_SIZE       (384 * 1024 * 1024)
23 20
/* Read-only data offset. */
40 37
    bool     ebreak;           /* Program terminated via EBREAK. */
41 38
    reg_t    modified;         /* Index of the last modified register. */
42 39
    bool     reserved;         /* An LR reservation is held. */
43 40
    u32      reservation;      /* Address of the reservation. */
44 41
    u32      reservation_size; /* Width of the reserved access in bytes. */
42 +
43 +
    /* Integer vector registers and CSRs. */
44 +
    struct vector_state vector;
45 45
};
46 46
47 47
/* Guest memory sizes chosen on the command line, in bytes. */
48 48
struct guest_sizes {
49 49
    u32 memory; /* Physical memory. */
jit.h +0 -1
11 11
#ifndef JIT_H
12 12
#define JIT_H
13 13
14 14
#include "riscv.h"
15 15
#include "types.h"
16 -
#include <stdbool.h>
17 16
18 17
/* Forward declarations. */
19 18
struct cpu;
20 19
21 20
/* ---------------------------------------------------------------------------
machine.c +1 -0
68 68
    for (u32 id = 0; id < config->hart_count; id++) {
69 69
        struct machine_hart *h = &m.harts[id];
70 70
        h->id                  = id;
71 71
        h->pc                  = config->entry;
72 72
        h->privilege           = 3;
73 +
        h->vector.vtype        = 1ULL << 63;
73 74
        h->regs[2]             = m.stack_base[id] + config->stack_size;
74 75
        h->regs[10]            = id;
75 76
        h->regs[11]            = MACHINE_FDT_BASE;
76 77
        m.clint.compare[id]    = UINT64_MAX;
77 78
    }
machine_cpu.c +43 -0
188 188
    }
189 189
    *result = word ? sext(value, 32) : value;
190 190
    return true;
191 191
}
192 192
193 +
/* Memory fault details for one machine vector instruction. */
194 +
struct vector_access {
195 +
    struct machine      *m;
196 +
    struct machine_hart *h;
197 +
    u64                  addr;
198 +
    u64                  cause;
199 +
};
200 +
201 +
/* Transfer one vector element through the machine bus. */
202 +
static bool machine_vector_access(
203 +
    void *context, u64 address, u32 size, bool store, u64 *value
204 +
) {
205 +
    struct vector_access *access = context;
206 +
    access->addr                 = address;
207 +
    return machine_bus(
208 +
        access->m,
209 +
        access->h,
210 +
        address,
211 +
        size,
212 +
        store ? ACCESS_STORE : 0,
213 +
        value,
214 +
        &access->cause
215 +
    );
216 +
}
217 +
193 218
void machine_step(struct machine *m, struct machine_hart *h) {
194 219
    h->wrote_instret = false;
195 220
    u64 pending      = machine_pending(m, h) & h->mie;
196 221
    if (pending)
197 222
        h->waiting = false;
227 252
    enum bitmanip_op bm = bitmanip_decode(ins);
228 253
    if (bm != BM_NONE) {
229 254
        result = bitmanip_execute(bm, ins, a, b);
230 255
        goto retire;
231 256
    }
257 +
    if (opcode == 0x57 || opcode == 7 || opcode == 39) {
258 +
        if (!(h->mstatus & (3ULL << 9)))
259 +
            goto illegal;
260 +
        struct vector_access access = { m, h, 0, 0 };
261 +
        enum vector_result   vr     = vector_execute(
262 +
            &h->vector, h->regs, ins, machine_vector_access, &access
263 +
        );
264 +
        if (vr == VECTOR_ILLEGAL)
265 +
            goto illegal;
266 +
        h->mstatus |= 3ULL << 9;
267 +
        if (vr == VECTOR_FAULT) {
268 +
            cause   = access.cause;
269 +
            address = access.addr;
270 +
            goto fault;
271 +
        }
272 +
        write_rd = false;
273 +
        goto retire;
274 +
    }
232 275
    switch (opcode) {
233 276
    case OP_LUI:
234 277
        result = sext(ins & 0xfffff000U, 32);
235 278
        break;
236 279
    case OP_AUIPC:
machine_csr.c +18 -2
44 44
        h->pc += 4 * (cause & ~MCAUSE_INTERRUPT);
45 45
}
46 46
47 47
/* Keep the implemented mstatus fields; MPP holds only M or U. */
48 48
static u64 status_warl(u64 value) {
49 -
    u64 result = value & (MSTATUS_MIE | MSTATUS_MPIE | MSTATUS_MPP);
49 +
    u64 result =
50 +
        value & (MSTATUS_MIE | MSTATUS_MPIE | MSTATUS_MPP | (3ULL << 9));
50 51
    if ((result & MSTATUS_MPP) != MSTATUS_MPP)
51 52
        result &= ~MSTATUS_MPP;
52 53
    return result;
53 54
}
54 55
59 60
        return false;
60 61
    if (csr == CSR_INSTRET && h->privilege == 0)
61 62
        return false; /* mcounteren is zero. */
62 63
    switch (csr) {
63 64
    case CSR_MSTATUS:
64 -
        *value = h->mstatus | (2ULL << 32); /* UXL: U mode is 64-bit. */
65 +
        *value =
66 +
            h->mstatus | (2ULL << 32) |
67 +
            (((h->mstatus >> 9) & 3) == 3 ? 1ULL << 63
68 +
                                          : 0); /* UXL: U mode is 64-bit. */
65 69
        break;
66 70
    case CSR_MIE:
67 71
        *value = h->mie;
68 72
        break;
69 73
    case CSR_MTVEC:
129 133
        break;
130 134
    }
131 135
}
132 136
133 137
bool machine_csr(struct machine *m, struct machine_hart *h, u32 instruction) {
138 +
    u32 vector_address = instruction >> 20;
139 +
    if (vector_address == 8 || vector_address == 9 || vector_address == 10 ||
140 +
        vector_address == 15 ||
141 +
        (vector_address >= 0xc20 && vector_address <= 0xc22)) {
142 +
        if (!(h->mstatus & (3ULL << 9)))
143 +
            return false;
144 +
        if (!vector_csr(&h->vector, instruction, h->regs))
145 +
            return false;
146 +
        h->mstatus |= 3ULL << 9;
147 +
        return true;
148 +
    }
149 +
134 150
    u32 funct = (instruction >> 12) & 7;
135 151
    u32 csr   = instruction >> 20;
136 152
    u32 rd    = (instruction >> 7) & 31;
137 153
    u32 rs    = (instruction >> 15) & 31;
138 154
    if (funct == 0 || funct == 4)
machine_fdt.c +5 -1
167 167
        begin(&f, name);
168 168
        string(&f, "device_type", "cpu");
169 169
        string(&f, "compatible", "riscv");
170 170
        scalar(&f, "reg", id);
171 171
        string(&f, "status", "okay");
172 -
        string(&f, "riscv,isa", "rv64ima_zba_zbb_zbs_zicsr_zifencei");
172 +
        string(
173 +
            &f,
174 +
            "riscv,isa",
175 +
            "rv64ima_zba_zbb_zbs_zicsr_zifencei_zvbb_zve64x_zvl256b"
176 +
        );
173 177
        begin(&f, "interrupt-controller");
174 178
        scalar(&f, "#interrupt-cells", 1);
175 179
        property(&f, "interrupt-controller", "", 0);
176 180
        string(&f, "compatible", "riscv,cpu-intc");
177 181
        scalar(&f, "phandle", id + 1);
machine_internal.h +6 -0
1 1
/* State and interfaces shared by the machine implementation files. */
2 2
#ifndef MACHINE_INTERNAL_H
3 3
#define MACHINE_INTERNAL_H
4 4
5 5
#include "machine.h"
6 +
#include "vector.h"
6 7
7 8
/* Fixed physical addresses, also published in the FDT. */
8 9
#define MACHINE_FDT_BASE    (MACHINE_RAM_BASE + 0x00800000U)
9 10
#define MACHINE_FDT_SIZE    0x00010000U
10 11
#define MACHINE_CLINT_BASE  0x02000000U
68 69
    bool waiting;          /* WFI sleep until an enabled interrupt pends. */
69 70
    bool reserved;         /* Whether the LR reservation is live. */
70 71
    u64  reservation;      /* Exact physical address of the last LR. */
71 72
    u32  reservation_size; /* Width of the last LR, 4 or 8 bytes. */
72 73
    bool wrote_instret;    /* The current instruction wrote minstret. */
74 +
75 +
    /* Integer vector registers and CSRs. */
76 +
    struct vector_state vector;
73 77
};
74 78
75 79
/* Shared CLINT state. mtime is writable, so it is kept apart from the
76 80
 * scheduler clock. */
77 81
struct machine_clint {
141 145
void machine_devices_update(struct machine *m);
142 146
/* Return the next future timer/edge wake tick, bounded by the run deadline. */
143 147
u64 machine_next_event(const struct machine *m);
144 148
/* Whether the PLIC has a notification above threshold for a context. */
145 149
bool machine_plic_notify(const struct machine *m, u32 context);
150 +
146 151
/* Derive a hart's hardware mip bits from CLINT and PLIC state. */
147 152
static inline u64 machine_pending(
148 153
    const struct machine *m, const struct machine_hart *h
149 154
) {
150 155
    u64 pending = 0;
154 159
        pending |= MIP_MTIP;
155 160
    if (machine_plic_notify(m, h->id))
156 161
        pending |= MIP_MEIP;
157 162
    return pending;
158 163
}
164 +
159 165
/* Deliver an exception or interrupt; no instruction retires. */
160 166
void machine_trap(
161 167
    struct machine *m, struct machine_hart *h, u64 cause, u64 value
162 168
);
163 169
/* Read/modify/write a CSR instruction; false means illegal instruction. */
riscv/debug.c +3 -4
1 1
#include <stdarg.h>
2 2
#include <stdio.h>
3 3
#include <string.h>
4 4
#include <unistd.h>
5 5
6 -
#include "../bitmanip.h"
7 -
#include "../color.h"
8 -
#include "../riscv.h"
9 -
6 +
#include "bitmanip.h"
7 +
#include "color.h"
10 8
#include "debug.h"
9 +
#include "riscv.h"
11 10
12 11
/* Width for opcode field alignment. */
13 12
#define OP_WIDTH 6
14 13
15 14
/* Load instructions based on funct3. */
riscv/debug.h +2 -2
1 1
#ifndef OP_DEBUG_H
2 2
#define OP_DEBUG_H
3 3
4 4
#include <stdio.h>
5 5
6 -
#include "../riscv.h"
7 -
#include "../types.h"
6 +
#include "riscv.h"
7 +
#include "types.h"
8 8
9 9
/* Maximum length of a component string (op, reg, imm). */
10 10
#define MAX_PART_LEN      16
11 11
/* Maximum length of an instruction string, including colors. */
12 12
#define MAX_INSTR_STR_LEN 96
test/vector.c added +299 -0
1 +
#include <assert.h>
2 +
#include <string.h>
3 +
4 +
#include "vector.h"
5 +
6 +
/* Transfer a checked element in test memory. */
7 +
static bool mem(void *ctx, u64 address, u32 size, bool store, u64 *value) {
8 +
    u8 *bytes = ctx;
9 +
    if (address > 4096 || size > 4096 - address)
10 +
        return false;
11 +
    if (!store)
12 +
        *value = 0;
13 +
    for (u32 j = 0; j < size; j++) {
14 +
        if (store)
15 +
            bytes[address + j] = *value >> (j * 8);
16 +
        else
17 +
            *value |= (u64)bytes[address + j] << (j * 8);
18 +
    }
19 +
    return true;
20 +
}
21 +
22 +
/* Set one little-endian element in a register group. */
23 +
static void put(struct vector_state *v, u32 r, u32 i, u32 bits, u64 n) {
24 +
    u8 *p = &v->reg[r][0] + i * (bits / 8);
25 +
    for (u32 j = 0; j < bits / 8; j++)
26 +
        p[j] = n >> (8 * j);
27 +
}
28 +
29 +
/* Read one little-endian element in a register group. */
30 +
static u64 get(struct vector_state *v, u32 r, u32 i, u32 bits) {
31 +
    u64 n = 0;
32 +
    u8 *p = &v->reg[r][0] + i * (bits / 8);
33 +
    for (u32 j = 0; j < bits / 8; j++)
34 +
        n |= (u64)p[j] << (8 * j);
35 +
    return n;
36 +
}
37 +
38 +
/* Construct OP-V operands independently of the instruction catalog. */
39 +
static u32 alu(u32 fn, u32 form, u32 d, u32 a, u32 b, bool mask) {
40 +
    return fn << 26 | (!mask) << 25 | a << 20 | b << 15 | form << 12 | d << 7 |
41 +
           0x57;
42 +
}
43 +
44 +
/* Check rotations, masking and tail preservation for each element width. */
45 +
static void rotations(void) {
46 +
    for (u32 bits = 8, sew = 0; bits <= 64; bits *= 2, sew++) {
47 +
        struct vector_state v           = { 0 };
48 +
        u64                 x[32]       = { 0 };
49 +
        u8                  bytes[4096] = { 0 };
50 +
        v.vtype                         = sew << 3;
51 +
        v.vl                            = 3;
52 +
        u64 mask = bits == 64 ? ~0ULL : (1ULL << bits) - 1;
53 +
        u64 a    = 0x81a53c779e0142ffULL & mask;
54 +
        for (u32 i = 0; i < 4; i++) {
55 +
            put(&v, 8, i, bits, 0x55);
56 +
            put(&v, 16, i, bits, a);
57 +
        }
58 +
        for (u32 n = 0; n < 64; n++) {
59 +
            u32 ins = alu(0x14 | (n >> 5), 3, 8, 16, n & 31, false);
60 +
            assert(vector_execute(&v, x, ins, mem, bytes) == VECTOR_OK);
61 +
            u32 k        = n & (bits - 1);
62 +
            u64 expected = k ? ((a >> k) | (a << (bits - k))) & mask : a;
63 +
            assert(get(&v, 8, 0, bits) == expected);
64 +
            assert(get(&v, 8, 3, bits) == 0x55);
65 +
        }
66 +
        v.reg[0][0] = 5;
67 +
        put(&v, 8, 1, bits, 0x55);
68 +
        assert(
69 +
            vector_execute(&v, x, alu(0, 0, 8, 16, 16, true), mem, bytes) ==
70 +
            VECTOR_OK
71 +
        );
72 +
        assert(get(&v, 8, 0, bits) == ((a + a) & mask));
73 +
        assert(get(&v, 8, 1, bits) == 0x55);
74 +
        assert(get(&v, 8, 2, bits) == ((a + a) & mask));
75 +
    }
76 +
}
77 +
78 +
/* Check memory restart, fault-only-first loads and segmented addressing. */
79 +
static void memory_cases(void) {
80 +
    struct vector_state v           = { 0 };
81 +
    u64                 x[32]       = { 0 };
82 +
    u8                  bytes[4096] = { 0 };
83 +
    v.vtype                         = 16;
84 +
    v.vl                            = 4;
85 +
    x[10]                           = 4092;
86 +
    bytes[4092]                     = 17;
87 +
    u32 load                        = 0x02056407; /* vle32.v v8, (a0) */
88 +
    assert(vector_execute(&v, x, load, mem, bytes) == VECTOR_FAULT);
89 +
    assert(v.vstart == 1 && get(&v, 8, 0, 32) == 17);
90 +
    x[10]    = 0;
91 +
    bytes[4] = 19;
92 +
    assert(vector_execute(&v, x, load, mem, bytes) == VECTOR_OK);
93 +
    assert(v.vstart == 0 && get(&v, 8, 0, 32) == 17 && get(&v, 8, 1, 32) == 19);
94 +
    x[10] = 4092;
95 +
    assert(vector_execute(&v, x, load | (16 << 20), mem, bytes) == VECTOR_OK);
96 +
    assert(v.vl == 1 && v.vstart == 0);
97 +
    x[10]     = 0;
98 +
    v.vl      = 2;
99 +
    bytes[0]  = 1;
100 +
    bytes[4]  = 2;
101 +
    bytes[8]  = 3;
102 +
    bytes[12] = 4;
103 +
    assert(vector_execute(&v, x, load | (1u << 29), mem, bytes) == VECTOR_OK);
104 +
    assert(get(&v, 8, 0, 32) == 1 && get(&v, 9, 0, 32) == 2);
105 +
    assert(get(&v, 8, 1, 32) == 3 && get(&v, 9, 1, 32) == 4);
106 +
}
107 +
108 +
/* Check fixed-point saturation, extension and packed mask reductions. */
109 +
static void integer_cases(void) {
110 +
    struct vector_state v           = { 0 };
111 +
    u64                 x[32]       = { 0 };
112 +
    u8                  bytes[4096] = { 0 };
113 +
    v.vl                            = 4;
114 +
    put(&v, 16, 0, 8, 255);
115 +
    put(&v, 24, 0, 8, 1);
116 +
    assert(
117 +
        vector_execute(&v, x, alu(32, 0, 8, 16, 24, false), mem, bytes) ==
118 +
        VECTOR_OK
119 +
    );
120 +
    assert(get(&v, 8, 0, 8) == 255 && v.vxsat == 1);
121 +
    put(&v, 16, 0, 8, 127);
122 +
    put(&v, 24, 0, 8, 1);
123 +
    assert(
124 +
        vector_execute(&v, x, alu(33, 0, 8, 16, 24, false), mem, bytes) ==
125 +
        VECTOR_OK
126 +
    );
127 +
    assert(get(&v, 8, 0, 8) == 127);
128 +
    v.vtype      = 16;
129 +
    v.reg[31][0] = 0x80;
130 +
    assert(
131 +
        vector_execute(&v, x, alu(18, 2, 8, 31, 5, false), mem, bytes) ==
132 +
        VECTOR_OK
133 +
    );
134 +
    assert(get(&v, 8, 0, 32) == 0xffffff80);
135 +
    v.vtype = 3;
136 +
    v.vl    = 256;
137 +
    memset(v.reg[31], 255, VECTOR_BYTES);
138 +
    assert(
139 +
        vector_execute(&v, x, alu(16, 2, 10, 31, 16, false), mem, bytes) ==
140 +
        VECTOR_OK
141 +
    );
142 +
    assert(x[10] == 256);
143 +
    assert(
144 +
        vector_execute(&v, x, alu(16, 2, 10, 31, 17, false), mem, bytes) ==
145 +
        VECTOR_OK
146 +
    );
147 +
    assert(x[10] == 0);
148 +
    /* Destination alignment is checked before a register group is written. */
149 +
    struct vector_state before = v;
150 +
    assert(
151 +
        vector_execute(&v, x, alu(0, 0, 31, 16, 24, false), mem, bytes) ==
152 +
        VECTOR_ILLEGAL
153 +
    );
154 +
    assert(memcmp(&before, &v, sizeof(v)) == 0);
155 +
}
156 +
157 +
/* Check each unary Zvbb operation against fixed element results. */
158 +
static void bit_operations(void) {
159 +
    struct vector_state v           = { 0 };
160 +
    u64                 x[32]       = { 0 };
161 +
    u8                  bytes[4096] = { 0 };
162 +
    const u32           selectors[] = { 8, 9, 10, 12, 13, 14 };
163 +
    const u32 expected[] = { 0x808040c0, 0x03020101, 0xc0408080, 7, 0, 5 };
164 +
    v.vtype              = 16;
165 +
    v.vl                 = 1;
166 +
    put(&v, 16, 0, 32, 0x01010203);
167 +
    for (u32 i = 0; i < 6; i++) {
168 +
        assert(
169 +
            vector_execute(
170 +
                &v, x, alu(18, 2, 8, 16, selectors[i], false), mem, bytes
171 +
            ) == VECTOR_OK
172 +
        );
173 +
        assert(get(&v, 8, 0, 32) == expected[i]);
174 +
    }
175 +
    v.vtype = 16;
176 +
    put(&v, 16, 0, 32, 1);
177 +
    assert(
178 +
        vector_execute(&v, x, alu(53, 3, 8, 16, 31, false), mem, bytes) ==
179 +
        VECTOR_OK
180 +
    );
181 +
    assert(get(&v, 8, 0, 64) == 1ULL << 31);
182 +
}
183 +
184 +
/* Mask results and scalar moves occupy one register for every LMUL. */
185 +
static void mask_registers(void) {
186 +
    struct vector_state v           = { 0 };
187 +
    u64                 x[32]       = { 0 };
188 +
    u8                  bytes[4096] = { 0 };
189 +
    v.vtype                         = 3;
190 +
    v.vl                            = 8;
191 +
    v.reg[31][0]                    = 4;
192 +
    assert(
193 +
        vector_execute(&v, x, alu(20, 2, 3, 31, 1, false), mem, bytes) ==
194 +
        VECTOR_OK
195 +
    );
196 +
    assert(v.reg[3][0] == 3);
197 +
    assert(
198 +
        vector_execute(&v, x, alu(20, 2, 31, 31, 1, false), mem, bytes) ==
199 +
        VECTOR_ILLEGAL
200 +
    );
201 +
    assert(
202 +
        vector_execute(&v, x, alu(20, 2, 8, 8, 16, false), mem, bytes) ==
203 +
        VECTOR_ILLEGAL
204 +
    );
205 +
    x[10] = 23;
206 +
    assert(
207 +
        vector_execute(&v, x, alu(16, 6, 3, 0, 10, false), mem, bytes) ==
208 +
        VECTOR_OK
209 +
    );
210 +
    assert(v.reg[3][0] == 23);
211 +
}
212 +
213 +
/* Check SIMD register groups, aliases, masks, tails and restart positions. */
214 +
static void native_lanes(void) {
215 +
    const u32 lms[] = { 0, 1, 2, 3, 5, 6, 7 };
216 +
    for (u32 w = 0; w < 4; w++) {
217 +
        u32 bits = 8u << w;
218 +
        u64 mask = bits == 64 ? ~0ULL : (1ULL << bits) - 1;
219 +
        for (u32 lm = 0; lm < sizeof(lms) / sizeof(lms[0]); lm++) {
220 +
            u32 bytes = lms[lm] < 4 ? VECTOR_BYTES << lms[lm]
221 +
                                    : VECTOR_BYTES >> (8 - lms[lm]);
222 +
            u32 max   = bytes * 8 / bits;
223 +
            if (!max)
224 +
                continue;
225 +
            for (u32 k = 0; k < 4; k++)
226 +
                for (u32 mode = 0; mode < 8; mode++) {
227 +
                    struct vector_state v     = { 0 };
228 +
                    u64                 x[32] = { 0 };
229 +
                    for (u32 j = 0; j < sizeof(v.reg); j++)
230 +
                        ((u8 *)v.reg)[j] = j * 37 + 129;
231 +
                    v.vtype     = w * 8 + lms[lm];
232 +
                    v.vl        = mode & 1 ? max : max - 1;
233 +
                    v.vstart    = mode & 2 ? max / 3 : 0;
234 +
                    bool masked = mode & 4;
235 +
                    u32  d      = mode & 1 ? 24 : 8;
236 +
                    u32  ops[]  = {
237 +
                         alu(0, 0, d, 24, 16, masked),
238 +
                         alu(37, 2, d, 24, 16, masked),
239 +
                         alu(20, 4, d, 24, 10, masked),
240 +
                         alu(18, 2, d, 24, 14, masked),
241 +
                    };
242 +
                    x[10]                    = bits - 1;
243 +
                    struct vector_state want = v;
244 +
                    for (u32 i = v.vstart; i < v.vl; i++) {
245 +
                        if (masked && !((v.reg[0][i / 8] >> (i % 8)) & 1))
246 +
                            continue;
247 +
                        u64 a = get(&v, 24, i, bits);
248 +
                        u64 b = get(&v, 16, i, bits);
249 +
                        u64 z = k == 0   ? a + b
250 +
                                : k == 1 ? a * b
251 +
                                : k == 2 ? (a >> (bits - 1)) | (a << 1)
252 +
                                         : (u64)__builtin_popcountll(a);
253 +
                        put(&want, d, i, bits, z & mask);
254 +
                    }
255 +
                    want.vstart = 0;
256 +
                    assert(
257 +
                        vector_execute(&v, x, ops[k], mem, NULL) == VECTOR_OK
258 +
                    );
259 +
                    assert(memcmp(&v, &want, sizeof(v)) == 0);
260 +
                }
261 +
        }
262 +
    }
263 +
}
264 +
265 +
/* Exercise integer vector execution without host image loading. */
266 +
int main(void) {
267 +
    native_lanes();
268 +
    mask_registers();
269 +
    bit_operations();
270 +
    rotations();
271 +
    memory_cases();
272 +
    integer_cases();
273 +
    struct vector_state shift    = { 0 };
274 +
    u64                 sx[32]   = { 0 };
275 +
    u8                  sm[4096] = { 0 };
276 +
    shift.vtype                  = 24;
277 +
    shift.vl                     = 1;
278 +
    put(&shift, 16, 0, 64, 1);
279 +
    assert(
280 +
        vector_execute(&shift, sx, alu(37, 3, 8, 16, 31, false), mem, sm) ==
281 +
        VECTOR_OK
282 +
    );
283 +
    assert(get(&shift, 8, 0, 64) == 1ULL << 31);
284 +
    struct vector_state v           = { 0 };
285 +
    u64                 x[32]       = { 0 };
286 +
    u8                  bytes[4096] = { 0 };
287 +
    assert(vector_execute(&v, x, 0xc1027557, mem, bytes) == VECTOR_OK);
288 +
    assert(v.vl == 4 && v.vtype == 16 && x[10] == 4);
289 +
    for (int i = 0; i < 4; i++) {
290 +
        v.reg[9][i * 4]  = i + 1;
291 +
        v.reg[10][i * 4] = 10;
292 +
    }
293 +
    assert(vector_execute(&v, x, 0x02950457, mem, bytes) == VECTOR_OK);
294 +
    for (int i = 0; i < 4; i++)
295 +
        assert(v.reg[8][i * 4] == i + 11);
296 +
    assert(vector_execute(&v, x, 0x52943457, mem, bytes) == VECTOR_OK);
297 +
    assert(v.reg[8][0] == 0 && v.reg[8][3] == 1);
298 +
    return 0;
299 +
}
test/vector_machine.c added +49 -0
1 +
/* Integer vector execution through the machine instruction dispatcher. */
2 +
#include <assert.h>
3 +
#include <string.h>
4 +
5 +
#include "machine_internal.h"
6 +
7 +
/* Write one instruction to machine RAM. */
8 +
static void instruction(u8 *ram, u32 ins) {
9 +
    for (u32 i = 0; i < 4; i++)
10 +
        ram[i] = ins >> (8 * i);
11 +
}
12 +
13 +
int main(void) {
14 +
    u8                    ram[4096] = { 0 };
15 +
    struct machine_config config    = { 0 };
16 +
    config.ram                      = ram;
17 +
    config.memory_size              = sizeof(ram);
18 +
    config.hart_count               = 1;
19 +
    config.text.base                = MACHINE_RAM_BASE;
20 +
    config.text.size                = 64;
21 +
    struct machine machine          = { 0 };
22 +
    machine.config                  = &config;
23 +
    struct machine_hart *h          = &machine.harts[0];
24 +
    h->privilege                    = 3;
25 +
    h->pc                           = MACHINE_RAM_BASE;
26 +
    instruction(ram, 0xc1027557); /* vsetivli a0,4,e32,m1,tu,mu */
27 +
    machine_step(&machine, h);
28 +
    assert(h->mcause == CAUSE_ILLEGAL && h->vector.vl == 0);
29 +
    h->mstatus = 1ULL << 9;
30 +
    h->pc      = MACHINE_RAM_BASE;
31 +
    machine_step(&machine, h);
32 +
    assert(h->vector.vl == 4 && h->regs[10] == 4);
33 +
    assert(((h->mstatus >> 9) & 3) == 3);
34 +
    instruction(ram, 0x5e03b457); /* vmv.v.i v8,7 */
35 +
    h->pc = MACHINE_RAM_BASE;
36 +
    machine_step(&machine, h);
37 +
    assert(h->vector.reg[8][0] == 7);
38 +
    instruction(ram, 0x42802557); /* vmv.x.s a0,v8 */
39 +
    h->pc = MACHINE_RAM_BASE;
40 +
    machine_step(&machine, h);
41 +
    assert(h->regs[10] == 7);
42 +
    instruction(ram, 0x02056407); /* vle32.v v8,(a0) */
43 +
    h->regs[10] = MACHINE_RAM_BASE + 4092;
44 +
    h->pc       = MACHINE_RAM_BASE;
45 +
    machine_step(&machine, h);
46 +
    assert(h->mcause == CAUSE_LOAD_FAULT && h->vector.vstart == 1);
47 +
    assert(h->mtval == MACHINE_RAM_BASE + 4096);
48 +
    return 0;
49 +
}
types.h +2 -5
1 1
#ifndef TYPES_H
2 2
#define TYPES_H
3 3
4 +
#include <stdbool.h>
5 +
4 6
typedef unsigned char      u8;
5 7
typedef unsigned short     u16;
6 8
typedef unsigned int       u32;
7 9
typedef unsigned long long u64;
8 10
typedef signed char        i8;
13 15
typedef double             f64;
14 16
15 17
typedef unsigned long usize;
16 18
typedef long          isize;
17 19
18 -
typedef u8 bool;
19 -
20 -
#define true  1
21 -
#define false 0
22 -
23 20
#ifndef NULL
24 21
#define NULL ((void *)0)
25 22
#endif
26 23
27 24
#define U8_MIN  0
vector.c added +1371 -0
1 +
/* Integer vector instruction execution. */
2 +
#include "vector.h"
3 +
4 +
/* One instruction's fixed opcode fields. */
5 +
struct vector_opcode {
6 +
    const char *name;
7 +
    u32         value, mask;
8 +
};
9 +
10 +
static const struct vector_opcode vector_opcodes[] = {
11 +
    { "vsetivli", 0xc0007057, 0xc000707f },
12 +
    { "vsetvli", 0x00007057, 0x8000707f },
13 +
    { "vsetvl", 0x80007057, 0xfe00707f },
14 +
    { "vlm.v", 0x02b00007, 0xfff0707f },
15 +
    { "vsm.v", 0x02b00027, 0xfff0707f },
16 +
    { "vle8.v", 0x00000007, 0x1df0707f },
17 +
    { "vle16.v", 0x00005007, 0x1df0707f },
18 +
    { "vle32.v", 0x00006007, 0x1df0707f },
19 +
    { "vle64.v", 0x00007007, 0x1df0707f },
20 +
    { "vse8.v", 0x00000027, 0x1df0707f },
21 +
    { "vse16.v", 0x00005027, 0x1df0707f },
22 +
    { "vse32.v", 0x00006027, 0x1df0707f },
23 +
    { "vse64.v", 0x00007027, 0x1df0707f },
24 +
    { "vluxei8.v", 0x04000007, 0x1c00707f },
25 +
    { "vluxei16.v", 0x04005007, 0x1c00707f },
26 +
    { "vluxei32.v", 0x04006007, 0x1c00707f },
27 +
    { "vluxei64.v", 0x04007007, 0x1c00707f },
28 +
    { "vsuxei8.v", 0x04000027, 0x1c00707f },
29 +
    { "vsuxei16.v", 0x04005027, 0x1c00707f },
30 +
    { "vsuxei32.v", 0x04006027, 0x1c00707f },
31 +
    { "vsuxei64.v", 0x04007027, 0x1c00707f },
32 +
    { "vlse8.v", 0x08000007, 0x1c00707f },
33 +
    { "vlse16.v", 0x08005007, 0x1c00707f },
34 +
    { "vlse32.v", 0x08006007, 0x1c00707f },
35 +
    { "vlse64.v", 0x08007007, 0x1c00707f },
36 +
    { "vsse8.v", 0x08000027, 0x1c00707f },
37 +
    { "vsse16.v", 0x08005027, 0x1c00707f },
38 +
    { "vsse32.v", 0x08006027, 0x1c00707f },
39 +
    { "vsse64.v", 0x08007027, 0x1c00707f },
40 +
    { "vloxei8.v", 0x0c000007, 0x1c00707f },
41 +
    { "vloxei16.v", 0x0c005007, 0x1c00707f },
42 +
    { "vloxei32.v", 0x0c006007, 0x1c00707f },
43 +
    { "vloxei64.v", 0x0c007007, 0x1c00707f },
44 +
    { "vsoxei8.v", 0x0c000027, 0x1c00707f },
45 +
    { "vsoxei16.v", 0x0c005027, 0x1c00707f },
46 +
    { "vsoxei32.v", 0x0c006027, 0x1c00707f },
47 +
    { "vsoxei64.v", 0x0c007027, 0x1c00707f },
48 +
    { "vle8ff.v", 0x01000007, 0x1df0707f },
49 +
    { "vle16ff.v", 0x01005007, 0x1df0707f },
50 +
    { "vle32ff.v", 0x01006007, 0x1df0707f },
51 +
    { "vle64ff.v", 0x01007007, 0x1df0707f },
52 +
    { "vl1re8.v", 0x02800007, 0xfff0707f },
53 +
    { "vl1re16.v", 0x02805007, 0xfff0707f },
54 +
    { "vl1re32.v", 0x02806007, 0xfff0707f },
55 +
    { "vl1re64.v", 0x02807007, 0xfff0707f },
56 +
    { "vl2re8.v", 0x22800007, 0xfff0707f },
57 +
    { "vl2re16.v", 0x22805007, 0xfff0707f },
58 +
    { "vl2re32.v", 0x22806007, 0xfff0707f },
59 +
    { "vl2re64.v", 0x22807007, 0xfff0707f },
60 +
    { "vl4re8.v", 0x62800007, 0xfff0707f },
61 +
    { "vl4re16.v", 0x62805007, 0xfff0707f },
62 +
    { "vl4re32.v", 0x62806007, 0xfff0707f },
63 +
    { "vl4re64.v", 0x62807007, 0xfff0707f },
64 +
    { "vl8re8.v", 0xe2800007, 0xfff0707f },
65 +
    { "vl8re16.v", 0xe2805007, 0xfff0707f },
66 +
    { "vl8re32.v", 0xe2806007, 0xfff0707f },
67 +
    { "vl8re64.v", 0xe2807007, 0xfff0707f },
68 +
    { "vs1r.v", 0x02800027, 0xfff0707f },
69 +
    { "vs2r.v", 0x22800027, 0xfff0707f },
70 +
    { "vs4r.v", 0x62800027, 0xfff0707f },
71 +
    { "vs8r.v", 0xe2800027, 0xfff0707f },
72 +
    { "vadd.vx", 0x00004057, 0xfc00707f },
73 +
    { "vsub.vx", 0x08004057, 0xfc00707f },
74 +
    { "vrsub.vx", 0x0c004057, 0xfc00707f },
75 +
    { "vminu.vx", 0x10004057, 0xfc00707f },
76 +
    { "vmin.vx", 0x14004057, 0xfc00707f },
77 +
    { "vmaxu.vx", 0x18004057, 0xfc00707f },
78 +
    { "vmax.vx", 0x1c004057, 0xfc00707f },
79 +
    { "vand.vx", 0x24004057, 0xfc00707f },
80 +
    { "vor.vx", 0x28004057, 0xfc00707f },
81 +
    { "vxor.vx", 0x2c004057, 0xfc00707f },
82 +
    { "vrgather.vx", 0x30004057, 0xfc00707f },
83 +
    { "vslideup.vx", 0x38004057, 0xfc00707f },
84 +
    { "vslidedown.vx", 0x3c004057, 0xfc00707f },
85 +
    { "vadc.vxm", 0x40004057, 0xfe00707f },
86 +
    { "vmadc.vxm", 0x44004057, 0xfe00707f },
87 +
    { "vmadc.vx", 0x46004057, 0xfe00707f },
88 +
    { "vsbc.vxm", 0x48004057, 0xfe00707f },
89 +
    { "vmsbc.vxm", 0x4c004057, 0xfe00707f },
90 +
    { "vmsbc.vx", 0x4e004057, 0xfe00707f },
91 +
    { "vmerge.vxm", 0x5c004057, 0xfe00707f },
92 +
    { "vmv.v.x", 0x5e004057, 0xfff0707f },
93 +
    { "vmseq.vx", 0x60004057, 0xfc00707f },
94 +
    { "vmsne.vx", 0x64004057, 0xfc00707f },
95 +
    { "vmsltu.vx", 0x68004057, 0xfc00707f },
96 +
    { "vmslt.vx", 0x6c004057, 0xfc00707f },
97 +
    { "vmsleu.vx", 0x70004057, 0xfc00707f },
98 +
    { "vmsle.vx", 0x74004057, 0xfc00707f },
99 +
    { "vmsgtu.vx", 0x78004057, 0xfc00707f },
100 +
    { "vmsgt.vx", 0x7c004057, 0xfc00707f },
101 +
    { "vsaddu.vx", 0x80004057, 0xfc00707f },
102 +
    { "vsadd.vx", 0x84004057, 0xfc00707f },
103 +
    { "vssubu.vx", 0x88004057, 0xfc00707f },
104 +
    { "vssub.vx", 0x8c004057, 0xfc00707f },
105 +
    { "vsll.vx", 0x94004057, 0xfc00707f },
106 +
    { "vsmul.vx", 0x9c004057, 0xfc00707f },
107 +
    { "vsrl.vx", 0xa0004057, 0xfc00707f },
108 +
    { "vsra.vx", 0xa4004057, 0xfc00707f },
109 +
    { "vssrl.vx", 0xa8004057, 0xfc00707f },
110 +
    { "vssra.vx", 0xac004057, 0xfc00707f },
111 +
    { "vnsrl.wx", 0xb0004057, 0xfc00707f },
112 +
    { "vnsra.wx", 0xb4004057, 0xfc00707f },
113 +
    { "vnclipu.wx", 0xb8004057, 0xfc00707f },
114 +
    { "vnclip.wx", 0xbc004057, 0xfc00707f },
115 +
    { "vadd.vv", 0x00000057, 0xfc00707f },
116 +
    { "vsub.vv", 0x08000057, 0xfc00707f },
117 +
    { "vminu.vv", 0x10000057, 0xfc00707f },
118 +
    { "vmin.vv", 0x14000057, 0xfc00707f },
119 +
    { "vmaxu.vv", 0x18000057, 0xfc00707f },
120 +
    { "vmax.vv", 0x1c000057, 0xfc00707f },
121 +
    { "vand.vv", 0x24000057, 0xfc00707f },
122 +
    { "vor.vv", 0x28000057, 0xfc00707f },
123 +
    { "vxor.vv", 0x2c000057, 0xfc00707f },
124 +
    { "vrgather.vv", 0x30000057, 0xfc00707f },
125 +
    { "vrgatherei16.vv", 0x38000057, 0xfc00707f },
126 +
    { "vadc.vvm", 0x40000057, 0xfe00707f },
127 +
    { "vmadc.vvm", 0x44000057, 0xfe00707f },
128 +
    { "vmadc.vv", 0x46000057, 0xfe00707f },
129 +
    { "vsbc.vvm", 0x48000057, 0xfe00707f },
130 +
    { "vmsbc.vvm", 0x4c000057, 0xfe00707f },
131 +
    { "vmsbc.vv", 0x4e000057, 0xfe00707f },
132 +
    { "vmerge.vvm", 0x5c000057, 0xfe00707f },
133 +
    { "vmv.v.v", 0x5e000057, 0xfff0707f },
134 +
    { "vmseq.vv", 0x60000057, 0xfc00707f },
135 +
    { "vmsne.vv", 0x64000057, 0xfc00707f },
136 +
    { "vmsltu.vv", 0x68000057, 0xfc00707f },
137 +
    { "vmslt.vv", 0x6c000057, 0xfc00707f },
138 +
    { "vmsleu.vv", 0x70000057, 0xfc00707f },
139 +
    { "vmsle.vv", 0x74000057, 0xfc00707f },
140 +
    { "vsaddu.vv", 0x80000057, 0xfc00707f },
141 +
    { "vsadd.vv", 0x84000057, 0xfc00707f },
142 +
    { "vssubu.vv", 0x88000057, 0xfc00707f },
143 +
    { "vssub.vv", 0x8c000057, 0xfc00707f },
144 +
    { "vsll.vv", 0x94000057, 0xfc00707f },
145 +
    { "vsmul.vv", 0x9c000057, 0xfc00707f },
146 +
    { "vsrl.vv", 0xa0000057, 0xfc00707f },
147 +
    { "vsra.vv", 0xa4000057, 0xfc00707f },
148 +
    { "vssrl.vv", 0xa8000057, 0xfc00707f },
149 +
    { "vssra.vv", 0xac000057, 0xfc00707f },
150 +
    { "vnsrl.wv", 0xb0000057, 0xfc00707f },
151 +
    { "vnsra.wv", 0xb4000057, 0xfc00707f },
152 +
    { "vnclipu.wv", 0xb8000057, 0xfc00707f },
153 +
    { "vnclip.wv", 0xbc000057, 0xfc00707f },
154 +
    { "vwredsumu.vs", 0xc0000057, 0xfc00707f },
155 +
    { "vwredsum.vs", 0xc4000057, 0xfc00707f },
156 +
    { "vadd.vi", 0x00003057, 0xfc00707f },
157 +
    { "vrsub.vi", 0x0c003057, 0xfc00707f },
158 +
    { "vand.vi", 0x24003057, 0xfc00707f },
159 +
    { "vor.vi", 0x28003057, 0xfc00707f },
160 +
    { "vxor.vi", 0x2c003057, 0xfc00707f },
161 +
    { "vrgather.vi", 0x30003057, 0xfc00707f },
162 +
    { "vslideup.vi", 0x38003057, 0xfc00707f },
163 +
    { "vslidedown.vi", 0x3c003057, 0xfc00707f },
164 +
    { "vadc.vim", 0x40003057, 0xfe00707f },
165 +
    { "vmadc.vim", 0x44003057, 0xfe00707f },
166 +
    { "vmadc.vi", 0x46003057, 0xfe00707f },
167 +
    { "vmerge.vim", 0x5c003057, 0xfe00707f },
168 +
    { "vmv.v.i", 0x5e003057, 0xfff0707f },
169 +
    { "vmseq.vi", 0x60003057, 0xfc00707f },
170 +
    { "vmsne.vi", 0x64003057, 0xfc00707f },
171 +
    { "vmsleu.vi", 0x70003057, 0xfc00707f },
172 +
    { "vmsle.vi", 0x74003057, 0xfc00707f },
173 +
    { "vmsgtu.vi", 0x78003057, 0xfc00707f },
174 +
    { "vmsgt.vi", 0x7c003057, 0xfc00707f },
175 +
    { "vsaddu.vi", 0x80003057, 0xfc00707f },
176 +
    { "vsadd.vi", 0x84003057, 0xfc00707f },
177 +
    { "vsll.vi", 0x94003057, 0xfc00707f },
178 +
    { "vmv1r.v", 0x9e003057, 0xfe0ff07f },
179 +
    { "vmv2r.v", 0x9e00b057, 0xfe0ff07f },
180 +
    { "vmv4r.v", 0x9e01b057, 0xfe0ff07f },
181 +
    { "vmv8r.v", 0x9e03b057, 0xfe0ff07f },
182 +
    { "vsrl.vi", 0xa0003057, 0xfc00707f },
183 +
    { "vsra.vi", 0xa4003057, 0xfc00707f },
184 +
    { "vssrl.vi", 0xa8003057, 0xfc00707f },
185 +
    { "vssra.vi", 0xac003057, 0xfc00707f },
186 +
    { "vnsrl.wi", 0xb0003057, 0xfc00707f },
187 +
    { "vnsra.wi", 0xb4003057, 0xfc00707f },
188 +
    { "vnclipu.wi", 0xb8003057, 0xfc00707f },
189 +
    { "vnclip.wi", 0xbc003057, 0xfc00707f },
190 +
    { "vredsum.vs", 0x00002057, 0xfc00707f },
191 +
    { "vredand.vs", 0x04002057, 0xfc00707f },
192 +
    { "vredor.vs", 0x08002057, 0xfc00707f },
193 +
    { "vredxor.vs", 0x0c002057, 0xfc00707f },
194 +
    { "vredminu.vs", 0x10002057, 0xfc00707f },
195 +
    { "vredmin.vs", 0x14002057, 0xfc00707f },
196 +
    { "vredmaxu.vs", 0x18002057, 0xfc00707f },
197 +
    { "vredmax.vs", 0x1c002057, 0xfc00707f },
198 +
    { "vaaddu.vv", 0x20002057, 0xfc00707f },
199 +
    { "vaadd.vv", 0x24002057, 0xfc00707f },
200 +
    { "vasubu.vv", 0x28002057, 0xfc00707f },
201 +
    { "vasub.vv", 0x2c002057, 0xfc00707f },
202 +
    { "vmv.x.s", 0x42002057, 0xfe0ff07f },
203 +
    { "vzext.vf8", 0x48012057, 0xfc0ff07f },
204 +
    { "vsext.vf8", 0x4801a057, 0xfc0ff07f },
205 +
    { "vzext.vf4", 0x48022057, 0xfc0ff07f },
206 +
    { "vsext.vf4", 0x4802a057, 0xfc0ff07f },
207 +
    { "vzext.vf2", 0x48032057, 0xfc0ff07f },
208 +
    { "vsext.vf2", 0x4803a057, 0xfc0ff07f },
209 +
    { "vcompress.vm", 0x5e002057, 0xfe00707f },
210 +
    { "vmandn.mm", 0x62002057, 0xfe00707f },
211 +
    { "vmand.mm", 0x66002057, 0xfe00707f },
212 +
    { "vmor.mm", 0x6a002057, 0xfe00707f },
213 +
    { "vmxor.mm", 0x6e002057, 0xfe00707f },
214 +
    { "vmorn.mm", 0x72002057, 0xfe00707f },
215 +
    { "vmnand.mm", 0x76002057, 0xfe00707f },
216 +
    { "vmnor.mm", 0x7a002057, 0xfe00707f },
217 +
    { "vmxnor.mm", 0x7e002057, 0xfe00707f },
218 +
    { "vmsbf.m", 0x5000a057, 0xfc0ff07f },
219 +
    { "vmsof.m", 0x50012057, 0xfc0ff07f },
220 +
    { "vmsif.m", 0x5001a057, 0xfc0ff07f },
221 +
    { "viota.m", 0x50082057, 0xfc0ff07f },
222 +
    { "vid.v", 0x5008a057, 0xfdfff07f },
223 +
    { "vcpop.m", 0x40082057, 0xfc0ff07f },
224 +
    { "vfirst.m", 0x4008a057, 0xfc0ff07f },
225 +
    { "vdivu.vv", 0x80002057, 0xfc00707f },
226 +
    { "vdiv.vv", 0x84002057, 0xfc00707f },
227 +
    { "vremu.vv", 0x88002057, 0xfc00707f },
228 +
    { "vrem.vv", 0x8c002057, 0xfc00707f },
229 +
    { "vmulhu.vv", 0x90002057, 0xfc00707f },
230 +
    { "vmul.vv", 0x94002057, 0xfc00707f },
231 +
    { "vmulhsu.vv", 0x98002057, 0xfc00707f },
232 +
    { "vmulh.vv", 0x9c002057, 0xfc00707f },
233 +
    { "vmadd.vv", 0xa4002057, 0xfc00707f },
234 +
    { "vnmsub.vv", 0xac002057, 0xfc00707f },
235 +
    { "vmacc.vv", 0xb4002057, 0xfc00707f },
236 +
    { "vnmsac.vv", 0xbc002057, 0xfc00707f },
237 +
    { "vwaddu.vv", 0xc0002057, 0xfc00707f },
238 +
    { "vwadd.vv", 0xc4002057, 0xfc00707f },
239 +
    { "vwsubu.vv", 0xc8002057, 0xfc00707f },
240 +
    { "vwsub.vv", 0xcc002057, 0xfc00707f },
241 +
    { "vwaddu.wv", 0xd0002057, 0xfc00707f },
242 +
    { "vwadd.wv", 0xd4002057, 0xfc00707f },
243 +
    { "vwsubu.wv", 0xd8002057, 0xfc00707f },
244 +
    { "vwsub.wv", 0xdc002057, 0xfc00707f },
245 +
    { "vwmulu.vv", 0xe0002057, 0xfc00707f },
246 +
    { "vwmulsu.vv", 0xe8002057, 0xfc00707f },
247 +
    { "vwmul.vv", 0xec002057, 0xfc00707f },
248 +
    { "vwmaccu.vv", 0xf0002057, 0xfc00707f },
249 +
    { "vwmacc.vv", 0xf4002057, 0xfc00707f },
250 +
    { "vwmaccsu.vv", 0xfc002057, 0xfc00707f },
251 +
    { "vaaddu.vx", 0x20006057, 0xfc00707f },
252 +
    { "vaadd.vx", 0x24006057, 0xfc00707f },
253 +
    { "vasubu.vx", 0x28006057, 0xfc00707f },
254 +
    { "vasub.vx", 0x2c006057, 0xfc00707f },
255 +
    { "vmv.s.x", 0x42006057, 0xfff0707f },
256 +
    { "vslide1up.vx", 0x38006057, 0xfc00707f },
257 +
    { "vslide1down.vx", 0x3c006057, 0xfc00707f },
258 +
    { "vdivu.vx", 0x80006057, 0xfc00707f },
259 +
    { "vdiv.vx", 0x84006057, 0xfc00707f },
260 +
    { "vremu.vx", 0x88006057, 0xfc00707f },
261 +
    { "vrem.vx", 0x8c006057, 0xfc00707f },
262 +
    { "vmulhu.vx", 0x90006057, 0xfc00707f },
263 +
    { "vmul.vx", 0x94006057, 0xfc00707f },
264 +
    { "vmulhsu.vx", 0x98006057, 0xfc00707f },
265 +
    { "vmulh.vx", 0x9c006057, 0xfc00707f },
266 +
    { "vmadd.vx", 0xa4006057, 0xfc00707f },
267 +
    { "vnmsub.vx", 0xac006057, 0xfc00707f },
268 +
    { "vmacc.vx", 0xb4006057, 0xfc00707f },
269 +
    { "vnmsac.vx", 0xbc006057, 0xfc00707f },
270 +
    { "vwaddu.vx", 0xc0006057, 0xfc00707f },
271 +
    { "vwadd.vx", 0xc4006057, 0xfc00707f },
272 +
    { "vwsubu.vx", 0xc8006057, 0xfc00707f },
273 +
    { "vwsub.vx", 0xcc006057, 0xfc00707f },
274 +
    { "vwaddu.wx", 0xd0006057, 0xfc00707f },
275 +
    { "vwadd.wx", 0xd4006057, 0xfc00707f },
276 +
    { "vwsubu.wx", 0xd8006057, 0xfc00707f },
277 +
    { "vwsub.wx", 0xdc006057, 0xfc00707f },
278 +
    { "vwmulu.vx", 0xe0006057, 0xfc00707f },
279 +
    { "vwmulsu.vx", 0xe8006057, 0xfc00707f },
280 +
    { "vwmul.vx", 0xec006057, 0xfc00707f },
281 +
    { "vwmaccu.vx", 0xf0006057, 0xfc00707f },
282 +
    { "vwmacc.vx", 0xf4006057, 0xfc00707f },
283 +
    { "vwmaccus.vx", 0xf8006057, 0xfc00707f },
284 +
    { "vwmaccsu.vx", 0xfc006057, 0xfc00707f },
285 +
    { "vandn.vv", 0x04000057, 0xfc00707f },
286 +
    { "vandn.vx", 0x04004057, 0xfc00707f },
287 +
    { "vbrev.v", 0x48052057, 0xfc0ff07f },
288 +
    { "vbrev8.v", 0x48042057, 0xfc0ff07f },
289 +
    { "vrev8.v", 0x4804a057, 0xfc0ff07f },
290 +
    { "vclz.v", 0x48062057, 0xfc0ff07f },
291 +
    { "vctz.v", 0x4806a057, 0xfc0ff07f },
292 +
    { "vcpop.v", 0x48072057, 0xfc0ff07f },
293 +
    { "vrol.vv", 0x54000057, 0xfc00707f },
294 +
    { "vrol.vx", 0x54004057, 0xfc00707f },
295 +
    { "vror.vv", 0x50000057, 0xfc00707f },
296 +
    { "vror.vx", 0x50004057, 0xfc00707f },
297 +
    { "vror.vi", 0x50003057, 0xf800707f },
298 +
    { "vwsll.vv", 0xd4000057, 0xfc00707f },
299 +
    { "vwsll.vx", 0xd4004057, 0xfc00707f },
300 +
    { "vwsll.vi", 0xd4003057, 0xfc00707f },
301 +
};
302 +
303 +
/* Return an element mask without a shift by the host word width. */
304 +
static u64 vector_mask(u32 bits) {
305 +
    return bits == 64 ? ~0ULL : (1ULL << bits) - 1;
306 +
}
307 +
308 +
/* Sign-extend an element to the host word width. */
309 +
static i64 vector_signed(u64 value, u32 bits) {
310 +
    u64 sign = 1ULL << (bits - 1);
311 +
    return (i64)((value ^ sign) - sign);
312 +
}
313 +
314 +
/* Read an element from a validated register group. */
315 +
static u64 vector_get(const struct vector_state *v, u32 r, u32 i, u32 bits) {
316 +
    u64 value  = 0;
317 +
    u32 offset = r * VECTOR_BYTES + i * (bits / 8);
318 +
    for (u32 j = 0; j < bits / 8; j++)
319 +
        value |= (u64)((const u8 *)v->reg)[offset + j] << (8 * j);
320 +
    return value;
321 +
}
322 +
323 +
/* Write an element to a validated register group. */
324 +
static void vector_put(
325 +
    struct vector_state *v, u32 r, u32 i, u32 bits, u64 value
326 +
) {
327 +
    u32 offset = r * VECTOR_BYTES + i * (bits / 8);
328 +
    for (u32 j = 0; j < bits / 8; j++)
329 +
        ((u8 *)v->reg)[offset + j] = value >> (8 * j);
330 +
}
331 +
332 +
/* Read a packed mask bit. */
333 +
static bool vector_bit(const struct vector_state *v, u32 r, u32 i) {
334 +
    return (v->reg[r][i / 8] >> (i % 8)) & 1;
335 +
}
336 +
337 +
/* Write a packed mask bit. */
338 +
static void vector_set_bit(struct vector_state *v, u32 r, u32 i, bool bit) {
339 +
    u8 mask          = 1u << (i % 8);
340 +
    v->reg[r][i / 8] = (v->reg[r][i / 8] & ~mask) | (bit ? mask : 0);
341 +
}
342 +
343 +
/* Check register group alignment and architectural bounds. */
344 +
static bool vector_group(u32 reg, u32 bytes) {
345 +
    u32 count = (bytes + VECTOR_BYTES - 1) / VECTOR_BYTES;
346 +
    return bytes >= VECTOR_BYTES / 8 && count && count <= 8 &&
347 +
           reg + count <= 32 && reg % count == 0;
348 +
}
349 +
350 +
/* Process one register per host vector. memcpy permits unaligned registers.
351 +
 * Each lane has an unsigned element type so arithmetic wraps at SEW. */
352 +
#define VECTOR_NATIVE(bits)                                                     \
353 +
    static bool vector_native##bits(                                            \
354 +
        struct vector_state *v, u64 x[32], u32 ins, u32 fn                      \
355 +
    ) {                                                                         \
356 +
        typedef u##bits uv __attribute__((vector_size(VECTOR_BYTES)));          \
357 +
        typedef i##bits sv __attribute__((vector_size(VECTOR_BYTES)));          \
358 +
        typedef u8      bv __attribute__((vector_size(VECTOR_BYTES)));          \
359 +
        u32             d = (ins >> 7) & 31, s1 = (ins >> 15) & 31;             \
360 +
        u32             s2 = (ins >> 20) & 31, form = (ins >> 12) & 7;          \
361 +
        bool            vm     = (ins >> 25) & 1;                               \
362 +
        u32             lanes  = VECTOR_BYTES * 8 / bits;                       \
363 +
        u64             scalar = form == 3 ? (u64)vector_signed(s1, 5) : x[s1]; \
364 +
        if (form == 3 && fn >= 20 && fn <= 22)                                  \
365 +
            scalar = s1 | ((ins >> 21) & 32);                                   \
366 +
        if (form == 3 && (fn == 37 || fn == 40 || fn == 41))                    \
367 +
            scalar = s1;                                                        \
368 +
        uv splat;                                                               \
369 +
        for (u32 j = 0; j < lanes; j++)                                         \
370 +
            splat[j] = scalar;                                                  \
371 +
        for (u32 i = v->vstart / lanes * lanes; i < v->vl; i += lanes) {        \
372 +
            u32 off = i * (bits / 8);                                           \
373 +
            uv  a, b = splat, old, z;                                           \
374 +
            memcpy(&a, (u8 *)v->reg + s2 * VECTOR_BYTES + off, sizeof(a));      \
375 +
            memcpy(&old, (u8 *)v->reg + d * VECTOR_BYTES + off, sizeof(old));   \
376 +
            if (form == 0 || (form == 2 && fn >= 96))                           \
377 +
                memcpy(&b, (u8 *)v->reg + s1 * VECTOR_BYTES + off, sizeof(b));  \
378 +
            uv n = b & (bits - 1), back = -n & (bits - 1);                      \
379 +
            switch (fn) {                                                       \
380 +
            case 0:                                                             \
381 +
                z = a + b;                                                      \
382 +
                break;                                                          \
383 +
            case 1:                                                             \
384 +
                z = a & ~b;                                                     \
385 +
                break;                                                          \
386 +
            case 2:                                                             \
387 +
                z = a - b;                                                      \
388 +
                break;                                                          \
389 +
            case 3:                                                             \
390 +
                z = b - a;                                                      \
391 +
                break;                                                          \
392 +
            case 4:                                                             \
393 +
                z = __builtin_elementwise_min(a, b);                            \
394 +
                break;                                                          \
395 +
            case 5:                                                             \
396 +
                z = (uv)__builtin_elementwise_min((sv)a, (sv)b);                \
397 +
                break;                                                          \
398 +
            case 6:                                                             \
399 +
                z = __builtin_elementwise_max(a, b);                            \
400 +
                break;                                                          \
401 +
            case 7:                                                             \
402 +
                z = (uv)__builtin_elementwise_max((sv)a, (sv)b);                \
403 +
                break;                                                          \
404 +
            case 9:                                                             \
405 +
                z = a & b;                                                      \
406 +
                break;                                                          \
407 +
            case 10:                                                            \
408 +
                z = a | b;                                                      \
409 +
                break;                                                          \
410 +
            case 11:                                                            \
411 +
                z = a ^ b;                                                      \
412 +
                break;                                                          \
413 +
            case 20:                                                            \
414 +
            case 22:                                                            \
415 +
                z = (a >> n) | (a << back);                                     \
416 +
                break;                                                          \
417 +
            case 21:                                                            \
418 +
                z = form == 3 ? (a >> n) | (a << back)                          \
419 +
                              : (a << n) | (a >> back);                         \
420 +
                break;                                                          \
421 +
            case 23:                                                            \
422 +
                if (!vm)                                                        \
423 +
                    return false;                                               \
424 +
                z = b;                                                          \
425 +
                break;                                                          \
426 +
            case 37:                                                            \
427 +
                z = a << n;                                                     \
428 +
                break;                                                          \
429 +
            case 40:                                                            \
430 +
                z = a >> n;                                                     \
431 +
                break;                                                          \
432 +
            case 41:                                                            \
433 +
                z = (uv)((sv)a >> n);                                           \
434 +
                break;                                                          \
435 +
            case 72:                                                            \
436 +
                z = (uv)__builtin_elementwise_bitreverse((bv)a);                \
437 +
                break;                                                          \
438 +
            case 73:                                                            \
439 +
                z = (uv)__builtin_elementwise_bitreverse((bv)a);                \
440 +
                z = __builtin_elementwise_bitreverse(z);                        \
441 +
                break;                                                          \
442 +
            case 74:                                                            \
443 +
                z = __builtin_elementwise_bitreverse(a);                        \
444 +
                break;                                                          \
445 +
            case 76:                                                            \
446 +
                z = __builtin_elementwise_clzg(a, (a ^ a) + bits);              \
447 +
                break;                                                          \
448 +
            case 77:                                                            \
449 +
                z = __builtin_elementwise_ctzg(a, (a ^ a) + bits);              \
450 +
                break;                                                          \
451 +
            case 78:                                                            \
452 +
                z = __builtin_elementwise_popcount(a);                          \
453 +
                break;                                                          \
454 +
            case 101:                                                           \
455 +
                z = a * b;                                                      \
456 +
                break;                                                          \
457 +
            case 105:                                                           \
458 +
                z = b * old + a;                                                \
459 +
                break;                                                          \
460 +
            case 107:                                                           \
461 +
                z = a - b * old;                                                \
462 +
                break;                                                          \
463 +
            case 109:                                                           \
464 +
                z = a * b + old;                                                \
465 +
                break;                                                          \
466 +
            case 111:                                                           \
467 +
                z = old - a * b;                                                \
468 +
                break;                                                          \
469 +
            default:                                                            \
470 +
                return false;                                                   \
471 +
            }                                                                   \
472 +
            if (!vm || i < v->vstart || v->vl - i < lanes) {                    \
473 +
                uv mask;                                                        \
474 +
                for (u32 j = 0; j < lanes; j++)                                 \
475 +
                    mask[j] = i + j >= v->vstart && i + j < v->vl &&            \
476 +
                                      (vm || vector_bit(v, 0, i + j))           \
477 +
                                  ? (u##bits) ~0ULL                             \
478 +
                                  : 0;                                          \
479 +
                z = (z & mask) | (old & ~mask);                                 \
480 +
            }                                                                   \
481 +
            memcpy((u8 *)v->reg + d * VECTOR_BYTES + off, &z, sizeof(z));       \
482 +
        }                                                                       \
483 +
        v->vstart = 0;                                                          \
484 +
        return true;                                                            \
485 +
    }
486 +
487 +
VECTOR_NATIVE(8)
488 +
489 +
VECTOR_NATIVE(16)
490 +
491 +
VECTOR_NATIVE(32)
492 +
493 +
VECTOR_NATIVE(64)
494 +
495 +
#undef VECTOR_NATIVE
496 +
497 +
/* Select lane-wise operations after architectural operand validation.
498 +
 * Host vectors use the register file's little-endian element layout. */
499 +
static bool vector_native(struct vector_state *v, u64 x[32], u32 ins, u32 sew) {
500 +
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
501 +
    u32 form = (ins >> 12) & 7, fn = ins >> 26;
502 +
    if (form == 2 || form == 6) {
503 +
        if (form == 2 && fn == 18) {
504 +
            u32 sel = (ins >> 15) & 31;
505 +
            if (sel != 8 && sel != 9 && sel != 10 && sel != 12 && sel != 13 &&
506 +
                sel != 14)
507 +
                return false;
508 +
            fn = 64 + sel;
509 +
        } else if (fn == 37 || fn == 41 || fn == 43 || fn == 45 || fn == 47)
510 +
            fn += 64;
511 +
        else
512 +
            return false;
513 +
    } else if (form == 0 || form == 3 || form == 4) {
514 +
        switch (fn) {
515 +
        case 0:
516 +
        case 1:
517 +
        case 2:
518 +
        case 3:
519 +
        case 4:
520 +
        case 5:
521 +
        case 6:
522 +
        case 7:
523 +
        case 9:
524 +
        case 10:
525 +
        case 11:
526 +
        case 20:
527 +
        case 21:
528 +
        case 22:
529 +
        case 37:
530 +
        case 40:
531 +
        case 41:
532 +
            break;
533 +
        case 23:
534 +
            if (!((ins >> 25) & 1))
535 +
                return false;
536 +
            break;
537 +
        default:
538 +
            return false;
539 +
        }
540 +
    } else
541 +
        return false;
542 +
    switch (sew) {
543 +
    case 8:
544 +
        return vector_native8(v, x, ins, fn);
545 +
    case 16:
546 +
        return vector_native16(v, x, ins, fn);
547 +
    case 32:
548 +
        return vector_native32(v, x, ins, fn);
549 +
    case 64:
550 +
        return vector_native64(v, x, ins, fn);
551 +
    }
552 +
#endif
553 +
    return false;
554 +
}
555 +
556 +
/* Fixed-point rounding increment for a right shift. */
557 +
static u64 vector_round(u64 value, u32 shift, u32 mode) {
558 +
    if (!shift)
559 +
        return 0;
560 +
    u64  low  = value & vector_mask(shift);
561 +
    bool high = (value >> (shift - 1)) & 1;
562 +
    switch (mode) {
563 +
    case 0:
564 +
        return high;
565 +
    case 1:
566 +
        return high &&
567 +
               ((low & vector_mask(shift - 1)) || ((value >> shift) & 1));
568 +
    case 2:
569 +
        return 0;
570 +
    default:
571 +
        return low && !((value >> shift) & 1);
572 +
    }
573 +
}
574 +
575 +
/* Read or write a vector CSR. */
576 +
bool vector_csr(struct vector_state *v, u32 ins, u64 x[32]) {
577 +
    u32 csr = ins >> 20, f3 = (ins >> 12) & 7, rd = (ins >> 7) & 31,
578 +
        rs = (ins >> 15) & 31;
579 +
    u64 old;
580 +
    switch (csr) {
581 +
    case 8:
582 +
        old = v->vstart;
583 +
        break;
584 +
    case 9:
585 +
        old = v->vxsat;
586 +
        break;
587 +
    case 10:
588 +
        old = v->vxrm;
589 +
        break;
590 +
    case 15:
591 +
        old = v->vxrm * 2 + v->vxsat;
592 +
        break;
593 +
    case 0xc20:
594 +
        old = v->vl;
595 +
        break;
596 +
    case 0xc21:
597 +
        old = v->vtype;
598 +
        break;
599 +
    case 0xc22:
600 +
        old = VECTOR_BYTES;
601 +
        break;
602 +
    default:
603 +
        return false;
604 +
    }
605 +
    if (!(f3 & 3))
606 +
        return false;
607 +
    u64  value = (f3 & 4) ? rs : x[rs];
608 +
    bool write = (f3 & 3) == 1 || rs;
609 +
    if (write && csr >= 0xc00)
610 +
        return false;
611 +
    if (write) {
612 +
        if ((f3 & 3) == 2)
613 +
            value = old | value;
614 +
        if ((f3 & 3) == 3)
615 +
            value = old & ~value;
616 +
        switch (csr) {
617 +
        case 8:
618 +
            v->vstart = value & (VECTOR_BYTES * 8 - 1);
619 +
            break;
620 +
        case 9:
621 +
            v->vxsat = value & 1;
622 +
            break;
623 +
        case 10:
624 +
            v->vxrm = value & 3;
625 +
            break;
626 +
        case 15:
627 +
            v->vxrm  = (value >> 1) & 3;
628 +
            v->vxsat = value & 1;
629 +
            break;
630 +
        }
631 +
    }
632 +
    if (rd)
633 +
        x[rd] = old;
634 +
    return true;
635 +
}
636 +
637 +
/* Execute one integer vector instruction. Inactive and tail elements are
638 +
 * preserved. Faulting memory operations retain the first uncompleted index. */
639 +
enum vector_result vector_execute(
640 +
    struct vector_state *v, u64 x[32], u32 ins, vector_memory memory, void *ctx
641 +
) {
642 +
    u32 op = ins & 127, f3 = (ins >> 12) & 7, d = (ins >> 7) & 31,
643 +
        s1 = (ins >> 15) & 31, s2 = (ins >> 20) & 31;
644 +
    if (op == 0x73)
645 +
        return vector_csr(v, ins, x) ? VECTOR_OK : VECTOR_ILLEGAL;
646 +
    const char *name = NULL;
647 +
    for (u32 j = 0; j < sizeof(vector_opcodes) / sizeof(vector_opcodes[0]); j++)
648 +
        if ((ins & vector_opcodes[j].mask) == vector_opcodes[j].value) {
649 +
            name = vector_opcodes[j].name;
650 +
            break;
651 +
        }
652 +
    if (!name)
653 +
        return VECTOR_ILLEGAL;
654 +
    if (op == 0x57 && f3 == 7) {
655 +
        u64 type, avl;
656 +
        if ((ins >> 30) == 3) {
657 +
            type = (ins >> 20) & 1023;
658 +
            avl  = s1;
659 +
        } else {
660 +
            type = (ins >> 31) ? x[s2] : (ins >> 20) & 2047;
661 +
            avl  = s1 ? x[s1] : d ? ~0ULL : v->vl;
662 +
        }
663 +
        u32  sew = 8u << ((type >> 3) & 7), lm = type & 7;
664 +
        bool invalid = (type & ~255ULL) || sew > 64 || lm == 4;
665 +
        u32  bytes   = lm < 4 ? VECTOR_BYTES << lm : VECTOR_BYTES >> (8 - lm);
666 +
        if (bytes * 8 < sew)
667 +
            invalid = true;
668 +
        u64 max = invalid ? 0 : bytes * 8 / sew;
669 +
        if (!s1 && !d && (ins >> 30) != 3 && !invalid) {
670 +
            u32 oldlm = v->vtype & 7, oldsew = 8u << ((v->vtype >> 3) & 7);
671 +
            u32 oldbytes =
672 +
                oldlm < 4 ? VECTOR_BYTES << oldlm : VECTOR_BYTES >> (8 - oldlm);
673 +
            if ((v->vtype >> 63) || oldbytes * 8 / oldsew != max)
674 +
                return VECTOR_ILLEGAL;
675 +
        }
676 +
        v->vtype  = invalid ? 1ULL << 63 : type;
677 +
        v->vl     = invalid ? 0 : avl < max ? avl : max;
678 +
        v->vstart = 0;
679 +
        if (d)
680 +
            x[d] = v->vl;
681 +
        return VECTOR_OK;
682 +
    }
683 +
    u32  fn  = ins >> 26;
684 +
    u32  sew = 8u << ((v->vtype >> 3) & 7), lm = v->vtype & 7;
685 +
    bool independent =
686 +
        ((op == 7 || op == 39) && ((ins >> 26) & 3) == 0 && s2 == 8) ||
687 +
        (op == 0x57 && f3 == 3 && fn == 0x27);
688 +
    if (v->vtype >> 63 || sew > 64 || lm == 4) {
689 +
        if (!independent)
690 +
            return VECTOR_ILLEGAL;
691 +
        sew = 8;
692 +
        lm  = 0;
693 +
    }
694 +
    u32  bytes    = lm < 4 ? VECTOR_BYTES << lm : VECTOR_BYTES >> (8 - lm);
695 +
    u32  max      = bytes * 8 / sew;
696 +
    bool unmasked = (ins >> 25) & 1;
697 +
    if (op == 7 || op == 39) {
698 +
        struct vector_state source = *v;
699 +
        bool                store  = op == 39;
700 +
        u32                 mode = (ins >> 26) & 3, fields = (ins >> 29) + 1;
701 +
        u32                 width = f3 == 0   ? 8
702 +
                                    : f3 == 5 ? 16
703 +
                                    : f3 == 6 ? 32
704 +
                                    : f3 == 7 ? 64
705 +
                                              : 0;
706 +
        if (!width)
707 +
            return VECTOR_ILLEGAL;
708 +
        u32  eb = mode & 1 ? sew : width, group = bytes * eb / sew;
709 +
        u32  step  = (group + VECTOR_BYTES - 1) / VECTOR_BYTES;
710 +
        bool whole = mode == 0 && s2 == 8, mask = mode == 0 && s2 == 11;
711 +
        u32  count = v->vl;
712 +
        if (whole) {
713 +
            group  = VECTOR_BYTES;
714 +
            step   = 1;
715 +
            eb     = width;
716 +
            count  = fields * VECTOR_BYTES / (eb / 8);
717 +
            fields = 1;
718 +
        }
719 +
        if (mask) {
720 +
            group = VECTOR_BYTES;
721 +
            step  = 1;
722 +
            eb    = 8;
723 +
            count = (v->vl + 7) / 8;
724 +
        }
725 +
        if (whole) {
726 +
            u32 n = (ins >> 29) + 1;
727 +
            if ((n != 1 && n != 2 && n != 4 && n != 8) || d % n || d + n > 32)
728 +
                return VECTOR_ILLEGAL;
729 +
        } else if (
730 +
            !vector_group(d, group) || d + fields * step > 32 ||
731 +
            fields * group > 8 * VECTOR_BYTES
732 +
        )
733 +
            return VECTOR_ILLEGAL;
734 +
        if ((mode & 1) && !vector_group(s2, bytes * width / sew))
735 +
            return VECTOR_ILLEGAL;
736 +
        if ((mode & 1) && !store) {
737 +
            u32 index_count =
738 +
                (bytes * width / sew + VECTOR_BYTES - 1) / VECTOR_BYTES;
739 +
            if (d < s2 + index_count && s2 < d + fields * step)
740 +
                return VECTOR_ILLEGAL;
741 +
        }
742 +
        if (!unmasked && !store && d == 0)
743 +
            return VECTOR_ILLEGAL;
744 +
        for (u32 i = v->vstart; i < count; i++) {
745 +
            if (!whole && !mask && !unmasked && !vector_bit(&source, 0, i))
746 +
                continue;
747 +
            u64 stride = mode == 2 ? x[s2] : fields * (eb / 8);
748 +
            u64 offset = (mode & 1) ? vector_get(&source, s2, i, width)
749 +
                                    : (u64)i * stride;
750 +
            for (u32 field = 0; field < fields; field++) {
751 +
                u32 reg     = d + field * step;
752 +
                u64 address = x[s1] + offset + field * (eb / 8);
753 +
                u64 value   = store ? vector_get(&source, reg, i, eb) : 0;
754 +
                if (!memory(ctx, address, eb / 8, store, &value)) {
755 +
                    if (!store && mode == 0 && s2 == 16 && i > 0) {
756 +
                        v->vl     = i;
757 +
                        v->vstart = 0;
758 +
                        return VECTOR_OK;
759 +
                    }
760 +
                    v->vstart = i;
761 +
                    return VECTOR_FAULT;
762 +
                }
763 +
                if (!store)
764 +
                    vector_put(v, reg, i, eb, value);
765 +
            }
766 +
        }
767 +
        v->vstart = 0;
768 +
        return VECTOR_OK;
769 +
    }
770 +
    if (f3 == 3 && fn == 0x27) {
771 +
        struct vector_state source = *v;
772 +
        u32                 count  = s1 + 1;
773 +
        if ((count != 1 && count != 2 && count != 4 && count != 8) ||
774 +
            d % count || s2 % count || d + count > 32 || s2 + count > 32)
775 +
            return VECTOR_ILLEGAL;
776 +
        for (u32 i = v->vstart; i < count * VECTOR_BYTES * 8 / sew; i++)
777 +
            vector_put(v, d, i, sew, vector_get(&source, s2, i, sew));
778 +
        v->vstart = 0;
779 +
        return VECTOR_OK;
780 +
    }
781 +
    bool maskdest = (f3 == 0 || f3 == 3 || f3 == 4) && fn >= 0x18 && fn <= 0x1f;
782 +
    bool masklogic      = f3 == 2 && fn >= 0x18 && fn <= 0x1f;
783 +
    bool wide_reduction = f3 == 0 && (fn == 0x30 || fn == 0x31);
784 +
    bool reduction      = (f3 == 2 && fn <= 7) || wide_reduction;
785 +
    bool widening       = ((f3 == 2 || f3 == 6) && fn >= 0x30) ||
786 +
                          ((f3 == 0 || f3 == 3 || f3 == 4) && fn == 0x35);
787 +
    bool narrowing =
788 +
        (f3 == 0 || f3 == 3 || f3 == 4) && fn >= 0x2c && fn <= 0x2f;
789 +
    bool wide_source = (f3 == 2 || f3 == 6) && fn >= 0x34 && fn <= 0x37;
790 +
    u32  dbits       = (widening || wide_reduction) ? sew * 2 : sew,
791 +
         sbits       = (narrowing || wide_source) ? sew * 2 : sew;
792 +
    if (dbits > 64 || sbits > 64)
793 +
        return VECTOR_ILLEGAL;
794 +
    bool scalarout   = f3 == 2 && fn == 0x10;
795 +
    bool scalarin    = f3 == 6 && fn == 0x10;
796 +
    bool mask_prefix = f3 == 2 && fn == 0x14 && s1 <= 3;
797 +
    if (scalarout && s1 != 0 && v->vstart)
798 +
        return VECTOR_ILLEGAL;
799 +
    bool mask_source = f3 == 2 && (fn == 0x14 || fn == 0x10);
800 +
    bool extension   = f3 == 2 && fn == 0x12 && s1 >= 2 && s1 <= 7;
801 +
    u32  srcbytes = bytes * sbits / sew, dstbytes = bytes * dbits / sew;
802 +
    if (scalarin)
803 +
        dstbytes = VECTOR_BYTES;
804 +
    if (extension) {
805 +
        u32 factor = 1u << (4 - s1 / 2);
806 +
        if (sew / factor < 8)
807 +
            return VECTOR_ILLEGAL;
808 +
        srcbytes = bytes / factor;
809 +
        sbits    = sew / factor;
810 +
    }
811 +
    if (mask_source || masklogic)
812 +
        srcbytes = VECTOR_BYTES;
813 +
    u32  src1bytes      = (f3 == 0 && fn == 14) ? bytes * 16 / sew : bytes;
814 +
    bool vector_source1 = (f3 == 0 || f3 == 2) && !scalarout && !mask_source &&
815 +
                          !extension && !(f3 == 2 && fn == 0x12);
816 +
    if (reduction || masklogic || (f3 == 2 && fn == 0x17))
817 +
        src1bytes = VECTOR_BYTES;
818 +
    if (!maskdest && !masklogic && !mask_prefix && !scalarout && !reduction &&
819 +
        !vector_group(d, dstbytes))
820 +
        return VECTOR_ILLEGAL;
821 +
    if (!vector_group(s2, srcbytes))
822 +
        return VECTOR_ILLEGAL;
823 +
    if (vector_source1 && !vector_group(s1, src1bytes))
824 +
        return VECTOR_ILLEGAL;
825 +
    if (!unmasked && d == 0 && !maskdest && !scalarout && !reduction)
826 +
        return VECTOR_ILLEGAL;
827 +
    if ((reduction || (f3 == 2 && ((fn == 0x14 && s1 != 17) || fn == 0x17))) &&
828 +
        v->vstart)
829 +
        return VECTOR_ILLEGAL;
830 +
    u32  dn       = (maskdest || masklogic || mask_prefix || reduction)
831 +
                        ? 1
832 +
                        : (dstbytes + VECTOR_BYTES - 1) / VECTOR_BYTES;
833 +
    u32  sn       = (srcbytes + VECTOR_BYTES - 1) / VECTOR_BYTES;
834 +
    u32  s1n      = (src1bytes + VECTOR_BYTES - 1) / VECTOR_BYTES;
835 +
    bool overlap2 = d < s2 + sn && s2 < d + dn;
836 +
    bool overlap1 = vector_source1 && d < s1 + s1n && s1 < d + dn;
837 +
    if (f3 == 2 && fn == 0x14 && s1 != 17 && overlap2)
838 +
        return VECTOR_ILLEGAL;
839 +
    bool gather =
840 +
        (f3 == 0 || f3 == 3 || f3 == 4) && (fn == 12 || (fn == 14 && f3 == 0));
841 +
    bool slideup = fn == 14 && (f3 == 3 || f3 == 4 || f3 == 6);
842 +
    if (maskdest && ((overlap2 && d != s2) || (overlap1 && d != s1)))
843 +
        return VECTOR_ILLEGAL;
844 +
    if (gather && (overlap2 || overlap1))
845 +
        return VECTOR_ILLEGAL;
846 +
    if (slideup && overlap2)
847 +
        return VECTOR_ILLEGAL;
848 +
    if (f3 == 2 && fn == 0x17 &&
849 +
        (overlap2 || (d <= s1 && s1 < d + dn) || !unmasked))
850 +
        return VECTOR_ILLEGAL;
851 +
    if (widening && overlap2 && (srcbytes < VECTOR_BYTES || s2 + sn != d + dn))
852 +
        return VECTOR_ILLEGAL;
853 +
    if (widening && overlap1 &&
854 +
        (src1bytes < VECTOR_BYTES || s1 + s1n != d + dn))
855 +
        return VECTOR_ILLEGAL;
856 +
    if (narrowing && overlap2 && d != s2)
857 +
        return VECTOR_ILLEGAL;
858 +
    if (extension && overlap2 && (srcbytes < VECTOR_BYTES || s2 + sn != d + dn))
859 +
        return VECTOR_ILLEGAL;
860 +
    if (vector_native(v, x, ins, sew))
861 +
        return VECTOR_OK;
862 +
    struct vector_state source = *v;
863 +
    u64  acc        = vector_get(&source, s1, 0, wide_reduction ? dbits : sew),
864 +
         population = 0, first = ~0ULL;
865 +
    u32  compressed = 0;
866 +
    bool found      = false;
867 +
    if (scalarout && s1 == 0) {
868 +
        if (d)
869 +
            x[d] = (u64)vector_signed(vector_get(&source, s2, 0, sew), sew);
870 +
        v->vstart = 0;
871 +
        return VECTOR_OK;
872 +
    }
873 +
    if (f3 == 6 && fn == 0x10) {
874 +
        if (v->vl && !v->vstart)
875 +
            vector_put(v, d, 0, sew, x[s1]);
876 +
        v->vstart = 0;
877 +
        return VECTOR_OK;
878 +
    }
879 +
    for (u32 i = v->vstart; i < v->vl; i++) {
880 +
        bool active = unmasked || vector_bit(&source, 0, i),
881 +
             carry  = vector_bit(&source, 0, i);
882 +
        bool maskop = (f3 == 0 || f3 == 3 || f3 == 4) &&
883 +
                      ((fn >= 0x10 && fn <= 0x13) || fn == 0x17);
884 +
        if (!active && !maskop)
885 +
            continue;
886 +
        u64 a = (mask_source || masklogic) ? 0
887 +
                                           : vector_get(&source, s2, i, sbits),
888 +
            b;
889 +
        if (f3 == 3)
890 +
            b = (u64)vector_signed(s1, 5);
891 +
        else if (f3 == 4 || f3 == 6)
892 +
            b = x[s1];
893 +
        else if (
894 +
            !vector_source1 || masklogic || reduction || (f3 == 2 && fn == 0x17)
895 +
        )
896 +
            b = 0;
897 +
        else
898 +
            b = vector_get(&source, s1, i, (f3 == 0 && fn == 14) ? 16 : sew);
899 +
        if (f3 == 3 &&
900 +
            (fn == 12 || fn == 14 || fn == 15 || fn == 37 || fn >= 40))
901 +
            b = s1;
902 +
        b       &= vector_mask(sew);
903 +
        i64  sa = vector_signed(a, sbits), sb = vector_signed(b, sew);
904 +
        u32  shift = b & (sew - 1);
905 +
        u64  z     = 0;
906 +
        bool bit   = false;
907 +
        if (masklogic) {
908 +
            bool a = vector_bit(&source, s2, i), b = vector_bit(&source, s1, i);
909 +
            switch (fn) {
910 +
            case 0x18:
911 +
                bit = a && !b;
912 +
                break;
913 +
            case 0x19:
914 +
                bit = a && b;
915 +
                break;
916 +
            case 0x1a:
917 +
                bit = a || b;
918 +
                break;
919 +
            case 0x1b:
920 +
                bit = a != b;
921 +
                break;
922 +
            case 0x1c:
923 +
                bit = a || !b;
924 +
                break;
925 +
            case 0x1d:
926 +
                bit = !(a && b);
927 +
                break;
928 +
            case 0x1e:
929 +
                bit = !(a || b);
930 +
                break;
931 +
            case 0x1f:
932 +
                bit = a == b;
933 +
                break;
934 +
            }
935 +
            vector_set_bit(v, d, i, bit);
936 +
            continue;
937 +
        }
938 +
        if (reduction) {
939 +
            if (wide_reduction) {
940 +
                acc += fn == 0x31 ? (u64)sa : a;
941 +
                continue;
942 +
            }
943 +
            switch (fn) {
944 +
            case 0:
945 +
                acc += a;
946 +
                break;
947 +
            case 1:
948 +
                acc &= a;
949 +
                break;
950 +
            case 2:
951 +
                acc |= a;
952 +
                break;
953 +
            case 3:
954 +
                acc ^= a;
955 +
                break;
956 +
            case 4:
957 +
                if (a < acc)
958 +
                    acc = a;
959 +
                break;
960 +
            case 5:
961 +
                if (sa < vector_signed(acc, sew))
962 +
                    acc = a;
963 +
                break;
964 +
            case 6:
965 +
                if (a > acc)
966 +
                    acc = a;
967 +
                break;
968 +
            case 7:
969 +
                if (sa > vector_signed(acc, sew))
970 +
                    acc = a;
971 +
                break;
972 +
            }
973 +
            continue;
974 +
        }
975 +
        if (scalarout) {
976 +
            if (vector_bit(&source, s2, i)) {
977 +
                population++;
978 +
                if (first == ~0ULL)
979 +
                    first = i;
980 +
            }
981 +
            continue;
982 +
        }
983 +
        if (f3 == 2 && fn == 0x14) {
984 +
            bool selected = vector_bit(&source, s2, i);
985 +
            if (s1 == 16)
986 +
                z = population;
987 +
            else if (s1 == 17)
988 +
                z = i;
989 +
            else {
990 +
                bit = s1 == 1   ? !found && !selected
991 +
                      : s1 == 2 ? !found && selected
992 +
                                : !found;
993 +
                vector_set_bit(v, d, i, bit);
994 +
            }
995 +
            if (selected) {
996 +
                population++;
997 +
                found = true;
998 +
            }
999 +
            if (s1 <= 3)
1000 +
                continue;
1001 +
        } else if (f3 == 2 && fn == 0x17) {
1002 +
            if (vector_bit(&source, s1, i))
1003 +
                vector_put(v, d, compressed++, sew, a);
1004 +
            continue;
1005 +
        } else if (f3 == 2 && fn == 0x12) {
1006 +
            if (s1 >= 2 && s1 <= 7) {
1007 +
                u32 bits = sew >> (4 - s1 / 2);
1008 +
                if (bits < 8)
1009 +
                    return VECTOR_ILLEGAL;
1010 +
                z = vector_get(&source, s2, i, bits);
1011 +
                if (s1 & 1)
1012 +
                    z = vector_signed(z, bits);
1013 +
            } else if (s1 == 8 || s1 == 9 || s1 == 10) {
1014 +
                for (u32 j = 0; j < sew; j++) {
1015 +
                    u32 dst  = s1 == 10  ? sew - 1 - j
1016 +
                               : s1 == 8 ? (j & ~7) + (7 - j % 8)
1017 +
                                         : sew - 8 - (j & ~7) + j % 8;
1018 +
                    z       |= ((a >> j) & 1) << dst;
1019 +
                }
1020 +
            } else if (s1 == 12) {
1021 +
                z = a ? (u32)__builtin_clzll(a) - (64 - sew) : sew;
1022 +
            } else if (s1 == 13) {
1023 +
                z = a ? __builtin_ctzll(a) : sew;
1024 +
            } else if (s1 == 14) {
1025 +
                z = __builtin_popcountll(a);
1026 +
            } else
1027 +
                return VECTOR_ILLEGAL;
1028 +
        } else if (f3 == 0 || f3 == 3 || f3 == 4) {
1029 +
            switch (fn) {
1030 +
            case 0:
1031 +
                z = a + b;
1032 +
                break;
1033 +
            case 1:
1034 +
                z = a & ~b;
1035 +
                break;
1036 +
            case 2:
1037 +
                z = a - b;
1038 +
                break;
1039 +
            case 3:
1040 +
                z = b - a;
1041 +
                break;
1042 +
            case 4:
1043 +
                z = a < b ? a : b;
1044 +
                break;
1045 +
            case 5:
1046 +
                z = sa < sb ? a : b;
1047 +
                break;
1048 +
            case 6:
1049 +
                z = a > b ? a : b;
1050 +
                break;
1051 +
            case 7:
1052 +
                z = sa > sb ? a : b;
1053 +
                break;
1054 +
            case 9:
1055 +
                z = a & b;
1056 +
                break;
1057 +
            case 10:
1058 +
                z = a | b;
1059 +
                break;
1060 +
            case 11:
1061 +
                z = a ^ b;
1062 +
                break;
1063 +
            case 12: {
1064 +
                u64 index = f3 == 3 ? s1 : f3 == 4 ? x[s1] : b;
1065 +
                z = index < max ? vector_get(&source, s2, index, sew) : 0;
1066 +
                break;
1067 +
            }
1068 +
            case 14: {
1069 +
                if (f3 == 0) {
1070 +
                    u64 index = vector_get(&source, s1, i, 16);
1071 +
                    z = index < max ? vector_get(&source, s2, index, sew) : 0;
1072 +
                    break;
1073 +
                }
1074 +
                u64 amount = f3 == 3 ? s1 : x[s1];
1075 +
                if (i < amount)
1076 +
                    continue;
1077 +
                z = vector_get(&source, s2, i - amount, sew);
1078 +
                break;
1079 +
            }
1080 +
            case 15: {
1081 +
                u64 amount = f3 == 3 ? s1 : x[s1];
1082 +
                z          = amount < max && i < max - amount
1083 +
                                 ? vector_get(&source, s2, i + amount, sew)
1084 +
                                 : 0;
1085 +
                break;
1086 +
            }
1087 +
            case 16:
1088 +
            case 17: {
1089 +
                __uint128_t sum = (__uint128_t)a + b + (!unmasked && carry);
1090 +
                z               = (u64)sum;
1091 +
                bit             = (sum >> sew) & 1;
1092 +
                if (fn == 17) {
1093 +
                    vector_set_bit(v, d, i, bit);
1094 +
                    continue;
1095 +
                }
1096 +
                break;
1097 +
            }
1098 +
            case 18:
1099 +
            case 19: {
1100 +
                u64 c = !unmasked && carry;
1101 +
                z     = a - b - c;
1102 +
                bit   = a < b || (a == b && c);
1103 +
                if (fn == 19) {
1104 +
                    vector_set_bit(v, d, i, bit);
1105 +
                    continue;
1106 +
                }
1107 +
                break;
1108 +
            }
1109 +
            case 20:
1110 +
            case 21:
1111 +
            case 22: {
1112 +
                u32 n =
1113 +
                    f3 == 3 ? ((ins >> 15) & 31) | ((ins >> 21) & 32) : shift;
1114 +
                n         &= sew - 1;
1115 +
                bool left  = fn == 21 && f3 != 3;
1116 +
                z          = !n     ? a
1117 +
                             : left ? (a << n) | (a >> (sew - n))
1118 +
                                    : (a >> n) | (a << (sew - n));
1119 +
                break;
1120 +
            }
1121 +
            case 23:
1122 +
                z = unmasked ? b : carry ? b : a;
1123 +
                break;
1124 +
            case 24:
1125 +
                bit = a == b;
1126 +
                break;
1127 +
            case 25:
1128 +
                bit = a != b;
1129 +
                break;
1130 +
            case 26:
1131 +
                bit = a < b;
1132 +
                break;
1133 +
            case 27:
1134 +
                bit = sa < sb;
1135 +
                break;
1136 +
            case 28:
1137 +
                bit = a <= b;
1138 +
                break;
1139 +
            case 29:
1140 +
                bit = sa <= sb;
1141 +
                break;
1142 +
            case 30:
1143 +
                bit = a > b;
1144 +
                break;
1145 +
            case 31:
1146 +
                bit = sa > sb;
1147 +
                break;
1148 +
            case 32: {
1149 +
                __uint128_t sum   = (__uint128_t)a + b;
1150 +
                u64         limit = vector_mask(sew);
1151 +
                if (sum > limit) {
1152 +
                    z        = limit;
1153 +
                    v->vxsat = 1;
1154 +
                } else
1155 +
                    z = sum;
1156 +
                break;
1157 +
            }
1158 +
            case 33:
1159 +
            case 35: {
1160 +
                __int128 sum = fn == 33 ? (__int128)sa + sb : (__int128)sa - sb;
1161 +
                __int128 limit = (__int128)1 << (sew - 1);
1162 +
                if (sum >= limit) {
1163 +
                    sum      = limit - 1;
1164 +
                    v->vxsat = 1;
1165 +
                }
1166 +
                if (sum < -limit) {
1167 +
                    sum      = -limit;
1168 +
                    v->vxsat = 1;
1169 +
                }
1170 +
                z = sum;
1171 +
                break;
1172 +
            }
1173 +
            case 34:
1174 +
                if (a < b) {
1175 +
                    z        = 0;
1176 +
                    v->vxsat = 1;
1177 +
                } else
1178 +
                    z = a - b;
1179 +
                break;
1180 +
            case 37:
1181 +
                z = a << shift;
1182 +
                break;
1183 +
            case 39: {
1184 +
                __int128 product = (__int128)sa * sb;
1185 +
                u64      raw     = (u64)product;
1186 +
                __int128 result  = (product >> (sew - 1)) +
1187 +
                                   vector_round(raw, sew - 1, v->vxrm);
1188 +
                __int128 limit   = (__int128)1 << (sew - 1);
1189 +
                if (result >= limit) {
1190 +
                    result   = limit - 1;
1191 +
                    v->vxsat = 1;
1192 +
                }
1193 +
                z = result;
1194 +
                break;
1195 +
            }
1196 +
            case 40:
1197 +
                z = a >> shift;
1198 +
                break;
1199 +
            case 41:
1200 +
                z = sa >> shift;
1201 +
                break;
1202 +
            case 42:
1203 +
                z = (a >> shift) + vector_round(a, shift, v->vxrm);
1204 +
                break;
1205 +
            case 43:
1206 +
                z = (sa >> shift) + vector_round(a, shift, v->vxrm);
1207 +
                break;
1208 +
            case 44:
1209 +
                z = a >> (b & (sbits - 1));
1210 +
                break;
1211 +
            case 45:
1212 +
                z = sa >> (b & (sbits - 1));
1213 +
                break;
1214 +
            case 46:
1215 +
            case 47: {
1216 +
                u32 n = b & (sbits - 1);
1217 +
                if (fn == 46) {
1218 +
                    z = (a >> n) + vector_round(a, n, v->vxrm);
1219 +
                    if (z > vector_mask(sew)) {
1220 +
                        z        = vector_mask(sew);
1221 +
                        v->vxsat = 1;
1222 +
                    }
1223 +
                } else {
1224 +
                    i64 t  = (sa >> n) + vector_round(a, n, v->vxrm);
1225 +
                    i64 hi = (i64)(1ULL << (sew - 1)) - 1, lo = -hi - 1;
1226 +
                    if (t > hi) {
1227 +
                        t        = hi;
1228 +
                        v->vxsat = 1;
1229 +
                    }
1230 +
                    if (t < lo) {
1231 +
                        t        = lo;
1232 +
                        v->vxsat = 1;
1233 +
                    }
1234 +
                    z = t;
1235 +
                }
1236 +
                break;
1237 +
            }
1238 +
            case 53:
1239 +
                z = a << (b & (dbits - 1));
1240 +
                break;
1241 +
            default:
1242 +
                return VECTOR_ILLEGAL;
1243 +
            }
1244 +
        } else if (f3 == 2 || f3 == 6) {
1245 +
            u64 old = vector_get(&source, d, i, dbits);
1246 +
            switch (fn) {
1247 +
            case 8:
1248 +
            case 9:
1249 +
            case 10:
1250 +
            case 11: {
1251 +
                bool     sign = fn & 1, sub = fn & 2;
1252 +
                __int128 sum =
1253 +
                    sign ? (sub ? (__int128)sa - sb : (__int128)sa + sb)
1254 +
                         : (sub ? (__int128)a - b : (__int128)a + b);
1255 +
                z = (sum >> 1) + vector_round((u64)sum, 1, v->vxrm);
1256 +
                break;
1257 +
            }
1258 +
            case 14:
1259 +
                if (i == 0)
1260 +
                    z = b;
1261 +
                else
1262 +
                    z = vector_get(&source, s2, i - 1, sew);
1263 +
                break;
1264 +
            case 15:
1265 +
                if (i + 1 == v->vl)
1266 +
                    z = b;
1267 +
                else
1268 +
                    z = vector_get(&source, s2, i + 1, sew);
1269 +
                break;
1270 +
            case 32:
1271 +
                z = b ? a / b : vector_mask(sew);
1272 +
                break;
1273 +
            case 33:
1274 +
                if (!b)
1275 +
                    z = vector_mask(sew);
1276 +
                else if (
1277 +
                    sa == vector_signed(1ULL << (sew - 1), sew) && sb == -1
1278 +
                )
1279 +
                    z = a;
1280 +
                else
1281 +
                    z = sa / sb;
1282 +
                break;
1283 +
            case 34:
1284 +
                z = b ? a % b : a;
1285 +
                break;
1286 +
            case 35:
1287 +
                if (!b)
1288 +
                    z = a;
1289 +
                else if (sb == -1)
1290 +
                    z = 0;
1291 +
                else
1292 +
                    z = sa % sb;
1293 +
                break;
1294 +
            case 36:
1295 +
                z = ((__uint128_t)a * b) >> sew;
1296 +
                break;
1297 +
            case 37:
1298 +
                z = a * b;
1299 +
                break;
1300 +
            case 38:
1301 +
                z = ((__int128)sa * b) >> sew;
1302 +
                break;
1303 +
            case 39:
1304 +
                z = ((__int128)sa * sb) >> sew;
1305 +
                break;
1306 +
            case 41:
1307 +
                z = b * old + a;
1308 +
                break;
1309 +
            case 43:
1310 +
                z = -b * old + a;
1311 +
                break;
1312 +
            case 45:
1313 +
                z = a * b + old;
1314 +
                break;
1315 +
            case 47:
1316 +
                z = -a * b + old;
1317 +
                break;
1318 +
            case 48:
1319 +
            case 49:
1320 +
            case 50:
1321 +
            case 51:
1322 +
            case 52:
1323 +
            case 53:
1324 +
            case 54:
1325 +
            case 55: {
1326 +
                bool wide = fn >= 52, sign = fn & 1, sub = fn & 2;
1327 +
                u64  aa = wide   ? vector_get(&source, s2, i, dbits)
1328 +
                          : sign ? (u64)sa
1329 +
                                 : a;
1330 +
                u64  bb = sign ? (u64)sb : b;
1331 +
                z       = sub ? aa - bb : aa + bb;
1332 +
                break;
1333 +
            }
1334 +
            case 56:
1335 +
                z = a * b;
1336 +
                break;
1337 +
            case 58:
1338 +
                z = sa * b;
1339 +
                break;
1340 +
            case 59:
1341 +
                z = sa * sb;
1342 +
                break;
1343 +
            case 60:
1344 +
                z = old + a * b;
1345 +
                break;
1346 +
            case 61:
1347 +
                z = old + sa * sb;
1348 +
                break;
1349 +
            case 62:
1350 +
                z = old + sa * b;
1351 +
                break;
1352 +
            case 63:
1353 +
                z = old + a * sb;
1354 +
                break;
1355 +
            default:
1356 +
                return VECTOR_ILLEGAL;
1357 +
            }
1358 +
        } else
1359 +
            return VECTOR_ILLEGAL;
1360 +
        if (maskdest)
1361 +
            vector_set_bit(v, d, i, bit);
1362 +
        else
1363 +
            vector_put(v, d, i, dbits, z);
1364 +
    }
1365 +
    if (reduction && v->vl && source.vstart < v->vl)
1366 +
        vector_put(v, d, 0, wide_reduction ? dbits : sew, acc);
1367 +
    if (scalarout && d)
1368 +
        x[d] = s1 == 16 ? population : first;
1369 +
    v->vstart = 0;
1370 +
    return VECTOR_OK;
1371 +
}
vector.h added +39 -0
1 +
/* Integer vector state and execution for Zve64x and Zvbb. */
2 +
#ifndef VECTOR_H
3 +
#define VECTOR_H
4 +
5 +
#include <string.h>
6 +
7 +
#include "types.h"
8 +
9 +
/* Vector register length in bytes. ELEN is 64 bits. */
10 +
#define VECTOR_BYTES 32
11 +
12 +
/* Architectural vector state. Register groups occupy consecutive registers. */
13 +
struct vector_state {
14 +
    u8  reg[32][VECTOR_BYTES]; /* Architectural register bytes. */
15 +
    u64 vl;                    /* Number of active elements. */
16 +
    u64 vtype;                 /* Width, groups, policies and illegal bit. */
17 +
    u64 vstart;                /* First element to execute on restart. */
18 +
    u32 vxrm;                  /* Fixed-point rounding mode. */
19 +
    u32 vxsat;                 /* Sticky fixed-point saturation flag. */
20 +
};
21 +
22 +
/* Memory callback performs one checked element access. */
23 +
typedef bool (*vector_memory)(void *, u64, u32, bool, u64 *);
24 +
25 +
/* Execution result distinguishes illegal instructions from memory faults. */
26 +
enum vector_result { VECTOR_ILLEGAL, VECTOR_OK, VECTOR_FAULT };
27 +
28 +
/* Read or write an architectural vector CSR. */
29 +
bool vector_csr(struct vector_state *, u32 instruction, u64 registers[32]);
30 +
/* Execute an instruction with checked memory access. */
31 +
enum vector_result vector_execute(
32 +
    struct vector_state *,
33 +
    u64 registers[32],
34 +
    u32 instruction,
35 +
    vector_memory,
36 +
    void *context
37 +
);
38 +
39 +
#endif