vector.c 54.0 KiB raw
1
/* Integer vector instruction execution. */
2
#include "vector.h"
3
4
/* One instruction's fixed opcode fields. */
5
struct vector_opcode {
6
    const char *name;
7
    u32         value, mask;
8
};
9
10
static const struct vector_opcode vector_opcodes[] = {
11
    { "vsetivli", 0xc0007057, 0xc000707f },
12
    { "vsetvli", 0x00007057, 0x8000707f },
13
    { "vsetvl", 0x80007057, 0xfe00707f },
14
    { "vlm.v", 0x02b00007, 0xfff0707f },
15
    { "vsm.v", 0x02b00027, 0xfff0707f },
16
    { "vle8.v", 0x00000007, 0x1df0707f },
17
    { "vle16.v", 0x00005007, 0x1df0707f },
18
    { "vle32.v", 0x00006007, 0x1df0707f },
19
    { "vle64.v", 0x00007007, 0x1df0707f },
20
    { "vse8.v", 0x00000027, 0x1df0707f },
21
    { "vse16.v", 0x00005027, 0x1df0707f },
22
    { "vse32.v", 0x00006027, 0x1df0707f },
23
    { "vse64.v", 0x00007027, 0x1df0707f },
24
    { "vluxei8.v", 0x04000007, 0x1c00707f },
25
    { "vluxei16.v", 0x04005007, 0x1c00707f },
26
    { "vluxei32.v", 0x04006007, 0x1c00707f },
27
    { "vluxei64.v", 0x04007007, 0x1c00707f },
28
    { "vsuxei8.v", 0x04000027, 0x1c00707f },
29
    { "vsuxei16.v", 0x04005027, 0x1c00707f },
30
    { "vsuxei32.v", 0x04006027, 0x1c00707f },
31
    { "vsuxei64.v", 0x04007027, 0x1c00707f },
32
    { "vlse8.v", 0x08000007, 0x1c00707f },
33
    { "vlse16.v", 0x08005007, 0x1c00707f },
34
    { "vlse32.v", 0x08006007, 0x1c00707f },
35
    { "vlse64.v", 0x08007007, 0x1c00707f },
36
    { "vsse8.v", 0x08000027, 0x1c00707f },
37
    { "vsse16.v", 0x08005027, 0x1c00707f },
38
    { "vsse32.v", 0x08006027, 0x1c00707f },
39
    { "vsse64.v", 0x08007027, 0x1c00707f },
40
    { "vloxei8.v", 0x0c000007, 0x1c00707f },
41
    { "vloxei16.v", 0x0c005007, 0x1c00707f },
42
    { "vloxei32.v", 0x0c006007, 0x1c00707f },
43
    { "vloxei64.v", 0x0c007007, 0x1c00707f },
44
    { "vsoxei8.v", 0x0c000027, 0x1c00707f },
45
    { "vsoxei16.v", 0x0c005027, 0x1c00707f },
46
    { "vsoxei32.v", 0x0c006027, 0x1c00707f },
47
    { "vsoxei64.v", 0x0c007027, 0x1c00707f },
48
    { "vle8ff.v", 0x01000007, 0x1df0707f },
49
    { "vle16ff.v", 0x01005007, 0x1df0707f },
50
    { "vle32ff.v", 0x01006007, 0x1df0707f },
51
    { "vle64ff.v", 0x01007007, 0x1df0707f },
52
    { "vl1re8.v", 0x02800007, 0xfff0707f },
53
    { "vl1re16.v", 0x02805007, 0xfff0707f },
54
    { "vl1re32.v", 0x02806007, 0xfff0707f },
55
    { "vl1re64.v", 0x02807007, 0xfff0707f },
56
    { "vl2re8.v", 0x22800007, 0xfff0707f },
57
    { "vl2re16.v", 0x22805007, 0xfff0707f },
58
    { "vl2re32.v", 0x22806007, 0xfff0707f },
59
    { "vl2re64.v", 0x22807007, 0xfff0707f },
60
    { "vl4re8.v", 0x62800007, 0xfff0707f },
61
    { "vl4re16.v", 0x62805007, 0xfff0707f },
62
    { "vl4re32.v", 0x62806007, 0xfff0707f },
63
    { "vl4re64.v", 0x62807007, 0xfff0707f },
64
    { "vl8re8.v", 0xe2800007, 0xfff0707f },
65
    { "vl8re16.v", 0xe2805007, 0xfff0707f },
66
    { "vl8re32.v", 0xe2806007, 0xfff0707f },
67
    { "vl8re64.v", 0xe2807007, 0xfff0707f },
68
    { "vs1r.v", 0x02800027, 0xfff0707f },
69
    { "vs2r.v", 0x22800027, 0xfff0707f },
70
    { "vs4r.v", 0x62800027, 0xfff0707f },
71
    { "vs8r.v", 0xe2800027, 0xfff0707f },
72
    { "vadd.vx", 0x00004057, 0xfc00707f },
73
    { "vsub.vx", 0x08004057, 0xfc00707f },
74
    { "vrsub.vx", 0x0c004057, 0xfc00707f },
75
    { "vminu.vx", 0x10004057, 0xfc00707f },
76
    { "vmin.vx", 0x14004057, 0xfc00707f },
77
    { "vmaxu.vx", 0x18004057, 0xfc00707f },
78
    { "vmax.vx", 0x1c004057, 0xfc00707f },
79
    { "vand.vx", 0x24004057, 0xfc00707f },
80
    { "vor.vx", 0x28004057, 0xfc00707f },
81
    { "vxor.vx", 0x2c004057, 0xfc00707f },
82
    { "vrgather.vx", 0x30004057, 0xfc00707f },
83
    { "vslideup.vx", 0x38004057, 0xfc00707f },
84
    { "vslidedown.vx", 0x3c004057, 0xfc00707f },
85
    { "vadc.vxm", 0x40004057, 0xfe00707f },
86
    { "vmadc.vxm", 0x44004057, 0xfe00707f },
87
    { "vmadc.vx", 0x46004057, 0xfe00707f },
88
    { "vsbc.vxm", 0x48004057, 0xfe00707f },
89
    { "vmsbc.vxm", 0x4c004057, 0xfe00707f },
90
    { "vmsbc.vx", 0x4e004057, 0xfe00707f },
91
    { "vmerge.vxm", 0x5c004057, 0xfe00707f },
92
    { "vmv.v.x", 0x5e004057, 0xfff0707f },
93
    { "vmseq.vx", 0x60004057, 0xfc00707f },
94
    { "vmsne.vx", 0x64004057, 0xfc00707f },
95
    { "vmsltu.vx", 0x68004057, 0xfc00707f },
96
    { "vmslt.vx", 0x6c004057, 0xfc00707f },
97
    { "vmsleu.vx", 0x70004057, 0xfc00707f },
98
    { "vmsle.vx", 0x74004057, 0xfc00707f },
99
    { "vmsgtu.vx", 0x78004057, 0xfc00707f },
100
    { "vmsgt.vx", 0x7c004057, 0xfc00707f },
101
    { "vsaddu.vx", 0x80004057, 0xfc00707f },
102
    { "vsadd.vx", 0x84004057, 0xfc00707f },
103
    { "vssubu.vx", 0x88004057, 0xfc00707f },
104
    { "vssub.vx", 0x8c004057, 0xfc00707f },
105
    { "vsll.vx", 0x94004057, 0xfc00707f },
106
    { "vsmul.vx", 0x9c004057, 0xfc00707f },
107
    { "vsrl.vx", 0xa0004057, 0xfc00707f },
108
    { "vsra.vx", 0xa4004057, 0xfc00707f },
109
    { "vssrl.vx", 0xa8004057, 0xfc00707f },
110
    { "vssra.vx", 0xac004057, 0xfc00707f },
111
    { "vnsrl.wx", 0xb0004057, 0xfc00707f },
112
    { "vnsra.wx", 0xb4004057, 0xfc00707f },
113
    { "vnclipu.wx", 0xb8004057, 0xfc00707f },
114
    { "vnclip.wx", 0xbc004057, 0xfc00707f },
115
    { "vadd.vv", 0x00000057, 0xfc00707f },
116
    { "vsub.vv", 0x08000057, 0xfc00707f },
117
    { "vminu.vv", 0x10000057, 0xfc00707f },
118
    { "vmin.vv", 0x14000057, 0xfc00707f },
119
    { "vmaxu.vv", 0x18000057, 0xfc00707f },
120
    { "vmax.vv", 0x1c000057, 0xfc00707f },
121
    { "vand.vv", 0x24000057, 0xfc00707f },
122
    { "vor.vv", 0x28000057, 0xfc00707f },
123
    { "vxor.vv", 0x2c000057, 0xfc00707f },
124
    { "vrgather.vv", 0x30000057, 0xfc00707f },
125
    { "vrgatherei16.vv", 0x38000057, 0xfc00707f },
126
    { "vadc.vvm", 0x40000057, 0xfe00707f },
127
    { "vmadc.vvm", 0x44000057, 0xfe00707f },
128
    { "vmadc.vv", 0x46000057, 0xfe00707f },
129
    { "vsbc.vvm", 0x48000057, 0xfe00707f },
130
    { "vmsbc.vvm", 0x4c000057, 0xfe00707f },
131
    { "vmsbc.vv", 0x4e000057, 0xfe00707f },
132
    { "vmerge.vvm", 0x5c000057, 0xfe00707f },
133
    { "vmv.v.v", 0x5e000057, 0xfff0707f },
134
    { "vmseq.vv", 0x60000057, 0xfc00707f },
135
    { "vmsne.vv", 0x64000057, 0xfc00707f },
136
    { "vmsltu.vv", 0x68000057, 0xfc00707f },
137
    { "vmslt.vv", 0x6c000057, 0xfc00707f },
138
    { "vmsleu.vv", 0x70000057, 0xfc00707f },
139
    { "vmsle.vv", 0x74000057, 0xfc00707f },
140
    { "vsaddu.vv", 0x80000057, 0xfc00707f },
141
    { "vsadd.vv", 0x84000057, 0xfc00707f },
142
    { "vssubu.vv", 0x88000057, 0xfc00707f },
143
    { "vssub.vv", 0x8c000057, 0xfc00707f },
144
    { "vsll.vv", 0x94000057, 0xfc00707f },
145
    { "vsmul.vv", 0x9c000057, 0xfc00707f },
146
    { "vsrl.vv", 0xa0000057, 0xfc00707f },
147
    { "vsra.vv", 0xa4000057, 0xfc00707f },
148
    { "vssrl.vv", 0xa8000057, 0xfc00707f },
149
    { "vssra.vv", 0xac000057, 0xfc00707f },
150
    { "vnsrl.wv", 0xb0000057, 0xfc00707f },
151
    { "vnsra.wv", 0xb4000057, 0xfc00707f },
152
    { "vnclipu.wv", 0xb8000057, 0xfc00707f },
153
    { "vnclip.wv", 0xbc000057, 0xfc00707f },
154
    { "vwredsumu.vs", 0xc0000057, 0xfc00707f },
155
    { "vwredsum.vs", 0xc4000057, 0xfc00707f },
156
    { "vadd.vi", 0x00003057, 0xfc00707f },
157
    { "vrsub.vi", 0x0c003057, 0xfc00707f },
158
    { "vand.vi", 0x24003057, 0xfc00707f },
159
    { "vor.vi", 0x28003057, 0xfc00707f },
160
    { "vxor.vi", 0x2c003057, 0xfc00707f },
161
    { "vrgather.vi", 0x30003057, 0xfc00707f },
162
    { "vslideup.vi", 0x38003057, 0xfc00707f },
163
    { "vslidedown.vi", 0x3c003057, 0xfc00707f },
164
    { "vadc.vim", 0x40003057, 0xfe00707f },
165
    { "vmadc.vim", 0x44003057, 0xfe00707f },
166
    { "vmadc.vi", 0x46003057, 0xfe00707f },
167
    { "vmerge.vim", 0x5c003057, 0xfe00707f },
168
    { "vmv.v.i", 0x5e003057, 0xfff0707f },
169
    { "vmseq.vi", 0x60003057, 0xfc00707f },
170
    { "vmsne.vi", 0x64003057, 0xfc00707f },
171
    { "vmsleu.vi", 0x70003057, 0xfc00707f },
172
    { "vmsle.vi", 0x74003057, 0xfc00707f },
173
    { "vmsgtu.vi", 0x78003057, 0xfc00707f },
174
    { "vmsgt.vi", 0x7c003057, 0xfc00707f },
175
    { "vsaddu.vi", 0x80003057, 0xfc00707f },
176
    { "vsadd.vi", 0x84003057, 0xfc00707f },
177
    { "vsll.vi", 0x94003057, 0xfc00707f },
178
    { "vmv1r.v", 0x9e003057, 0xfe0ff07f },
179
    { "vmv2r.v", 0x9e00b057, 0xfe0ff07f },
180
    { "vmv4r.v", 0x9e01b057, 0xfe0ff07f },
181
    { "vmv8r.v", 0x9e03b057, 0xfe0ff07f },
182
    { "vsrl.vi", 0xa0003057, 0xfc00707f },
183
    { "vsra.vi", 0xa4003057, 0xfc00707f },
184
    { "vssrl.vi", 0xa8003057, 0xfc00707f },
185
    { "vssra.vi", 0xac003057, 0xfc00707f },
186
    { "vnsrl.wi", 0xb0003057, 0xfc00707f },
187
    { "vnsra.wi", 0xb4003057, 0xfc00707f },
188
    { "vnclipu.wi", 0xb8003057, 0xfc00707f },
189
    { "vnclip.wi", 0xbc003057, 0xfc00707f },
190
    { "vredsum.vs", 0x00002057, 0xfc00707f },
191
    { "vredand.vs", 0x04002057, 0xfc00707f },
192
    { "vredor.vs", 0x08002057, 0xfc00707f },
193
    { "vredxor.vs", 0x0c002057, 0xfc00707f },
194
    { "vredminu.vs", 0x10002057, 0xfc00707f },
195
    { "vredmin.vs", 0x14002057, 0xfc00707f },
196
    { "vredmaxu.vs", 0x18002057, 0xfc00707f },
197
    { "vredmax.vs", 0x1c002057, 0xfc00707f },
198
    { "vaaddu.vv", 0x20002057, 0xfc00707f },
199
    { "vaadd.vv", 0x24002057, 0xfc00707f },
200
    { "vasubu.vv", 0x28002057, 0xfc00707f },
201
    { "vasub.vv", 0x2c002057, 0xfc00707f },
202
    { "vmv.x.s", 0x42002057, 0xfe0ff07f },
203
    { "vzext.vf8", 0x48012057, 0xfc0ff07f },
204
    { "vsext.vf8", 0x4801a057, 0xfc0ff07f },
205
    { "vzext.vf4", 0x48022057, 0xfc0ff07f },
206
    { "vsext.vf4", 0x4802a057, 0xfc0ff07f },
207
    { "vzext.vf2", 0x48032057, 0xfc0ff07f },
208
    { "vsext.vf2", 0x4803a057, 0xfc0ff07f },
209
    { "vcompress.vm", 0x5e002057, 0xfe00707f },
210
    { "vmandn.mm", 0x62002057, 0xfe00707f },
211
    { "vmand.mm", 0x66002057, 0xfe00707f },
212
    { "vmor.mm", 0x6a002057, 0xfe00707f },
213
    { "vmxor.mm", 0x6e002057, 0xfe00707f },
214
    { "vmorn.mm", 0x72002057, 0xfe00707f },
215
    { "vmnand.mm", 0x76002057, 0xfe00707f },
216
    { "vmnor.mm", 0x7a002057, 0xfe00707f },
217
    { "vmxnor.mm", 0x7e002057, 0xfe00707f },
218
    { "vmsbf.m", 0x5000a057, 0xfc0ff07f },
219
    { "vmsof.m", 0x50012057, 0xfc0ff07f },
220
    { "vmsif.m", 0x5001a057, 0xfc0ff07f },
221
    { "viota.m", 0x50082057, 0xfc0ff07f },
222
    { "vid.v", 0x5008a057, 0xfdfff07f },
223
    { "vcpop.m", 0x40082057, 0xfc0ff07f },
224
    { "vfirst.m", 0x4008a057, 0xfc0ff07f },
225
    { "vdivu.vv", 0x80002057, 0xfc00707f },
226
    { "vdiv.vv", 0x84002057, 0xfc00707f },
227
    { "vremu.vv", 0x88002057, 0xfc00707f },
228
    { "vrem.vv", 0x8c002057, 0xfc00707f },
229
    { "vmulhu.vv", 0x90002057, 0xfc00707f },
230
    { "vmul.vv", 0x94002057, 0xfc00707f },
231
    { "vmulhsu.vv", 0x98002057, 0xfc00707f },
232
    { "vmulh.vv", 0x9c002057, 0xfc00707f },
233
    { "vmadd.vv", 0xa4002057, 0xfc00707f },
234
    { "vnmsub.vv", 0xac002057, 0xfc00707f },
235
    { "vmacc.vv", 0xb4002057, 0xfc00707f },
236
    { "vnmsac.vv", 0xbc002057, 0xfc00707f },
237
    { "vwaddu.vv", 0xc0002057, 0xfc00707f },
238
    { "vwadd.vv", 0xc4002057, 0xfc00707f },
239
    { "vwsubu.vv", 0xc8002057, 0xfc00707f },
240
    { "vwsub.vv", 0xcc002057, 0xfc00707f },
241
    { "vwaddu.wv", 0xd0002057, 0xfc00707f },
242
    { "vwadd.wv", 0xd4002057, 0xfc00707f },
243
    { "vwsubu.wv", 0xd8002057, 0xfc00707f },
244
    { "vwsub.wv", 0xdc002057, 0xfc00707f },
245
    { "vwmulu.vv", 0xe0002057, 0xfc00707f },
246
    { "vwmulsu.vv", 0xe8002057, 0xfc00707f },
247
    { "vwmul.vv", 0xec002057, 0xfc00707f },
248
    { "vwmaccu.vv", 0xf0002057, 0xfc00707f },
249
    { "vwmacc.vv", 0xf4002057, 0xfc00707f },
250
    { "vwmaccsu.vv", 0xfc002057, 0xfc00707f },
251
    { "vaaddu.vx", 0x20006057, 0xfc00707f },
252
    { "vaadd.vx", 0x24006057, 0xfc00707f },
253
    { "vasubu.vx", 0x28006057, 0xfc00707f },
254
    { "vasub.vx", 0x2c006057, 0xfc00707f },
255
    { "vmv.s.x", 0x42006057, 0xfff0707f },
256
    { "vslide1up.vx", 0x38006057, 0xfc00707f },
257
    { "vslide1down.vx", 0x3c006057, 0xfc00707f },
258
    { "vdivu.vx", 0x80006057, 0xfc00707f },
259
    { "vdiv.vx", 0x84006057, 0xfc00707f },
260
    { "vremu.vx", 0x88006057, 0xfc00707f },
261
    { "vrem.vx", 0x8c006057, 0xfc00707f },
262
    { "vmulhu.vx", 0x90006057, 0xfc00707f },
263
    { "vmul.vx", 0x94006057, 0xfc00707f },
264
    { "vmulhsu.vx", 0x98006057, 0xfc00707f },
265
    { "vmulh.vx", 0x9c006057, 0xfc00707f },
266
    { "vmadd.vx", 0xa4006057, 0xfc00707f },
267
    { "vnmsub.vx", 0xac006057, 0xfc00707f },
268
    { "vmacc.vx", 0xb4006057, 0xfc00707f },
269
    { "vnmsac.vx", 0xbc006057, 0xfc00707f },
270
    { "vwaddu.vx", 0xc0006057, 0xfc00707f },
271
    { "vwadd.vx", 0xc4006057, 0xfc00707f },
272
    { "vwsubu.vx", 0xc8006057, 0xfc00707f },
273
    { "vwsub.vx", 0xcc006057, 0xfc00707f },
274
    { "vwaddu.wx", 0xd0006057, 0xfc00707f },
275
    { "vwadd.wx", 0xd4006057, 0xfc00707f },
276
    { "vwsubu.wx", 0xd8006057, 0xfc00707f },
277
    { "vwsub.wx", 0xdc006057, 0xfc00707f },
278
    { "vwmulu.vx", 0xe0006057, 0xfc00707f },
279
    { "vwmulsu.vx", 0xe8006057, 0xfc00707f },
280
    { "vwmul.vx", 0xec006057, 0xfc00707f },
281
    { "vwmaccu.vx", 0xf0006057, 0xfc00707f },
282
    { "vwmacc.vx", 0xf4006057, 0xfc00707f },
283
    { "vwmaccus.vx", 0xf8006057, 0xfc00707f },
284
    { "vwmaccsu.vx", 0xfc006057, 0xfc00707f },
285
    { "vandn.vv", 0x04000057, 0xfc00707f },
286
    { "vandn.vx", 0x04004057, 0xfc00707f },
287
    { "vbrev.v", 0x48052057, 0xfc0ff07f },
288
    { "vbrev8.v", 0x48042057, 0xfc0ff07f },
289
    { "vrev8.v", 0x4804a057, 0xfc0ff07f },
290
    { "vclz.v", 0x48062057, 0xfc0ff07f },
291
    { "vctz.v", 0x4806a057, 0xfc0ff07f },
292
    { "vcpop.v", 0x48072057, 0xfc0ff07f },
293
    { "vrol.vv", 0x54000057, 0xfc00707f },
294
    { "vrol.vx", 0x54004057, 0xfc00707f },
295
    { "vror.vv", 0x50000057, 0xfc00707f },
296
    { "vror.vx", 0x50004057, 0xfc00707f },
297
    { "vror.vi", 0x50003057, 0xf800707f },
298
    { "vwsll.vv", 0xd4000057, 0xfc00707f },
299
    { "vwsll.vx", 0xd4004057, 0xfc00707f },
300
    { "vwsll.vi", 0xd4003057, 0xfc00707f },
301
};
302
303
/* Return an element mask without a shift by the host word width. */
304
static u64 vector_mask(u32 bits) {
305
    return bits == 64 ? ~0ULL : (1ULL << bits) - 1;
306
}
307
308
/* Sign-extend an element to the host word width. */
309
static i64 vector_signed(u64 value, u32 bits) {
310
    u64 sign = 1ULL << (bits - 1);
311
    return (i64)((value ^ sign) - sign);
312
}
313
314
/* Read an element from a validated register group. */
315
static u64 vector_get(const struct vector_state *v, u32 r, u32 i, u32 bits) {
316
    u64 value  = 0;
317
    u32 offset = r * VECTOR_BYTES + i * (bits / 8);
318
    for (u32 j = 0; j < bits / 8; j++)
319
        value |= (u64)((const u8 *)v->reg)[offset + j] << (8 * j);
320
    return value;
321
}
322
323
/* Write an element to a validated register group. */
324
static void vector_put(
325
    struct vector_state *v, u32 r, u32 i, u32 bits, u64 value
326
) {
327
    u32 offset = r * VECTOR_BYTES + i * (bits / 8);
328
    for (u32 j = 0; j < bits / 8; j++)
329
        ((u8 *)v->reg)[offset + j] = value >> (8 * j);
330
}
331
332
/* Read a packed mask bit. */
333
static bool vector_bit(const struct vector_state *v, u32 r, u32 i) {
334
    return (v->reg[r][i / 8] >> (i % 8)) & 1;
335
}
336
337
/* Write a packed mask bit. */
338
static void vector_set_bit(struct vector_state *v, u32 r, u32 i, bool bit) {
339
    u8 mask          = 1u << (i % 8);
340
    v->reg[r][i / 8] = (v->reg[r][i / 8] & ~mask) | (bit ? mask : 0);
341
}
342
343
/* Check register group alignment and architectural bounds. */
344
static bool vector_group(u32 reg, u32 bytes) {
345
    u32 count = (bytes + VECTOR_BYTES - 1) / VECTOR_BYTES;
346
    return bytes >= VECTOR_BYTES / 8 && count && count <= 8 &&
347
           reg + count <= 32 && reg % count == 0;
348
}
349
350
/* Process one register per host vector. memcpy permits unaligned registers.
351
 * Each lane has an unsigned element type so arithmetic wraps at SEW. */
352
#define VECTOR_NATIVE(bits)                                                     \
353
    static bool vector_native##bits(                                            \
354
        struct vector_state *v, u64 x[32], u32 ins, u32 fn                      \
355
    ) {                                                                         \
356
        typedef u##bits uv __attribute__((vector_size(VECTOR_BYTES)));          \
357
        typedef i##bits sv __attribute__((vector_size(VECTOR_BYTES)));          \
358
        typedef u8      bv __attribute__((vector_size(VECTOR_BYTES)));          \
359
        u32             d = (ins >> 7) & 31, s1 = (ins >> 15) & 31;             \
360
        u32             s2 = (ins >> 20) & 31, form = (ins >> 12) & 7;          \
361
        bool            vm     = (ins >> 25) & 1;                               \
362
        u32             lanes  = VECTOR_BYTES * 8 / bits;                       \
363
        u64             scalar = form == 3 ? (u64)vector_signed(s1, 5) : x[s1]; \
364
        if (form == 3 && fn >= 20 && fn <= 22)                                  \
365
            scalar = s1 | ((ins >> 21) & 32);                                   \
366
        if (form == 3 && (fn == 37 || fn == 40 || fn == 41))                    \
367
            scalar = s1;                                                        \
368
        uv splat;                                                               \
369
        for (u32 j = 0; j < lanes; j++)                                         \
370
            splat[j] = scalar;                                                  \
371
        for (u32 i = v->vstart / lanes * lanes; i < v->vl; i += lanes) {        \
372
            u32 off = i * (bits / 8);                                           \
373
            uv  a, b = splat, old, z;                                           \
374
            memcpy(&a, (u8 *)v->reg + s2 * VECTOR_BYTES + off, sizeof(a));      \
375
            memcpy(&old, (u8 *)v->reg + d * VECTOR_BYTES + off, sizeof(old));   \
376
            if (form == 0 || (form == 2 && fn >= 96))                           \
377
                memcpy(&b, (u8 *)v->reg + s1 * VECTOR_BYTES + off, sizeof(b));  \
378
            uv n = b & (bits - 1), back = -n & (bits - 1);                      \
379
            switch (fn) {                                                       \
380
            case 0:                                                             \
381
                z = a + b;                                                      \
382
                break;                                                          \
383
            case 1:                                                             \
384
                z = a & ~b;                                                     \
385
                break;                                                          \
386
            case 2:                                                             \
387
                z = a - b;                                                      \
388
                break;                                                          \
389
            case 3:                                                             \
390
                z = b - a;                                                      \
391
                break;                                                          \
392
            case 4:                                                             \
393
                z = __builtin_elementwise_min(a, b);                            \
394
                break;                                                          \
395
            case 5:                                                             \
396
                z = (uv)__builtin_elementwise_min((sv)a, (sv)b);                \
397
                break;                                                          \
398
            case 6:                                                             \
399
                z = __builtin_elementwise_max(a, b);                            \
400
                break;                                                          \
401
            case 7:                                                             \
402
                z = (uv)__builtin_elementwise_max((sv)a, (sv)b);                \
403
                break;                                                          \
404
            case 9:                                                             \
405
                z = a & b;                                                      \
406
                break;                                                          \
407
            case 10:                                                            \
408
                z = a | b;                                                      \
409
                break;                                                          \
410
            case 11:                                                            \
411
                z = a ^ b;                                                      \
412
                break;                                                          \
413
            case 20:                                                            \
414
            case 22:                                                            \
415
                z = (a >> n) | (a << back);                                     \
416
                break;                                                          \
417
            case 21:                                                            \
418
                z = form == 3 ? (a >> n) | (a << back)                          \
419
                              : (a << n) | (a >> back);                         \
420
                break;                                                          \
421
            case 23:                                                            \
422
                if (!vm)                                                        \
423
                    return false;                                               \
424
                z = b;                                                          \
425
                break;                                                          \
426
            case 37:                                                            \
427
                z = a << n;                                                     \
428
                break;                                                          \
429
            case 40:                                                            \
430
                z = a >> n;                                                     \
431
                break;                                                          \
432
            case 41:                                                            \
433
                z = (uv)((sv)a >> n);                                           \
434
                break;                                                          \
435
            case 72:                                                            \
436
                z = (uv)__builtin_elementwise_bitreverse((bv)a);                \
437
                break;                                                          \
438
            case 73:                                                            \
439
                z = (uv)__builtin_elementwise_bitreverse((bv)a);                \
440
                z = __builtin_elementwise_bitreverse(z);                        \
441
                break;                                                          \
442
            case 74:                                                            \
443
                z = __builtin_elementwise_bitreverse(a);                        \
444
                break;                                                          \
445
            case 76:                                                            \
446
                z = __builtin_elementwise_clzg(a, (a ^ a) + bits);              \
447
                break;                                                          \
448
            case 77:                                                            \
449
                z = __builtin_elementwise_ctzg(a, (a ^ a) + bits);              \
450
                break;                                                          \
451
            case 78:                                                            \
452
                z = __builtin_elementwise_popcount(a);                          \
453
                break;                                                          \
454
            case 101:                                                           \
455
                z = a * b;                                                      \
456
                break;                                                          \
457
            case 105:                                                           \
458
                z = b * old + a;                                                \
459
                break;                                                          \
460
            case 107:                                                           \
461
                z = a - b * old;                                                \
462
                break;                                                          \
463
            case 109:                                                           \
464
                z = a * b + old;                                                \
465
                break;                                                          \
466
            case 111:                                                           \
467
                z = old - a * b;                                                \
468
                break;                                                          \
469
            default:                                                            \
470
                return false;                                                   \
471
            }                                                                   \
472
            if (!vm || i < v->vstart || v->vl - i < lanes) {                    \
473
                uv mask;                                                        \
474
                for (u32 j = 0; j < lanes; j++)                                 \
475
                    mask[j] = i + j >= v->vstart && i + j < v->vl &&            \
476
                                      (vm || vector_bit(v, 0, i + j))           \
477
                                  ? (u##bits) ~0ULL                             \
478
                                  : 0;                                          \
479
                z = (z & mask) | (old & ~mask);                                 \
480
            }                                                                   \
481
            memcpy((u8 *)v->reg + d * VECTOR_BYTES + off, &z, sizeof(z));       \
482
        }                                                                       \
483
        v->vstart = 0;                                                          \
484
        return true;                                                            \
485
    }
486
487
VECTOR_NATIVE(8)
488
489
VECTOR_NATIVE(16)
490
491
VECTOR_NATIVE(32)
492
493
VECTOR_NATIVE(64)
494
495
#undef VECTOR_NATIVE
496
497
/* Select lane-wise operations after architectural operand validation.
498
 * Host vectors use the register file's little-endian element layout. */
499
static bool vector_native(struct vector_state *v, u64 x[32], u32 ins, u32 sew) {
500
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
501
    u32 form = (ins >> 12) & 7, fn = ins >> 26;
502
    if (form == 2 || form == 6) {
503
        if (form == 2 && fn == 18) {
504
            u32 sel = (ins >> 15) & 31;
505
            if (sel != 8 && sel != 9 && sel != 10 && sel != 12 && sel != 13 &&
506
                sel != 14)
507
                return false;
508
            fn = 64 + sel;
509
        } else if (fn == 37 || fn == 41 || fn == 43 || fn == 45 || fn == 47)
510
            fn += 64;
511
        else
512
            return false;
513
    } else if (form == 0 || form == 3 || form == 4) {
514
        switch (fn) {
515
        case 0:
516
        case 1:
517
        case 2:
518
        case 3:
519
        case 4:
520
        case 5:
521
        case 6:
522
        case 7:
523
        case 9:
524
        case 10:
525
        case 11:
526
        case 20:
527
        case 21:
528
        case 22:
529
        case 37:
530
        case 40:
531
        case 41:
532
            break;
533
        case 23:
534
            if (!((ins >> 25) & 1))
535
                return false;
536
            break;
537
        default:
538
            return false;
539
        }
540
    } else
541
        return false;
542
    switch (sew) {
543
    case 8:
544
        return vector_native8(v, x, ins, fn);
545
    case 16:
546
        return vector_native16(v, x, ins, fn);
547
    case 32:
548
        return vector_native32(v, x, ins, fn);
549
    case 64:
550
        return vector_native64(v, x, ins, fn);
551
    }
552
#endif
553
    return false;
554
}
555
556
/* Fixed-point rounding increment for a right shift. */
557
static u64 vector_round(u64 value, u32 shift, u32 mode) {
558
    if (!shift)
559
        return 0;
560
    u64  low  = value & vector_mask(shift);
561
    bool high = (value >> (shift - 1)) & 1;
562
    switch (mode) {
563
    case 0:
564
        return high;
565
    case 1:
566
        return high &&
567
               ((low & vector_mask(shift - 1)) || ((value >> shift) & 1));
568
    case 2:
569
        return 0;
570
    default:
571
        return low && !((value >> shift) & 1);
572
    }
573
}
574
575
/* Read or write a vector CSR. */
576
bool vector_csr(struct vector_state *v, u32 ins, u64 x[32]) {
577
    u32 csr = ins >> 20, f3 = (ins >> 12) & 7, rd = (ins >> 7) & 31,
578
        rs = (ins >> 15) & 31;
579
    u64 old;
580
    switch (csr) {
581
    case 8:
582
        old = v->vstart;
583
        break;
584
    case 9:
585
        old = v->vxsat;
586
        break;
587
    case 10:
588
        old = v->vxrm;
589
        break;
590
    case 15:
591
        old = v->vxrm * 2 + v->vxsat;
592
        break;
593
    case 0xc20:
594
        old = v->vl;
595
        break;
596
    case 0xc21:
597
        old = v->vtype;
598
        break;
599
    case 0xc22:
600
        old = VECTOR_BYTES;
601
        break;
602
    default:
603
        return false;
604
    }
605
    if (!(f3 & 3))
606
        return false;
607
    u64  value = (f3 & 4) ? rs : x[rs];
608
    bool write = (f3 & 3) == 1 || rs;
609
    if (write && csr >= 0xc00)
610
        return false;
611
    if (write) {
612
        if ((f3 & 3) == 2)
613
            value = old | value;
614
        if ((f3 & 3) == 3)
615
            value = old & ~value;
616
        switch (csr) {
617
        case 8:
618
            v->vstart = value & (VECTOR_BYTES * 8 - 1);
619
            break;
620
        case 9:
621
            v->vxsat = value & 1;
622
            break;
623
        case 10:
624
            v->vxrm = value & 3;
625
            break;
626
        case 15:
627
            v->vxrm  = (value >> 1) & 3;
628
            v->vxsat = value & 1;
629
            break;
630
        }
631
    }
632
    if (rd)
633
        x[rd] = old;
634
    return true;
635
}
636
637
/* Execute one integer vector instruction. Inactive and tail elements are
638
 * preserved. Faulting memory operations retain the first uncompleted index. */
639
enum vector_result vector_execute(
640
    struct vector_state *v, u64 x[32], u32 ins, vector_memory memory, void *ctx
641
) {
642
    u32 op = ins & 127, f3 = (ins >> 12) & 7, d = (ins >> 7) & 31,
643
        s1 = (ins >> 15) & 31, s2 = (ins >> 20) & 31;
644
    if (op == 0x73)
645
        return vector_csr(v, ins, x) ? VECTOR_OK : VECTOR_ILLEGAL;
646
    const char *name = NULL;
647
    for (u32 j = 0; j < sizeof(vector_opcodes) / sizeof(vector_opcodes[0]); j++)
648
        if ((ins & vector_opcodes[j].mask) == vector_opcodes[j].value) {
649
            name = vector_opcodes[j].name;
650
            break;
651
        }
652
    if (!name)
653
        return VECTOR_ILLEGAL;
654
    if (op == 0x57 && f3 == 7) {
655
        u64 type, avl;
656
        if ((ins >> 30) == 3) {
657
            type = (ins >> 20) & 1023;
658
            avl  = s1;
659
        } else {
660
            type = (ins >> 31) ? x[s2] : (ins >> 20) & 2047;
661
            avl  = s1 ? x[s1] : d ? ~0ULL : v->vl;
662
        }
663
        u32  sew = 8u << ((type >> 3) & 7), lm = type & 7;
664
        bool invalid = (type & ~255ULL) || sew > 64 || lm == 4;
665
        u32  bytes   = lm < 4 ? VECTOR_BYTES << lm : VECTOR_BYTES >> (8 - lm);
666
        if (bytes * 8 < sew)
667
            invalid = true;
668
        u64 max = invalid ? 0 : bytes * 8 / sew;
669
        if (!s1 && !d && (ins >> 30) != 3 && !invalid) {
670
            u32 oldlm = v->vtype & 7, oldsew = 8u << ((v->vtype >> 3) & 7);
671
            u32 oldbytes =
672
                oldlm < 4 ? VECTOR_BYTES << oldlm : VECTOR_BYTES >> (8 - oldlm);
673
            if ((v->vtype >> 63) || oldbytes * 8 / oldsew != max)
674
                return VECTOR_ILLEGAL;
675
        }
676
        v->vtype  = invalid ? 1ULL << 63 : type;
677
        v->vl     = invalid ? 0 : avl < max ? avl : max;
678
        v->vstart = 0;
679
        if (d)
680
            x[d] = v->vl;
681
        return VECTOR_OK;
682
    }
683
    u32  fn  = ins >> 26;
684
    u32  sew = 8u << ((v->vtype >> 3) & 7), lm = v->vtype & 7;
685
    bool independent =
686
        ((op == 7 || op == 39) && ((ins >> 26) & 3) == 0 && s2 == 8) ||
687
        (op == 0x57 && f3 == 3 && fn == 0x27);
688
    if (v->vtype >> 63 || sew > 64 || lm == 4) {
689
        if (!independent)
690
            return VECTOR_ILLEGAL;
691
        sew = 8;
692
        lm  = 0;
693
    }
694
    u32  bytes    = lm < 4 ? VECTOR_BYTES << lm : VECTOR_BYTES >> (8 - lm);
695
    u32  max      = bytes * 8 / sew;
696
    bool unmasked = (ins >> 25) & 1;
697
    if (op == 7 || op == 39) {
698
        struct vector_state source = *v;
699
        bool                store  = op == 39;
700
        u32                 mode = (ins >> 26) & 3, fields = (ins >> 29) + 1;
701
        u32                 width = f3 == 0   ? 8
702
                                    : f3 == 5 ? 16
703
                                    : f3 == 6 ? 32
704
                                    : f3 == 7 ? 64
705
                                              : 0;
706
        if (!width)
707
            return VECTOR_ILLEGAL;
708
        u32  eb = mode & 1 ? sew : width, group = bytes * eb / sew;
709
        u32  step  = (group + VECTOR_BYTES - 1) / VECTOR_BYTES;
710
        bool whole = mode == 0 && s2 == 8, mask = mode == 0 && s2 == 11;
711
        u32  count = v->vl;
712
        if (whole) {
713
            group  = VECTOR_BYTES;
714
            step   = 1;
715
            eb     = width;
716
            count  = fields * VECTOR_BYTES / (eb / 8);
717
            fields = 1;
718
        }
719
        if (mask) {
720
            group = VECTOR_BYTES;
721
            step  = 1;
722
            eb    = 8;
723
            count = (v->vl + 7) / 8;
724
        }
725
        if (whole) {
726
            u32 n = (ins >> 29) + 1;
727
            if ((n != 1 && n != 2 && n != 4 && n != 8) || d % n || d + n > 32)
728
                return VECTOR_ILLEGAL;
729
        } else if (
730
            !vector_group(d, group) || d + fields * step > 32 ||
731
            fields * group > 8 * VECTOR_BYTES
732
        )
733
            return VECTOR_ILLEGAL;
734
        if ((mode & 1) && !vector_group(s2, bytes * width / sew))
735
            return VECTOR_ILLEGAL;
736
        if ((mode & 1) && !store) {
737
            u32 index_count =
738
                (bytes * width / sew + VECTOR_BYTES - 1) / VECTOR_BYTES;
739
            if (d < s2 + index_count && s2 < d + fields * step)
740
                return VECTOR_ILLEGAL;
741
        }
742
        if (!unmasked && !store && d == 0)
743
            return VECTOR_ILLEGAL;
744
        for (u32 i = v->vstart; i < count; i++) {
745
            if (!whole && !mask && !unmasked && !vector_bit(&source, 0, i))
746
                continue;
747
            u64 stride = mode == 2 ? x[s2] : fields * (eb / 8);
748
            u64 offset = (mode & 1) ? vector_get(&source, s2, i, width)
749
                                    : (u64)i * stride;
750
            for (u32 field = 0; field < fields; field++) {
751
                u32 reg     = d + field * step;
752
                u64 address = x[s1] + offset + field * (eb / 8);
753
                u64 value   = store ? vector_get(&source, reg, i, eb) : 0;
754
                if (!memory(ctx, address, eb / 8, store, &value)) {
755
                    if (!store && mode == 0 && s2 == 16 && i > 0) {
756
                        v->vl     = i;
757
                        v->vstart = 0;
758
                        return VECTOR_OK;
759
                    }
760
                    v->vstart = i;
761
                    return VECTOR_FAULT;
762
                }
763
                if (!store)
764
                    vector_put(v, reg, i, eb, value);
765
            }
766
        }
767
        v->vstart = 0;
768
        return VECTOR_OK;
769
    }
770
    if (f3 == 3 && fn == 0x27) {
771
        struct vector_state source = *v;
772
        u32                 count  = s1 + 1;
773
        if ((count != 1 && count != 2 && count != 4 && count != 8) ||
774
            d % count || s2 % count || d + count > 32 || s2 + count > 32)
775
            return VECTOR_ILLEGAL;
776
        for (u32 i = v->vstart; i < count * VECTOR_BYTES * 8 / sew; i++)
777
            vector_put(v, d, i, sew, vector_get(&source, s2, i, sew));
778
        v->vstart = 0;
779
        return VECTOR_OK;
780
    }
781
    bool maskdest = (f3 == 0 || f3 == 3 || f3 == 4) && fn >= 0x18 && fn <= 0x1f;
782
    bool masklogic      = f3 == 2 && fn >= 0x18 && fn <= 0x1f;
783
    bool wide_reduction = f3 == 0 && (fn == 0x30 || fn == 0x31);
784
    bool reduction      = (f3 == 2 && fn <= 7) || wide_reduction;
785
    bool widening       = ((f3 == 2 || f3 == 6) && fn >= 0x30) ||
786
                          ((f3 == 0 || f3 == 3 || f3 == 4) && fn == 0x35);
787
    bool narrowing =
788
        (f3 == 0 || f3 == 3 || f3 == 4) && fn >= 0x2c && fn <= 0x2f;
789
    bool wide_source = (f3 == 2 || f3 == 6) && fn >= 0x34 && fn <= 0x37;
790
    u32  dbits       = (widening || wide_reduction) ? sew * 2 : sew,
791
         sbits       = (narrowing || wide_source) ? sew * 2 : sew;
792
    if (dbits > 64 || sbits > 64)
793
        return VECTOR_ILLEGAL;
794
    bool scalarout   = f3 == 2 && fn == 0x10;
795
    bool scalarin    = f3 == 6 && fn == 0x10;
796
    bool mask_prefix = f3 == 2 && fn == 0x14 && s1 <= 3;
797
    if (scalarout && s1 != 0 && v->vstart)
798
        return VECTOR_ILLEGAL;
799
    bool mask_source = f3 == 2 && (fn == 0x14 || fn == 0x10);
800
    bool extension   = f3 == 2 && fn == 0x12 && s1 >= 2 && s1 <= 7;
801
    u32  srcbytes = bytes * sbits / sew, dstbytes = bytes * dbits / sew;
802
    if (scalarin)
803
        dstbytes = VECTOR_BYTES;
804
    if (extension) {
805
        u32 factor = 1u << (4 - s1 / 2);
806
        if (sew / factor < 8)
807
            return VECTOR_ILLEGAL;
808
        srcbytes = bytes / factor;
809
        sbits    = sew / factor;
810
    }
811
    if (mask_source || masklogic)
812
        srcbytes = VECTOR_BYTES;
813
    u32  src1bytes      = (f3 == 0 && fn == 14) ? bytes * 16 / sew : bytes;
814
    bool vector_source1 = (f3 == 0 || f3 == 2) && !scalarout && !mask_source &&
815
                          !extension && !(f3 == 2 && fn == 0x12);
816
    if (reduction || masklogic || (f3 == 2 && fn == 0x17))
817
        src1bytes = VECTOR_BYTES;
818
    if (!maskdest && !masklogic && !mask_prefix && !scalarout && !reduction &&
819
        !vector_group(d, dstbytes))
820
        return VECTOR_ILLEGAL;
821
    if (!vector_group(s2, srcbytes))
822
        return VECTOR_ILLEGAL;
823
    if (vector_source1 && !vector_group(s1, src1bytes))
824
        return VECTOR_ILLEGAL;
825
    if (!unmasked && d == 0 && !maskdest && !scalarout && !reduction)
826
        return VECTOR_ILLEGAL;
827
    if ((reduction || (f3 == 2 && ((fn == 0x14 && s1 != 17) || fn == 0x17))) &&
828
        v->vstart)
829
        return VECTOR_ILLEGAL;
830
    u32  dn       = (maskdest || masklogic || mask_prefix || reduction)
831
                        ? 1
832
                        : (dstbytes + VECTOR_BYTES - 1) / VECTOR_BYTES;
833
    u32  sn       = (srcbytes + VECTOR_BYTES - 1) / VECTOR_BYTES;
834
    u32  s1n      = (src1bytes + VECTOR_BYTES - 1) / VECTOR_BYTES;
835
    bool overlap2 = d < s2 + sn && s2 < d + dn;
836
    bool overlap1 = vector_source1 && d < s1 + s1n && s1 < d + dn;
837
    if (f3 == 2 && fn == 0x14 && s1 != 17 && overlap2)
838
        return VECTOR_ILLEGAL;
839
    bool gather =
840
        (f3 == 0 || f3 == 3 || f3 == 4) && (fn == 12 || (fn == 14 && f3 == 0));
841
    bool slideup = fn == 14 && (f3 == 3 || f3 == 4 || f3 == 6);
842
    if (maskdest && ((overlap2 && d != s2) || (overlap1 && d != s1)))
843
        return VECTOR_ILLEGAL;
844
    if (gather && (overlap2 || overlap1))
845
        return VECTOR_ILLEGAL;
846
    if (slideup && overlap2)
847
        return VECTOR_ILLEGAL;
848
    if (f3 == 2 && fn == 0x17 &&
849
        (overlap2 || (d <= s1 && s1 < d + dn) || !unmasked))
850
        return VECTOR_ILLEGAL;
851
    if (widening && overlap2 && (srcbytes < VECTOR_BYTES || s2 + sn != d + dn))
852
        return VECTOR_ILLEGAL;
853
    if (widening && overlap1 &&
854
        (src1bytes < VECTOR_BYTES || s1 + s1n != d + dn))
855
        return VECTOR_ILLEGAL;
856
    if (narrowing && overlap2 && d != s2)
857
        return VECTOR_ILLEGAL;
858
    if (extension && overlap2 && (srcbytes < VECTOR_BYTES || s2 + sn != d + dn))
859
        return VECTOR_ILLEGAL;
860
    if (vector_native(v, x, ins, sew))
861
        return VECTOR_OK;
862
    struct vector_state source = *v;
863
    u64  acc        = vector_get(&source, s1, 0, wide_reduction ? dbits : sew),
864
         population = 0, first = ~0ULL;
865
    u32  compressed = 0;
866
    bool found      = false;
867
    if (scalarout && s1 == 0) {
868
        if (d)
869
            x[d] = (u64)vector_signed(vector_get(&source, s2, 0, sew), sew);
870
        v->vstart = 0;
871
        return VECTOR_OK;
872
    }
873
    if (f3 == 6 && fn == 0x10) {
874
        if (v->vl && !v->vstart)
875
            vector_put(v, d, 0, sew, x[s1]);
876
        v->vstart = 0;
877
        return VECTOR_OK;
878
    }
879
    for (u32 i = v->vstart; i < v->vl; i++) {
880
        bool active = unmasked || vector_bit(&source, 0, i),
881
             carry  = vector_bit(&source, 0, i);
882
        bool maskop = (f3 == 0 || f3 == 3 || f3 == 4) &&
883
                      ((fn >= 0x10 && fn <= 0x13) || fn == 0x17);
884
        if (!active && !maskop)
885
            continue;
886
        u64 a = (mask_source || masklogic) ? 0
887
                                           : vector_get(&source, s2, i, sbits),
888
            b;
889
        if (f3 == 3)
890
            b = (u64)vector_signed(s1, 5);
891
        else if (f3 == 4 || f3 == 6)
892
            b = x[s1];
893
        else if (
894
            !vector_source1 || masklogic || reduction || (f3 == 2 && fn == 0x17)
895
        )
896
            b = 0;
897
        else
898
            b = vector_get(&source, s1, i, (f3 == 0 && fn == 14) ? 16 : sew);
899
        if (f3 == 3 &&
900
            (fn == 12 || fn == 14 || fn == 15 || fn == 37 || fn >= 40))
901
            b = s1;
902
        b       &= vector_mask(sew);
903
        i64  sa = vector_signed(a, sbits), sb = vector_signed(b, sew);
904
        u32  shift = b & (sew - 1);
905
        u64  z     = 0;
906
        bool bit   = false;
907
        if (masklogic) {
908
            bool a = vector_bit(&source, s2, i), b = vector_bit(&source, s1, i);
909
            switch (fn) {
910
            case 0x18:
911
                bit = a && !b;
912
                break;
913
            case 0x19:
914
                bit = a && b;
915
                break;
916
            case 0x1a:
917
                bit = a || b;
918
                break;
919
            case 0x1b:
920
                bit = a != b;
921
                break;
922
            case 0x1c:
923
                bit = a || !b;
924
                break;
925
            case 0x1d:
926
                bit = !(a && b);
927
                break;
928
            case 0x1e:
929
                bit = !(a || b);
930
                break;
931
            case 0x1f:
932
                bit = a == b;
933
                break;
934
            }
935
            vector_set_bit(v, d, i, bit);
936
            continue;
937
        }
938
        if (reduction) {
939
            if (wide_reduction) {
940
                acc += fn == 0x31 ? (u64)sa : a;
941
                continue;
942
            }
943
            switch (fn) {
944
            case 0:
945
                acc += a;
946
                break;
947
            case 1:
948
                acc &= a;
949
                break;
950
            case 2:
951
                acc |= a;
952
                break;
953
            case 3:
954
                acc ^= a;
955
                break;
956
            case 4:
957
                if (a < acc)
958
                    acc = a;
959
                break;
960
            case 5:
961
                if (sa < vector_signed(acc, sew))
962
                    acc = a;
963
                break;
964
            case 6:
965
                if (a > acc)
966
                    acc = a;
967
                break;
968
            case 7:
969
                if (sa > vector_signed(acc, sew))
970
                    acc = a;
971
                break;
972
            }
973
            continue;
974
        }
975
        if (scalarout) {
976
            if (vector_bit(&source, s2, i)) {
977
                population++;
978
                if (first == ~0ULL)
979
                    first = i;
980
            }
981
            continue;
982
        }
983
        if (f3 == 2 && fn == 0x14) {
984
            bool selected = vector_bit(&source, s2, i);
985
            if (s1 == 16)
986
                z = population;
987
            else if (s1 == 17)
988
                z = i;
989
            else {
990
                bit = s1 == 1   ? !found && !selected
991
                      : s1 == 2 ? !found && selected
992
                                : !found;
993
                vector_set_bit(v, d, i, bit);
994
            }
995
            if (selected) {
996
                population++;
997
                found = true;
998
            }
999
            if (s1 <= 3)
1000
                continue;
1001
        } else if (f3 == 2 && fn == 0x17) {
1002
            if (vector_bit(&source, s1, i))
1003
                vector_put(v, d, compressed++, sew, a);
1004
            continue;
1005
        } else if (f3 == 2 && fn == 0x12) {
1006
            if (s1 >= 2 && s1 <= 7) {
1007
                u32 bits = sew >> (4 - s1 / 2);
1008
                if (bits < 8)
1009
                    return VECTOR_ILLEGAL;
1010
                z = vector_get(&source, s2, i, bits);
1011
                if (s1 & 1)
1012
                    z = vector_signed(z, bits);
1013
            } else if (s1 == 8 || s1 == 9 || s1 == 10) {
1014
                for (u32 j = 0; j < sew; j++) {
1015
                    u32 dst  = s1 == 10  ? sew - 1 - j
1016
                               : s1 == 8 ? (j & ~7) + (7 - j % 8)
1017
                                         : sew - 8 - (j & ~7) + j % 8;
1018
                    z       |= ((a >> j) & 1) << dst;
1019
                }
1020
            } else if (s1 == 12) {
1021
                z = a ? (u32)__builtin_clzll(a) - (64 - sew) : sew;
1022
            } else if (s1 == 13) {
1023
                z = a ? __builtin_ctzll(a) : sew;
1024
            } else if (s1 == 14) {
1025
                z = __builtin_popcountll(a);
1026
            } else
1027
                return VECTOR_ILLEGAL;
1028
        } else if (f3 == 0 || f3 == 3 || f3 == 4) {
1029
            switch (fn) {
1030
            case 0:
1031
                z = a + b;
1032
                break;
1033
            case 1:
1034
                z = a & ~b;
1035
                break;
1036
            case 2:
1037
                z = a - b;
1038
                break;
1039
            case 3:
1040
                z = b - a;
1041
                break;
1042
            case 4:
1043
                z = a < b ? a : b;
1044
                break;
1045
            case 5:
1046
                z = sa < sb ? a : b;
1047
                break;
1048
            case 6:
1049
                z = a > b ? a : b;
1050
                break;
1051
            case 7:
1052
                z = sa > sb ? a : b;
1053
                break;
1054
            case 9:
1055
                z = a & b;
1056
                break;
1057
            case 10:
1058
                z = a | b;
1059
                break;
1060
            case 11:
1061
                z = a ^ b;
1062
                break;
1063
            case 12: {
1064
                u64 index = f3 == 3 ? s1 : f3 == 4 ? x[s1] : b;
1065
                z = index < max ? vector_get(&source, s2, index, sew) : 0;
1066
                break;
1067
            }
1068
            case 14: {
1069
                if (f3 == 0) {
1070
                    u64 index = vector_get(&source, s1, i, 16);
1071
                    z = index < max ? vector_get(&source, s2, index, sew) : 0;
1072
                    break;
1073
                }
1074
                u64 amount = f3 == 3 ? s1 : x[s1];
1075
                if (i < amount)
1076
                    continue;
1077
                z = vector_get(&source, s2, i - amount, sew);
1078
                break;
1079
            }
1080
            case 15: {
1081
                u64 amount = f3 == 3 ? s1 : x[s1];
1082
                z          = amount < max && i < max - amount
1083
                                 ? vector_get(&source, s2, i + amount, sew)
1084
                                 : 0;
1085
                break;
1086
            }
1087
            case 16:
1088
            case 17: {
1089
                __uint128_t sum = (__uint128_t)a + b + (!unmasked && carry);
1090
                z               = (u64)sum;
1091
                bit             = (sum >> sew) & 1;
1092
                if (fn == 17) {
1093
                    vector_set_bit(v, d, i, bit);
1094
                    continue;
1095
                }
1096
                break;
1097
            }
1098
            case 18:
1099
            case 19: {
1100
                u64 c = !unmasked && carry;
1101
                z     = a - b - c;
1102
                bit   = a < b || (a == b && c);
1103
                if (fn == 19) {
1104
                    vector_set_bit(v, d, i, bit);
1105
                    continue;
1106
                }
1107
                break;
1108
            }
1109
            case 20:
1110
            case 21:
1111
            case 22: {
1112
                u32 n =
1113
                    f3 == 3 ? ((ins >> 15) & 31) | ((ins >> 21) & 32) : shift;
1114
                n         &= sew - 1;
1115
                bool left  = fn == 21 && f3 != 3;
1116
                z          = !n     ? a
1117
                             : left ? (a << n) | (a >> (sew - n))
1118
                                    : (a >> n) | (a << (sew - n));
1119
                break;
1120
            }
1121
            case 23:
1122
                z = unmasked ? b : carry ? b : a;
1123
                break;
1124
            case 24:
1125
                bit = a == b;
1126
                break;
1127
            case 25:
1128
                bit = a != b;
1129
                break;
1130
            case 26:
1131
                bit = a < b;
1132
                break;
1133
            case 27:
1134
                bit = sa < sb;
1135
                break;
1136
            case 28:
1137
                bit = a <= b;
1138
                break;
1139
            case 29:
1140
                bit = sa <= sb;
1141
                break;
1142
            case 30:
1143
                bit = a > b;
1144
                break;
1145
            case 31:
1146
                bit = sa > sb;
1147
                break;
1148
            case 32: {
1149
                __uint128_t sum   = (__uint128_t)a + b;
1150
                u64         limit = vector_mask(sew);
1151
                if (sum > limit) {
1152
                    z        = limit;
1153
                    v->vxsat = 1;
1154
                } else
1155
                    z = sum;
1156
                break;
1157
            }
1158
            case 33:
1159
            case 35: {
1160
                __int128 sum = fn == 33 ? (__int128)sa + sb : (__int128)sa - sb;
1161
                __int128 limit = (__int128)1 << (sew - 1);
1162
                if (sum >= limit) {
1163
                    sum      = limit - 1;
1164
                    v->vxsat = 1;
1165
                }
1166
                if (sum < -limit) {
1167
                    sum      = -limit;
1168
                    v->vxsat = 1;
1169
                }
1170
                z = sum;
1171
                break;
1172
            }
1173
            case 34:
1174
                if (a < b) {
1175
                    z        = 0;
1176
                    v->vxsat = 1;
1177
                } else
1178
                    z = a - b;
1179
                break;
1180
            case 37:
1181
                z = a << shift;
1182
                break;
1183
            case 39: {
1184
                __int128 product = (__int128)sa * sb;
1185
                u64      raw     = (u64)product;
1186
                __int128 result  = (product >> (sew - 1)) +
1187
                                   vector_round(raw, sew - 1, v->vxrm);
1188
                __int128 limit   = (__int128)1 << (sew - 1);
1189
                if (result >= limit) {
1190
                    result   = limit - 1;
1191
                    v->vxsat = 1;
1192
                }
1193
                z = result;
1194
                break;
1195
            }
1196
            case 40:
1197
                z = a >> shift;
1198
                break;
1199
            case 41:
1200
                z = sa >> shift;
1201
                break;
1202
            case 42:
1203
                z = (a >> shift) + vector_round(a, shift, v->vxrm);
1204
                break;
1205
            case 43:
1206
                z = (sa >> shift) + vector_round(a, shift, v->vxrm);
1207
                break;
1208
            case 44:
1209
                z = a >> (b & (sbits - 1));
1210
                break;
1211
            case 45:
1212
                z = sa >> (b & (sbits - 1));
1213
                break;
1214
            case 46:
1215
            case 47: {
1216
                u32 n = b & (sbits - 1);
1217
                if (fn == 46) {
1218
                    z = (a >> n) + vector_round(a, n, v->vxrm);
1219
                    if (z > vector_mask(sew)) {
1220
                        z        = vector_mask(sew);
1221
                        v->vxsat = 1;
1222
                    }
1223
                } else {
1224
                    i64 t  = (sa >> n) + vector_round(a, n, v->vxrm);
1225
                    i64 hi = (i64)(1ULL << (sew - 1)) - 1, lo = -hi - 1;
1226
                    if (t > hi) {
1227
                        t        = hi;
1228
                        v->vxsat = 1;
1229
                    }
1230
                    if (t < lo) {
1231
                        t        = lo;
1232
                        v->vxsat = 1;
1233
                    }
1234
                    z = t;
1235
                }
1236
                break;
1237
            }
1238
            case 53:
1239
                z = a << (b & (dbits - 1));
1240
                break;
1241
            default:
1242
                return VECTOR_ILLEGAL;
1243
            }
1244
        } else if (f3 == 2 || f3 == 6) {
1245
            u64 old = vector_get(&source, d, i, dbits);
1246
            switch (fn) {
1247
            case 8:
1248
            case 9:
1249
            case 10:
1250
            case 11: {
1251
                bool     sign = fn & 1, sub = fn & 2;
1252
                __int128 sum =
1253
                    sign ? (sub ? (__int128)sa - sb : (__int128)sa + sb)
1254
                         : (sub ? (__int128)a - b : (__int128)a + b);
1255
                z = (sum >> 1) + vector_round((u64)sum, 1, v->vxrm);
1256
                break;
1257
            }
1258
            case 14:
1259
                if (i == 0)
1260
                    z = b;
1261
                else
1262
                    z = vector_get(&source, s2, i - 1, sew);
1263
                break;
1264
            case 15:
1265
                if (i + 1 == v->vl)
1266
                    z = b;
1267
                else
1268
                    z = vector_get(&source, s2, i + 1, sew);
1269
                break;
1270
            case 32:
1271
                z = b ? a / b : vector_mask(sew);
1272
                break;
1273
            case 33:
1274
                if (!b)
1275
                    z = vector_mask(sew);
1276
                else if (
1277
                    sa == vector_signed(1ULL << (sew - 1), sew) && sb == -1
1278
                )
1279
                    z = a;
1280
                else
1281
                    z = sa / sb;
1282
                break;
1283
            case 34:
1284
                z = b ? a % b : a;
1285
                break;
1286
            case 35:
1287
                if (!b)
1288
                    z = a;
1289
                else if (sb == -1)
1290
                    z = 0;
1291
                else
1292
                    z = sa % sb;
1293
                break;
1294
            case 36:
1295
                z = ((__uint128_t)a * b) >> sew;
1296
                break;
1297
            case 37:
1298
                z = a * b;
1299
                break;
1300
            case 38:
1301
                z = ((__int128)sa * b) >> sew;
1302
                break;
1303
            case 39:
1304
                z = ((__int128)sa * sb) >> sew;
1305
                break;
1306
            case 41:
1307
                z = b * old + a;
1308
                break;
1309
            case 43:
1310
                z = -b * old + a;
1311
                break;
1312
            case 45:
1313
                z = a * b + old;
1314
                break;
1315
            case 47:
1316
                z = -a * b + old;
1317
                break;
1318
            case 48:
1319
            case 49:
1320
            case 50:
1321
            case 51:
1322
            case 52:
1323
            case 53:
1324
            case 54:
1325
            case 55: {
1326
                bool wide = fn >= 52, sign = fn & 1, sub = fn & 2;
1327
                u64  aa = wide   ? vector_get(&source, s2, i, dbits)
1328
                          : sign ? (u64)sa
1329
                                 : a;
1330
                u64  bb = sign ? (u64)sb : b;
1331
                z       = sub ? aa - bb : aa + bb;
1332
                break;
1333
            }
1334
            case 56:
1335
                z = a * b;
1336
                break;
1337
            case 58:
1338
                z = sa * b;
1339
                break;
1340
            case 59:
1341
                z = sa * sb;
1342
                break;
1343
            case 60:
1344
                z = old + a * b;
1345
                break;
1346
            case 61:
1347
                z = old + sa * sb;
1348
                break;
1349
            case 62:
1350
                z = old + sa * b;
1351
                break;
1352
            case 63:
1353
                z = old + a * sb;
1354
                break;
1355
            default:
1356
                return VECTOR_ILLEGAL;
1357
            }
1358
        } else
1359
            return VECTOR_ILLEGAL;
1360
        if (maskdest)
1361
            vector_set_bit(v, d, i, bit);
1362
        else
1363
            vector_put(v, d, i, dbits, z);
1364
    }
1365
    if (reduction && v->vl && source.vstart < v->vl)
1366
        vector_put(v, d, 0, wide_reduction ? dbits : sew, acc);
1367
    if (scalarout && d)
1368
        x[d] = s1 == 16 ? population : first;
1369
    v->vstart = 0;
1370
    return VECTOR_OK;
1371
}