emulator: Support Zba, Zbb, and Zbs

2ce71aa63250fbf65f1dc130763a6cc339bab1d5fbc1a58f946b2f4c3cda4e72
Run all RV64 bit manipulation instructions in the hosted interpreter,
machine interpreter, and JIT. Decode the instructions in the debugger
and report the extensions in the machine device tree.

Use direct register loads, scaled LEA, immediate bit operations,
native count instructions, and SIMD byte comparisons to reduce host
instruction counts. Check host CPU features before selecting native
extensions. Add execution coverage for operand boundaries and
register aliases in all three engines.

Assisted-by: Codex:gpt-6
Alexis Sellier committed ago 1 parent 8b28a824
Makefile +1 -1
18 18
                 image_raw.c host_posix.c jit.c riscv/debug.c io.c riscv.c \
19 19
                 machine.c machine_cpu.c machine_csr.c machine_devices.c machine_fdt.c
20 20
21 21
default: bin/emulator
22 22
23 -
bin/emulator: $(EMULATOR_SRCS) jit.h muldiv.h color.h types.h io.h host.h hosted.h riscv.h riscv/debug.h machine.h machine_internal.h
23 +
bin/emulator: $(EMULATOR_SRCS) jit.h bitmanip.h muldiv.h color.h types.h io.h host.h hosted.h riscv.h riscv/debug.h machine.h machine_internal.h
24 24
	@echo "cc    emulator => $@"
25 25
	@mkdir -p bin
26 26
	@$(CC) $(CFLAGS) $(LDFLAGS) $(EMULATOR_SRCS) -o $@
27 27
28 28
install: bin/emulator
bitmanip.h added +253 -0
1 +
/* RV64 Zba, Zbb, and Zbs instruction decoding and execution. */
2 +
#ifndef BITMANIP_H
3 +
#define BITMANIP_H
4 +
5 +
#include "types.h"
6 +
#include <emmintrin.h>
7 +
8 +
/* Required bit manipulation operations. Zero denotes another instruction. */
9 +
enum bitmanip_op {
10 +
    BM_NONE,
11 +
    BM_ADDUW,
12 +
    BM_SH1ADD,
13 +
    BM_SH2ADD,
14 +
    BM_SH3ADD,
15 +
    BM_SH1ADDUW,
16 +
    BM_SH2ADDUW,
17 +
    BM_SH3ADDUW,
18 +
    BM_ANDN,
19 +
    BM_ORN,
20 +
    BM_XNOR,
21 +
    BM_MIN,
22 +
    BM_MINU,
23 +
    BM_MAX,
24 +
    BM_MAXU,
25 +
    BM_ROL,
26 +
    BM_ROR,
27 +
    BM_ROLW,
28 +
    BM_RORW,
29 +
    BM_BCLR,
30 +
    BM_BEXT,
31 +
    BM_BINV,
32 +
    BM_BSET,
33 +
    BM_CLZ,
34 +
    BM_CTZ,
35 +
    BM_CPOP,
36 +
    BM_CLZW,
37 +
    BM_CTZW,
38 +
    BM_CPOPW,
39 +
    BM_SEXTB,
40 +
    BM_SEXTH,
41 +
    BM_ORCB,
42 +
    BM_REV8,
43 +
    BM_ZEXTH,
44 +
    BM_SLLIUW,
45 +
    BM_RORI,
46 +
    BM_RORIW,
47 +
    BM_BCLRI,
48 +
    BM_BEXTI,
49 +
    BM_BINVI,
50 +
    BM_BSETI,
51 +
};
52 +
53 +
/* Decode only architecturally defined operand forms. */
54 +
static inline enum bitmanip_op bitmanip_decode(u32 ins) {
55 +
    switch (ins & 0xfe00707fU) {
56 +
    case 0x6000501b:
57 +
        return BM_RORIW;
58 +
    case 0x0800003b:
59 +
        return BM_ADDUW;
60 +
    case 0x20002033:
61 +
        return BM_SH1ADD;
62 +
    case 0x20004033:
63 +
        return BM_SH2ADD;
64 +
    case 0x20006033:
65 +
        return BM_SH3ADD;
66 +
    case 0x2000203b:
67 +
        return BM_SH1ADDUW;
68 +
    case 0x2000403b:
69 +
        return BM_SH2ADDUW;
70 +
    case 0x2000603b:
71 +
        return BM_SH3ADDUW;
72 +
    case 0x40007033:
73 +
        return BM_ANDN;
74 +
    case 0x40006033:
75 +
        return BM_ORN;
76 +
    case 0x40004033:
77 +
        return BM_XNOR;
78 +
    case 0x0a004033:
79 +
        return BM_MIN;
80 +
    case 0x0a005033:
81 +
        return BM_MINU;
82 +
    case 0x0a006033:
83 +
        return BM_MAX;
84 +
    case 0x0a007033:
85 +
        return BM_MAXU;
86 +
    case 0x60001033:
87 +
        return BM_ROL;
88 +
    case 0x60005033:
89 +
        return BM_ROR;
90 +
    case 0x6000103b:
91 +
        return BM_ROLW;
92 +
    case 0x6000503b:
93 +
        return BM_RORW;
94 +
    case 0x48001033:
95 +
        return BM_BCLR;
96 +
    case 0x48005033:
97 +
        return BM_BEXT;
98 +
    case 0x68001033:
99 +
        return BM_BINV;
100 +
    case 0x28001033:
101 +
        return BM_BSET;
102 +
    default:
103 +
        break;
104 +
    }
105 +
    switch (ins & 0xfff0707fU) {
106 +
    case 0x60001013:
107 +
        return BM_CLZ;
108 +
    case 0x60101013:
109 +
        return BM_CTZ;
110 +
    case 0x60201013:
111 +
        return BM_CPOP;
112 +
    case 0x6000101b:
113 +
        return BM_CLZW;
114 +
    case 0x6010101b:
115 +
        return BM_CTZW;
116 +
    case 0x6020101b:
117 +
        return BM_CPOPW;
118 +
    case 0x60401013:
119 +
        return BM_SEXTB;
120 +
    case 0x60501013:
121 +
        return BM_SEXTH;
122 +
    case 0x28705013:
123 +
        return BM_ORCB;
124 +
    case 0x6b805013:
125 +
        return BM_REV8;
126 +
    case 0x0800403b:
127 +
        return BM_ZEXTH;
128 +
    default:
129 +
        break;
130 +
    }
131 +
    switch (ins & 0xfc00707fU) {
132 +
    case 0x0800101b:
133 +
        return BM_SLLIUW;
134 +
    case 0x60005013:
135 +
        return BM_RORI;
136 +
    case 0x48001013:
137 +
        return BM_BCLRI;
138 +
    case 0x48005013:
139 +
        return BM_BEXTI;
140 +
    case 0x68001013:
141 +
        return BM_BINVI;
142 +
    case 0x28001013:
143 +
        return BM_BSETI;
144 +
    default:
145 +
        break;
146 +
    }
147 +
    return BM_NONE;
148 +
}
149 +
150 +
/* Evaluate a decoded operation with unsigned register values. */
151 +
static inline u64 bitmanip_execute(enum bitmanip_op op, u32 ins, u64 a, u64 b) {
152 +
    u32 n = (ins >> 20) & 63;
153 +
    switch (op) {
154 +
    case BM_ADDUW:
155 +
        return (u32)a + b;
156 +
    case BM_SH1ADD:
157 +
        return ((u64)a << 1) + b;
158 +
    case BM_SH2ADD:
159 +
        return ((u64)a << 2) + b;
160 +
    case BM_SH3ADD:
161 +
        return ((u64)a << 3) + b;
162 +
    case BM_SH1ADDUW:
163 +
        return ((u64)(u32)a << 1) + b;
164 +
    case BM_SH2ADDUW:
165 +
        return ((u64)(u32)a << 2) + b;
166 +
    case BM_SH3ADDUW:
167 +
        return ((u64)(u32)a << 3) + b;
168 +
    case BM_ANDN:
169 +
        return a & ~b;
170 +
    case BM_ORN:
171 +
        return a | ~b;
172 +
    case BM_XNOR:
173 +
        return ~(a ^ b);
174 +
    case BM_MIN:
175 +
        return (i64)a < (i64)b ? a : b;
176 +
    case BM_MINU:
177 +
        return a < b ? a : b;
178 +
    case BM_MAX:
179 +
        return (i64)a > (i64)b ? a : b;
180 +
    case BM_MAXU:
181 +
        return a > b ? a : b;
182 +
    case BM_ROL:
183 +
        n = (u32)b & 63;
184 +
        return (a << n) | (a >> ((0U - n) & 63));
185 +
    case BM_ROR:
186 +
        n = (u32)b & 63;
187 +
        return (a >> n) | (a << ((0U - n) & 63));
188 +
    case BM_ROLW:
189 +
        n = (u32)b & 31;
190 +
        return (u64)(i64)(i32)(((u32)a << n) | ((u32)a >> ((0U - n) & 31)));
191 +
    case BM_RORW:
192 +
        n = (u32)b & 31;
193 +
        return (u64)(i64)(i32)(((u32)a >> n) | ((u32)a << ((0U - n) & 31)));
194 +
    case BM_BCLR:
195 +
        n = (u32)b & 63;
196 +
        return a & ~(1ULL << n);
197 +
    case BM_BEXT:
198 +
        n = (u32)b & 63;
199 +
        return (a >> n) & 1;
200 +
    case BM_BINV:
201 +
        n = (u32)b & 63;
202 +
        return a ^ (1ULL << n);
203 +
    case BM_BSET:
204 +
        n = (u32)b & 63;
205 +
        return a | (1ULL << n);
206 +
    case BM_CLZ:
207 +
        return a ? (u64)__builtin_clzll(a) : 64;
208 +
    case BM_CTZ:
209 +
        return a ? (u64)__builtin_ctzll(a) : 64;
210 +
    case BM_CPOP:
211 +
        return (u64)__builtin_popcountll(a);
212 +
    case BM_CLZW:
213 +
        return (u32)a ? (u64)__builtin_clz((u32)a) : 32;
214 +
    case BM_CTZW:
215 +
        return (u32)a ? (u64)__builtin_ctz((u32)a) : 32;
216 +
    case BM_CPOPW:
217 +
        return (u64)__builtin_popcount((u32)a);
218 +
    case BM_SEXTB:
219 +
        return (u64)(i64)(i8)a;
220 +
    case BM_SEXTH:
221 +
        return (u64)(i64)(i16)a;
222 +
    case BM_ORCB: {
223 +
        __m128i bytes = _mm_cvtsi64_si128((i64)a);
224 +
        __m128i empty = _mm_cmpeq_epi8(bytes, _mm_setzero_si128());
225 +
        return (u64)_mm_cvtsi128_si64(_mm_xor_si128(empty, _mm_set1_epi8(-1)));
226 +
    }
227 +
    case BM_REV8:
228 +
        return __builtin_bswap64(a);
229 +
    case BM_ZEXTH:
230 +
        return (u16)a;
231 +
    case BM_SLLIUW:
232 +
        return (u64)(u32)a << n;
233 +
    case BM_RORI:
234 +
        n = n & 63;
235 +
        return (a >> n) | (a << ((0U - n) & 63));
236 +
    case BM_RORIW:
237 +
        n = n & 31;
238 +
        return (u64)(i64)(i32)(((u32)a >> n) | ((u32)a << ((0U - n) & 31)));
239 +
    case BM_BCLRI:
240 +
        return a & ~(1ULL << n);
241 +
    case BM_BEXTI:
242 +
        return (a >> n) & 1;
243 +
    case BM_BINVI:
244 +
        return a ^ (1ULL << n);
245 +
    case BM_BSETI:
246 +
        return a | (1ULL << n);
247 +
    case BM_NONE:
248 +
        break;
249 +
    }
250 +
    return 0;
251 +
}
252 +
253 +
#endif
emulator.c +12 -0
7 7
#include <string.h>
8 8
#include <sys/ioctl.h>
9 9
#include <termios.h>
10 10
#include <unistd.h>
11 11
12 +
#include "bitmanip.h"
12 13
#include "color.h"
13 14
#include "host.h"
14 15
#include "hosted.h"
15 16
#include "io.h"
16 17
#include "jit.h"
1564 1565
            }
1565 1566
        }
1566 1567
        printf("%s\n", COLOR_RESET);
1567 1568
    }
1568 1569
1570 +
    enum bitmanip_op bm = bitmanip_decode(ins.raw);
1571 +
    if (bm != BM_NONE) {
1572 +
        if (ins.r.rd != ZERO) {
1573 +
            cpu->regs[ins.r.rd] = bitmanip_execute(
1574 +
                bm, ins.raw, cpu->regs[ins.r.rs1], cpu->regs[ins.r.rs2]
1575 +
            );
1576 +
            cpu->modified = (reg_t)ins.r.rd;
1577 +
        }
1578 +
        goto retire;
1579 +
    }
1569 1580
    switch (opcode) {
1570 1581
    case OP_LUI:
1571 1582
        if (ins.u.rd != 0) {
1572 1583
            u32 lui_val = ins.u.imm_31_12 << 12;
1573 1584
            cpu->regs[ins.u.rd] =
2184 2195
        printf("Unknown opcode %02x at PC=%08x\n", opcode, cpu->pc);
2185 2196
        cpu->running = false;
2186 2197
        emit_fault_diagnostics(cpu, executed_pc);
2187 2198
        break;
2188 2199
    }
2200 +
retire:
2189 2201
    /* Register x0 is hardwired to zero. */
2190 2202
    cpu->regs[ZERO] = 0;
2191 2203
    cpu->pc         = pc_next;
2192 2204
2193 2205
    if (cpu->running) {
jit.c +348 -6
28 28
#include <stddef.h>
29 29
#include <stdint.h>
30 30
#include <string.h>
31 31
#include <sys/mman.h>
32 32
33 +
#include "bitmanip.h"
33 34
#include "jit.h"
34 35
#include "riscv.h"
35 36
#include "types.h"
36 37
37 38
/* ---------- x86-64 code emitter helpers --------------------------------- */
134 135
    emit_u8(e, modrm(3, ext, X_RAX));
135 136
}
136 137
137 138
/* ---------- Load/store guest register from register file --------------- */
138 139
139 -
/* Emit mov with [r12 + disp] addressing (r12 needs SIB). */
140 -
static void emit_r12_disp(
141 -
    struct emitter *e, u8 opcode, enum x86reg reg, u32 off
142 -
) {
143 -
    emit_u8(e, rex(true, reg >= 8, false, true)); /* B=1 for r12 */
144 -
    emit_u8(e, opcode);
140 +
/* Emit the ModR/M, SIB, and displacement for [r12 + off]. */
141 +
static void emit_r12_address(struct emitter *e, enum x86reg reg, u32 off) {
145 142
    if (off < 128) {
146 143
        emit_u8(e, modrm(1, reg, X_R12 & 7));
147 144
        emit_u8(e, sib(0, 4, X_R12 & 7));
148 145
        emit_u8(e, (u8)off);
149 146
    } else {
151 148
        emit_u8(e, sib(0, 4, X_R12 & 7));
152 149
        emit_u32(e, off);
153 150
    }
154 151
}
155 152
153 +
/* Emit mov with [r12 + disp] addressing (r12 needs SIB). */
154 +
static void emit_r12_disp(
155 +
    struct emitter *e, u8 opcode, enum x86reg reg, u32 off
156 +
) {
157 +
    emit_u8(e, rex(true, reg >= 8, false, true)); /* B=1 for r12 */
158 +
    emit_u8(e, opcode);
159 +
    emit_r12_address(e, reg, off);
160 +
}
161 +
156 162
/* Load guest register `guest` into host `dst` from [r12 + guest*8]. */
157 163
static void emit_load_guest(struct emitter *e, enum x86reg dst, u32 guest) {
158 164
    if (guest == 0) {
159 165
        /* x0 is always zero -- xor dst, dst. */
160 166
        emit_u8(e, rex(true, dst >= 8, false, dst >= 8));
163 169
        return;
164 170
    }
165 171
    emit_r12_disp(e, 0x8B, dst, guest * 8);
166 172
}
167 173
174 +
/* Load and zero-extend the low word of a guest register. */
175 +
static void emit_load_guest32(struct emitter *e, enum x86reg dst, u32 guest) {
176 +
    if (guest == 0) {
177 +
        emit_load_guest(e, dst, guest);
178 +
        return;
179 +
    }
180 +
    emit_u8(e, rex(false, dst >= 8, false, true));
181 +
    emit_u8(e, 0x8b);
182 +
    emit_r12_address(e, dst, guest * 8);
183 +
}
184 +
168 185
/* Store host `src` to guest register [r12 + guest*8]. */
169 186
static void emit_store_guest(struct emitter *e, u32 guest, enum x86reg src) {
170 187
    if (guest == 0)
171 188
        return;
172 189
    emit_r12_disp(e, 0x89, src, guest * 8);
739 756
    emit_rexw(e);
740 757
    emit_u8(e, 0x89);
741 758
    emit_u8(e, modrm(0, X_RCX, X_RAX));
742 759
}
743 760
761 +
/* Emit a required bit manipulation instruction using baseline x86-64. */
762 +
static void emit_bitmanip(struct emitter *e, instr_t ins, enum bitmanip_op op) {
763 +
    if (ins.r.rd == ZERO)
764 +
        return;
765 +
    if (op == BM_SEXTB || op == BM_SEXTH || op == BM_ZEXTH) {
766 +
        if (ins.r.rs1 == ZERO) {
767 +
            emit_load_rs1(e, ZERO);
768 +
        } else {
769 +
            emit_u8(e, rex(op != BM_ZEXTH, false, false, true));
770 +
            emit_u8(e, 0x0f);
771 +
            emit_u8(e, op == BM_SEXTB ? 0xbe : op == BM_SEXTH ? 0xbf : 0xb7);
772 +
            emit_r12_address(e, HTMP1, ins.r.rs1 * 8);
773 +
        }
774 +
        emit_store_rd(e, ins.r.rd);
775 +
        return;
776 +
    }
777 +
    if (op == BM_ORCB) {
778 +
        bool avx = __builtin_cpu_supports("avx");
779 +
        /* Compare each byte with zero, then invert the byte masks. */
780 +
        static const u8 masks[2][16] = {
781 +
            {
782 +
                0x66,
783 +
                0x0f,
784 +
                0xef,
785 +
                0xc9, /* pxor xmm1, xmm1 */
786 +
                0x66,
787 +
                0x0f,
788 +
                0x74,
789 +
                0xc1, /* pcmpeqb xmm0, xmm1 */
790 +
                0x66,
791 +
                0x0f,
792 +
                0x74,
793 +
                0xc9, /* pcmpeqb xmm1, xmm1 */
794 +
                0x66,
795 +
                0x0f,
796 +
                0xef,
797 +
                0xc1, /* pxor xmm0, xmm1 */
798 +
            },
799 +
            {
800 +
                0xc5,
801 +
                0xf1,
802 +
                0xef,
803 +
                0xc9, /* vpxor xmm1, xmm1, xmm1 */
804 +
                0xc5,
805 +
                0xf9,
806 +
                0x74,
807 +
                0xc1, /* vpcmpeqb xmm0, xmm0, xmm1 */
808 +
                0xc5,
809 +
                0xf1,
810 +
                0x74,
811 +
                0xc9, /* vpcmpeqb xmm1, xmm1, xmm1 */
812 +
                0xc5,
813 +
                0xf9,
814 +
                0xef,
815 +
                0xc1, /* vpxor xmm0, xmm0, xmm1 */
816 +
            },
817 +
        };
818 +
        if (ins.r.rs1 == ZERO) {
819 +
            emit_load_rs1(e, ZERO);
820 +
            emit_store_rd(e, ins.r.rd);
821 +
            return;
822 +
        }
823 +
        emit_u8(e, avx ? 0xc4 : 0xf3);
824 +
        emit_u8(e, avx ? 0xc1 : 0x41);
825 +
        emit_u8(e, avx ? 0x7a : 0x0f);
826 +
        emit_u8(e, 0x7e);
827 +
        emit_r12_address(e, X_RAX, ins.r.rs1 * 8); /* vmovq xmm0, [r12 + off] */
828 +
        for (u32 i = 0; i < sizeof(masks[0]); i++)
829 +
            emit_u8(e, masks[avx][i]);
830 +
        emit_u8(e, avx ? 0xc4 : 0x66);
831 +
        emit_u8(e, avx ? 0xc1 : 0x41);
832 +
        emit_u8(e, avx ? 0x79 : 0x0f);
833 +
        emit_u8(e, 0xd6);
834 +
        emit_r12_address(e, X_RAX, ins.r.rd * 8); /* vmovq [r12 + off], xmm0 */
835 +
        return;
836 +
    }
837 +
    bool word        = op == BM_ROLW || op == BM_RORW || op == BM_RORIW;
838 +
    bool word_source = op == BM_ADDUW || op == BM_SH1ADDUW ||
839 +
                       op == BM_SH2ADDUW || op == BM_SH3ADDUW ||
840 +
                       op == BM_SLLIUW;
841 +
    bool scaled_add  = op == BM_SH1ADD || op == BM_SH2ADD || op == BM_SH3ADD ||
842 +
                       op == BM_SH1ADDUW || op == BM_SH2ADDUW ||
843 +
                       op == BM_SH3ADDUW;
844 +
    if ((op == BM_ADDUW || scaled_add) && ins.r.rs1 == ZERO) {
845 +
        emit_load_rs1(e, ins.r.rs2);
846 +
        emit_store_rd(e, ins.r.rd);
847 +
        return;
848 +
    }
849 +
    if (word_source)
850 +
        emit_load_guest32(e, HTMP1, ins.r.rs1);
851 +
    else
852 +
        emit_load_rs1(e, ins.r.rs1);
853 +
    if ((ins.r.opcode == OP_OP || ins.r.opcode == OP_OP_32) &&
854 +
        !((op == BM_ADDUW || scaled_add) && ins.r.rs2 == ZERO))
855 +
        emit_load_rs2(e, ins.r.rs2);
856 +
    switch (op) {
857 +
    case BM_ADDUW:
858 +
    case BM_SH1ADD:
859 +
    case BM_SH2ADD:
860 +
    case BM_SH3ADD:
861 +
    case BM_SH1ADDUW:
862 +
    case BM_SH2ADDUW:
863 +
    case BM_SH3ADDUW: {
864 +
        u8 shift = op == BM_SH1ADD || op == BM_SH1ADDUW   ? 1
865 +
                   : op == BM_SH2ADD || op == BM_SH2ADDUW ? 2
866 +
                                                          : 3;
867 +
        if (op == BM_ADDUW) {
868 +
            if (ins.r.rs2 != ZERO)
869 +
                emit_alu_rax_rcx(e, 0x01);
870 +
        } else if (ins.r.rs2 == ZERO) {
871 +
            emit_rexw(e);
872 +
            emit_u8(e, 0xc1);
873 +
            emit_u8(e, 0xe0);
874 +
            emit_u8(e, shift);
875 +
        } else {
876 +
            emit_rexw(e);
877 +
            emit_u8(e, 0x8d);
878 +
            emit_u8(e, 0x04);
879 +
            emit_u8(
880 +
                e, sib(shift, X_RAX, X_RCX)
881 +
            ); /* lea rax, [rcx + rax*scale] */
882 +
        }
883 +
        break;
884 +
    }
885 +
    case BM_SLLIUW:
886 +
        emit_rexw(e);
887 +
        emit_u8(e, 0xc1);
888 +
        emit_u8(e, 0xe0);
889 +
        emit_u8(e, (ins.raw >> 20) & 63);
890 +
        break;
891 +
    case BM_ANDN:
892 +
    case BM_ORN:
893 +
        if (op == BM_ANDN && __builtin_cpu_supports("bmi")) {
894 +
            /* andn rax, rcx, rax */
895 +
            emit_u8(e, 0xc4);
896 +
            emit_u8(e, 0xe2);
897 +
            emit_u8(e, 0xf0);
898 +
            emit_u8(e, 0xf2);
899 +
            emit_u8(e, 0xc0);
900 +
        } else {
901 +
            emit_rexw(e);
902 +
            emit_u8(e, 0xf7);
903 +
            emit_u8(e, 0xd1); /* not rcx */
904 +
            emit_alu_rax_rcx(e, op == BM_ANDN ? 0x21 : 0x09);
905 +
        }
906 +
        break;
907 +
    case BM_XNOR:
908 +
        emit_alu_rax_rcx(e, 0x31);
909 +
        emit_rexw(e);
910 +
        emit_u8(e, 0xf7);
911 +
        emit_u8(e, 0xd0);
912 +
        break;
913 +
    case BM_MIN:
914 +
    case BM_MINU:
915 +
    case BM_MAX:
916 +
    case BM_MAXU:
917 +
        emit_cmp_rax_rcx(e);
918 +
        emit_rexw(e);
919 +
        emit_u8(e, 0x0f);
920 +
        emit_u8(
921 +
            e,
922 +
            op == BM_MIN    ? 0x4f
923 +
            : op == BM_MINU ? 0x47
924 +
            : op == BM_MAX  ? 0x4c
925 +
                            : 0x42
926 +
        );
927 +
        emit_u8(e, 0xc1); /* cmovcc rax, rcx */
928 +
        break;
929 +
    case BM_SEXTB:
930 +
    case BM_SEXTH:
931 +
    case BM_ZEXTH:
932 +
        break;
933 +
    case BM_REV8:
934 +
        emit_rexw(e);
935 +
        emit_u8(e, 0x0f);
936 +
        emit_u8(e, 0xc8);
937 +
        break;
938 +
    case BM_ROL:
939 +
    case BM_ROR:
940 +
    case BM_ROLW:
941 +
    case BM_RORW:
942 +
    case BM_RORI:
943 +
    case BM_RORIW:
944 +
        if (!word)
945 +
            emit_rexw(e);
946 +
        if (op == BM_RORI || op == BM_RORIW) {
947 +
            emit_u8(e, 0xc1);
948 +
            emit_u8(e, 0xc8);
949 +
            emit_u8(e, (ins.raw >> 20) & (word ? 31 : 63));
950 +
        } else {
951 +
            emit_u8(e, 0xd3);
952 +
            emit_u8(e, op == BM_ROL || op == BM_ROLW ? 0xc0 : 0xc8);
953 +
        }
954 +
        if (word)
955 +
            emit_movsxd_rax_eax(e);
956 +
        break;
957 +
    case BM_BEXT:
958 +
    case BM_BEXTI:
959 +
        if (op == BM_BEXT) {
960 +
            emit_shift_rax_cl(e, 5);
961 +
        } else if ((ins.raw >> 20) & 63) {
962 +
            emit_rexw(e);
963 +
            emit_u8(e, 0xc1);
964 +
            emit_u8(e, 0xe8);
965 +
            emit_u8(e, (ins.raw >> 20) & 63);
966 +
        }
967 +
        emit_u8(e, 0x83);
968 +
        emit_u8(e, 0xe0);
969 +
        emit_u8(e, 1); /* and eax, 1 */
970 +
        break;
971 +
    case BM_BCLR:
972 +
    case BM_BINV:
973 +
    case BM_BSET:
974 +
        emit_rexw(e);
975 +
        emit_u8(e, 0x0f);
976 +
        emit_u8(e, op == BM_BCLR ? 0xb3 : op == BM_BINV ? 0xbb : 0xab);
977 +
        emit_u8(e, 0xc8);
978 +
        break;
979 +
    case BM_BCLRI:
980 +
    case BM_BINVI:
981 +
    case BM_BSETI:
982 +
        emit_rexw(e);
983 +
        emit_u8(e, 0x0f);
984 +
        emit_u8(e, 0xba);
985 +
        emit_u8(
986 +
            e,
987 +
            modrm(
988 +
                3,
989 +
                op == BM_BCLRI   ? 6
990 +
                : op == BM_BINVI ? 7
991 +
                                 : 5,
992 +
                X_RAX
993 +
            )
994 +
        );
995 +
        emit_u8(e, (ins.raw >> 20) & 63);
996 +
        break;
997 +
    case BM_CLZ:
998 +
    case BM_CTZ:
999 +
    case BM_CLZW:
1000 +
    case BM_CTZW: {
1001 +
        bool wide    = op == BM_CLZ || op == BM_CTZ;
1002 +
        bool leading = op == BM_CLZ || op == BM_CLZW;
1003 +
        if ((leading ? __builtin_cpu_supports("lzcnt")
1004 +
                     : __builtin_cpu_supports("bmi"))) {
1005 +
            emit_u8(e, 0xf3);
1006 +
            if (wide)
1007 +
                emit_rexw(e);
1008 +
            emit_u8(e, 0x0f);
1009 +
            emit_u8(e, leading ? 0xbd : 0xbc);
1010 +
            emit_u8(e, 0xc0);
1011 +
            break;
1012 +
        }
1013 +
        if (wide)
1014 +
            emit_rexw(e);
1015 +
        emit_u8(e, 0x0f);
1016 +
        emit_u8(e, leading ? 0xbd : 0xbc);
1017 +
        emit_u8(e, 0xc0);
1018 +
        /* MOV preserves the zero flag from BSF/BSR. */
1019 +
        emit_mov_imm32_sx(
1020 +
            e, HTMP2, leading ? (wide ? 127 : 63) : (wide ? 64 : 32)
1021 +
        );
1022 +
        emit_rexw(e);
1023 +
        emit_u8(e, 0x0f);
1024 +
        emit_u8(e, 0x44);
1025 +
        emit_u8(e, 0xc1);
1026 +
        if (leading) {
1027 +
            emit_mov_imm32_sx(e, HTMP2, wide ? 63 : 31);
1028 +
            emit_alu_rax_rcx(e, 0x31);
1029 +
        }
1030 +
        break;
1031 +
    }
1032 +
    case BM_CPOP:
1033 +
    case BM_CPOPW: {
1034 +
        if (__builtin_cpu_supports("popcnt")) {
1035 +
            emit_u8(e, 0xf3);
1036 +
            if (op == BM_CPOP)
1037 +
                emit_rexw(e);
1038 +
            emit_u8(e, 0x0f);
1039 +
            emit_u8(e, 0xb8);
1040 +
            emit_u8(e, 0xc0);
1041 +
        } else {
1042 +
            if (op == BM_CPOPW) {
1043 +
                emit_u8(e, 0x89);
1044 +
                emit_u8(e, 0xc0);
1045 +
            }
1046 +
            /* Clear one set bit per iteration and count in RCX. */
1047 +
            emit_u8(e, 0x31);
1048 +
            emit_u8(e, 0xc9);
1049 +
            emit_rexw(e);
1050 +
            emit_u8(e, 0x85);
1051 +
            emit_u8(e, 0xc0);
1052 +
            emit_u8(e, 0x74);
1053 +
            emit_u8(e, 12);
1054 +
            emit_rexw(e);
1055 +
            emit_u8(e, 0x8d);
1056 +
            emit_u8(e, 0x50);
1057 +
            emit_u8(e, 0xff);
1058 +
            emit_rexw(e);
1059 +
            emit_u8(e, 0xff);
1060 +
            emit_u8(e, 0xc1);
1061 +
            emit_rexw(e);
1062 +
            emit_u8(e, 0x21);
1063 +
            emit_u8(e, 0xd0);
1064 +
            emit_u8(e, 0x75);
1065 +
            emit_u8(e, (u8)-12);
1066 +
            emit_rexw(e);
1067 +
            emit_u8(e, 0x89);
1068 +
            emit_u8(e, 0xc8);
1069 +
        }
1070 +
        break;
1071 +
    }
1072 +
    case BM_ORCB:
1073 +
        break;
1074 +
    case BM_NONE:
1075 +
        break;
1076 +
    }
1077 +
    emit_store_rd(e, ins.r.rd);
1078 +
}
1079 +
744 1080
/* ---------- Translate one RV64I instruction ----------------------------- */
745 1081
746 1082
/*
747 1083
 * Returns: true if the instruction ends the basic block (branch/jump/ecall),
748 1084
 *          false if execution should continue to the next instruction.
749 1085
 */
750 1086
static bool translate_insn(struct emitter *e, instr_t ins, u32 pc) {
751 1087
    u32 opcode  = ins.r.opcode;
752 1088
    u32 pc_next = pc + INSTR_SIZE;
753 1089
1090 +
    enum bitmanip_op bm = bitmanip_decode(ins.raw);
1091 +
    if (bm != BM_NONE) {
1092 +
        emit_bitmanip(e, ins, bm);
1093 +
        return false;
1094 +
    }
1095 +
754 1096
    switch (opcode) {
755 1097
756 1098
    case OP_LUI:
757 1099
        if (ins.u.rd != 0) {
758 1100
            emit_mov_imm32_sx(e, HTMP1, (i32)(ins.u.imm_31_12 << 12));
machine_cpu.c +7 -1
1 1
/* Instruction execution for one machine hart. */
2 2
#include <stdint.h>
3 3
#include <stdio.h>
4 4
5 +
#include "bitmanip.h"
5 6
#include "machine_internal.h"
6 7
#include "muldiv.h"
7 8
#include "riscv.h"
8 9
9 10
/* RV64 wraparound arithmetic stays unsigned; extension never shifts by 64. */
220 221
    u32  opcode = ins & 0x7f, rd = (ins >> 7) & 31;
221 222
    u32  f3 = (ins >> 12) & 7, rs1 = (ins >> 15) & 31, rs2 = (ins >> 20) & 31;
222 223
    u32  f7 = ins >> 25;
223 224
    u64  a = h->regs[rs1], b = h->regs[rs2];
224 225
    u64  immediate = sext(ins >> 20, 12), result = 0, next = h->pc + 4;
225 -
    bool write_rd = true;
226 +
    bool write_rd       = true;
227 +
    enum bitmanip_op bm = bitmanip_decode(ins);
228 +
    if (bm != BM_NONE) {
229 +
        result = bitmanip_execute(bm, ins, a, b);
230 +
        goto retire;
231 +
    }
226 232
    switch (opcode) {
227 233
    case OP_LUI:
228 234
        result = sext(ins & 0xfffff000U, 32);
229 235
        break;
230 236
    case OP_AUIPC:
machine_fdt.c +1 -1
167 167
        begin(&f, name);
168 168
        string(&f, "device_type", "cpu");
169 169
        string(&f, "compatible", "riscv");
170 170
        scalar(&f, "reg", id);
171 171
        string(&f, "status", "okay");
172 -
        string(&f, "riscv,isa", "rv64ima_zicsr_zifencei");
172 +
        string(&f, "riscv,isa", "rv64ima_zba_zbb_zbs_zicsr_zifencei");
173 173
        begin(&f, "interrupt-controller");
174 174
        scalar(&f, "#interrupt-cells", 1);
175 175
        property(&f, "interrupt-controller", "", 0);
176 176
        string(&f, "compatible", "riscv,cpu-intc");
177 177
        scalar(&f, "phandle", id + 1);
riscv/debug.c +192 -0
1 1
#include <stdarg.h>
2 2
#include <stdio.h>
3 3
#include <string.h>
4 4
#include <unistd.h>
5 5
6 +
#include "../bitmanip.h"
6 7
#include "../color.h"
7 8
#include "../riscv.h"
8 9
9 10
#include "debug.h"
10 11
158 159
struct instrparts instr_decode(instr_t instr) {
159 160
    struct instrparts comp;
160 161
    i32               imm;
161 162
    int               funct7_idx;
162 163
164 +
    enum bitmanip_op bm = bitmanip_decode(instr.raw);
165 +
    if (bm != BM_NONE) {
166 +
        const char *name = "";
167 +
        char        form = 'R';
168 +
        switch (bm) {
169 +
        case BM_ADDUW:
170 +
            name = "add.uw";
171 +
            form = 'R';
172 +
            break;
173 +
        case BM_SH1ADD:
174 +
            name = "sh1add";
175 +
            form = 'R';
176 +
            break;
177 +
        case BM_SH2ADD:
178 +
            name = "sh2add";
179 +
            form = 'R';
180 +
            break;
181 +
        case BM_SH3ADD:
182 +
            name = "sh3add";
183 +
            form = 'R';
184 +
            break;
185 +
        case BM_SH1ADDUW:
186 +
            name = "sh1add.uw";
187 +
            form = 'R';
188 +
            break;
189 +
        case BM_SH2ADDUW:
190 +
            name = "sh2add.uw";
191 +
            form = 'R';
192 +
            break;
193 +
        case BM_SH3ADDUW:
194 +
            name = "sh3add.uw";
195 +
            form = 'R';
196 +
            break;
197 +
        case BM_ANDN:
198 +
            name = "andn";
199 +
            form = 'R';
200 +
            break;
201 +
        case BM_ORN:
202 +
            name = "orn";
203 +
            form = 'R';
204 +
            break;
205 +
        case BM_XNOR:
206 +
            name = "xnor";
207 +
            form = 'R';
208 +
            break;
209 +
        case BM_MIN:
210 +
            name = "min";
211 +
            form = 'R';
212 +
            break;
213 +
        case BM_MINU:
214 +
            name = "minu";
215 +
            form = 'R';
216 +
            break;
217 +
        case BM_MAX:
218 +
            name = "max";
219 +
            form = 'R';
220 +
            break;
221 +
        case BM_MAXU:
222 +
            name = "maxu";
223 +
            form = 'R';
224 +
            break;
225 +
        case BM_ROL:
226 +
            name = "rol";
227 +
            form = 'R';
228 +
            break;
229 +
        case BM_ROR:
230 +
            name = "ror";
231 +
            form = 'R';
232 +
            break;
233 +
        case BM_ROLW:
234 +
            name = "rolw";
235 +
            form = 'R';
236 +
            break;
237 +
        case BM_RORW:
238 +
            name = "rorw";
239 +
            form = 'R';
240 +
            break;
241 +
        case BM_BCLR:
242 +
            name = "bclr";
243 +
            form = 'R';
244 +
            break;
245 +
        case BM_BEXT:
246 +
            name = "bext";
247 +
            form = 'R';
248 +
            break;
249 +
        case BM_BINV:
250 +
            name = "binv";
251 +
            form = 'R';
252 +
            break;
253 +
        case BM_BSET:
254 +
            name = "bset";
255 +
            form = 'R';
256 +
            break;
257 +
        case BM_CLZ:
258 +
            name = "clz";
259 +
            form = '2';
260 +
            break;
261 +
        case BM_CTZ:
262 +
            name = "ctz";
263 +
            form = '2';
264 +
            break;
265 +
        case BM_CPOP:
266 +
            name = "cpop";
267 +
            form = '2';
268 +
            break;
269 +
        case BM_CLZW:
270 +
            name = "clzw";
271 +
            form = '2';
272 +
            break;
273 +
        case BM_CTZW:
274 +
            name = "ctzw";
275 +
            form = '2';
276 +
            break;
277 +
        case BM_CPOPW:
278 +
            name = "cpopw";
279 +
            form = '2';
280 +
            break;
281 +
        case BM_SEXTB:
282 +
            name = "sext.b";
283 +
            form = '2';
284 +
            break;
285 +
        case BM_SEXTH:
286 +
            name = "sext.h";
287 +
            form = '2';
288 +
            break;
289 +
        case BM_ORCB:
290 +
            name = "orc.b";
291 +
            form = '2';
292 +
            break;
293 +
        case BM_REV8:
294 +
            name = "rev8";
295 +
            form = '2';
296 +
            break;
297 +
        case BM_ZEXTH:
298 +
            name = "zext.h";
299 +
            form = '2';
300 +
            break;
301 +
        case BM_SLLIUW:
302 +
            name = "slli.uw";
303 +
            form = 'I';
304 +
            break;
305 +
        case BM_RORI:
306 +
            name = "rori";
307 +
            form = 'I';
308 +
            break;
309 +
        case BM_RORIW:
310 +
            name = "roriw";
311 +
            form = 'I';
312 +
            break;
313 +
        case BM_BCLRI:
314 +
            name = "bclri";
315 +
            form = 'I';
316 +
            break;
317 +
        case BM_BEXTI:
318 +
            name = "bexti";
319 +
            form = 'I';
320 +
            break;
321 +
        case BM_BINVI:
322 +
            name = "binvi";
323 +
            form = 'I';
324 +
            break;
325 +
        case BM_BSETI:
326 +
            name = "bseti";
327 +
            form = 'I';
328 +
            break;
329 +
        case BM_NONE:
330 +
            break;
331 +
        }
332 +
        comp = parts(form);
333 +
        snprintf(comp.op, MAX_PART_LEN, "%s", name);
334 +
        snprintf(comp.rd, MAX_PART_LEN, "%s", reg_names[instr.r.rd]);
335 +
        snprintf(comp.rs1, MAX_PART_LEN, "%s", reg_names[instr.r.rs1]);
336 +
        if (form == 'R')
337 +
            snprintf(comp.rs2, MAX_PART_LEN, "%s", reg_names[instr.r.rs2]);
338 +
        if (form == 'I')
339 +
            snprintf(
340 +
                comp.imm,
341 +
                MAX_PART_LEN,
342 +
                "%u",
343 +
                (instr.raw >> 20) & (bm == BM_RORIW ? 31 : 63)
344 +
            );
345 +
        return comp;
346 +
    }
347 +
163 348
    /* Handle `nop` (`addi x0, x0, 0`). */
164 349
    if (instr.raw == 0x00000013) {
165 350
        comp = parts('?');
166 351
        strncpy(comp.op, "nop", MAX_PART_LEN);
167 352
        comp.is_pseudo = true;
503 688
            return;
504 689
        }
505 690
    }
506 691
507 692
    switch (i->type) {
693 +
    case '2':
694 +
        put(ctx, nocolor, " ");
695 +
        put(ctx, rdcolor, i->rd);
696 +
        put(ctx, nocolor, ", ");
697 +
        put(ctx, rscolor, i->rs1);
698 +
        break;
699 +
508 700
    case 'R':
509 701
        /* R-type: op rd, rs1, rs2. */
510 702
        put(ctx, nocolor, " ");
511 703
        put(ctx, rdcolor, i->rd);
512 704
        put(ctx, nocolor, ", ");