.githooks/
riscv/
test/
.clang-format
570 B
.gitignore
5 B
.gitsigners
112 B
AGENTS.md
227 B
LICENSE
1.1 KiB
Makefile
3.5 KiB
README
12.5 KiB
STYLE
781 B
abi_host.c
12.2 KiB
abi_linux.c
3.7 KiB
bitmanip.h
5.8 KiB
color.h
567 B
debug_map.c
5.1 KiB
debug_map.h
1.3 KiB
emulator.c
87.3 KiB
host.h
5.5 KiB
host_posix.c
11.2 KiB
hosted.h
5.7 KiB
image_native.c
4.6 KiB
image_rad0.c
2.7 KiB
image_raw.c
2.6 KiB
io.c
1.1 KiB
io.h
444 B
jit.c
99.2 KiB
jit.h
7.8 KiB
machine.c
4.4 KiB
machine.h
2.1 KiB
machine_cpu.c
13.6 KiB
machine_csr.c
4.6 KiB
machine_devices.c
10.8 KiB
machine_fdt.c
8.7 KiB
machine_internal.h
7.4 KiB
muldiv.h
1.0 KiB
profile.c
17.5 KiB
profile.h
1.4 KiB
riscv.c
755 B
riscv.h
4.3 KiB
types.h
1019 B
vector.c
54.0 KiB
vector.h
1.3 KiB
machine_cpu.c
raw
| 1 | /* Instruction execution for one machine hart. */ |
| 2 | #include <stdint.h> |
| 3 | #include <stdio.h> |
| 4 | |
| 5 | #include "bitmanip.h" |
| 6 | #include "machine_internal.h" |
| 7 | #include "muldiv.h" |
| 8 | #include "riscv.h" |
| 9 | |
| 10 | /* RV64 wraparound arithmetic stays unsigned; extension never shifts by 64. */ |
| 11 | static u64 sext(u64 value, u32 bits) { |
| 12 | if (bits == 64) |
| 13 | return value; |
| 14 | u64 sign = 1ULL << (bits - 1); |
| 15 | value &= (1ULL << bits) - 1; |
| 16 | return (value ^ sign) - sign; |
| 17 | } |
| 18 | |
| 19 | /* Perform an indivisible atomic operation in the sequential coherent bus model. |
| 20 | * aq/rl are satisfied by total instruction order; AMOs never target devices. */ |
| 21 | static bool atomic( |
| 22 | struct machine *m, |
| 23 | struct machine_hart *h, |
| 24 | u32 ins, |
| 25 | u64 address, |
| 26 | u64 source, |
| 27 | u64 *result, |
| 28 | u64 *cause |
| 29 | ) { |
| 30 | u32 width = ((ins >> 12) & 7) == 2 ? 4 : 8; |
| 31 | u32 operation = ins >> 27; |
| 32 | u64 old = 0; |
| 33 | if (operation != 2 && !machine_store_probe(m, h, address, width, cause)) { |
| 34 | h->reserved = false; |
| 35 | return false; |
| 36 | } |
| 37 | if (operation == 3) { |
| 38 | bool success = h->reserved && h->reservation == address && |
| 39 | h->reservation_size == width; |
| 40 | h->reserved = false; |
| 41 | if (success && !machine_bus( |
| 42 | m, |
| 43 | h, |
| 44 | address, |
| 45 | width, |
| 46 | ACCESS_STORE | ACCESS_ATOMIC, |
| 47 | &source, |
| 48 | cause |
| 49 | )) |
| 50 | return false; |
| 51 | *result = success ? 0 : 1; |
| 52 | return true; |
| 53 | } |
| 54 | /* Only LR can fail here: an AMO has passed the store probe. */ |
| 55 | if (!machine_bus(m, h, address, width, ACCESS_ATOMIC, &old, cause)) |
| 56 | return false; |
| 57 | *result = sext(old, width * 8); |
| 58 | if (operation == 2) { |
| 59 | h->reserved = true; |
| 60 | h->reservation = address; |
| 61 | h->reservation_size = width; |
| 62 | return true; |
| 63 | } |
| 64 | u64 value = 0; |
| 65 | if (width == 4) |
| 66 | source = (u32)source; |
| 67 | switch (operation) { |
| 68 | case 0: |
| 69 | value = old + source; |
| 70 | break; |
| 71 | case 1: |
| 72 | value = source; |
| 73 | break; |
| 74 | case 4: |
| 75 | value = old ^ source; |
| 76 | break; |
| 77 | case 8: |
| 78 | value = old | source; |
| 79 | break; |
| 80 | case 12: |
| 81 | value = old & source; |
| 82 | break; |
| 83 | case 16: |
| 84 | value = (i64)sext(old, width * 8) < (i64)sext(source, width * 8) |
| 85 | ? old |
| 86 | : source; |
| 87 | break; |
| 88 | case 20: |
| 89 | value = (i64)sext(old, width * 8) > (i64)sext(source, width * 8) |
| 90 | ? old |
| 91 | : source; |
| 92 | break; |
| 93 | case 24: |
| 94 | value = old < source ? old : source; |
| 95 | break; |
| 96 | case 28: |
| 97 | value = old > source ? old : source; |
| 98 | break; |
| 99 | } |
| 100 | return machine_bus( |
| 101 | m, h, address, width, ACCESS_STORE | ACCESS_ATOMIC, &value, cause |
| 102 | ); |
| 103 | } |
| 104 | |
| 105 | /* Compute an OP or OP-32 instruction into `result`. Returns false if the |
| 106 | * instruction is illegal. */ |
| 107 | static bool register_op(u32 ins, u64 a, u64 b, u64 *result) { |
| 108 | bool word = (ins & 0x7f) == OP_OP_32; |
| 109 | u32 f3 = (ins >> 12) & 7, f7 = ins >> 25; |
| 110 | u64 value = 0; |
| 111 | if (f7 == 1) { |
| 112 | if (word) { |
| 113 | if (f3 == 0) |
| 114 | value = (u64)(u32)a * (u32)b; |
| 115 | else if (f3 == 4 || f3 == 6) |
| 116 | value = divs64(sext(a, 32), sext(b, 32), f3 == 6); |
| 117 | else if (f3 == 5) |
| 118 | value = (u32)b ? (u32)a / (u32)b : UINT32_MAX; |
| 119 | else if (f3 == 7) |
| 120 | value = (u32)b ? (u32)a % (u32)b : (u32)a; |
| 121 | else |
| 122 | return false; |
| 123 | } else { |
| 124 | switch (f3) { |
| 125 | case 0: |
| 126 | value = a * b; |
| 127 | break; |
| 128 | case 1: |
| 129 | value = mulh64(a, b); |
| 130 | break; |
| 131 | case 2: |
| 132 | value = mulhsu64(a, b); |
| 133 | break; |
| 134 | case 3: |
| 135 | value = mulhu64(a, b); |
| 136 | break; |
| 137 | case 4: |
| 138 | value = divs64(a, b, false); |
| 139 | break; |
| 140 | case 5: |
| 141 | value = b ? a / b : UINT64_MAX; |
| 142 | break; |
| 143 | case 6: |
| 144 | value = divs64(a, b, true); |
| 145 | break; |
| 146 | case 7: |
| 147 | value = b ? a % b : a; |
| 148 | break; |
| 149 | } |
| 150 | } |
| 151 | } else { |
| 152 | if (f7 != 0 && f7 != 0x20) |
| 153 | return false; |
| 154 | if (f7 == 0x20 && f3 != 0 && f3 != 5) |
| 155 | return false; |
| 156 | if (word && f3 != 0 && f3 != 1 && f3 != 5) |
| 157 | return false; |
| 158 | u32 shift = b & (word ? 31 : 63); |
| 159 | switch (f3) { |
| 160 | case 0: |
| 161 | value = f7 ? a - b : a + b; |
| 162 | break; |
| 163 | case 1: |
| 164 | value = a << shift; |
| 165 | break; |
| 166 | case 2: |
| 167 | value = (i64)a < (i64)b; |
| 168 | break; |
| 169 | case 3: |
| 170 | value = a < b; |
| 171 | break; |
| 172 | case 4: |
| 173 | value = a ^ b; |
| 174 | break; |
| 175 | case 5: |
| 176 | if (word) |
| 177 | value = f7 ? (u32)((i32)a >> shift) : (u32)a >> shift; |
| 178 | else |
| 179 | value = f7 ? (u64)((i64)a >> shift) : a >> shift; |
| 180 | break; |
| 181 | case 6: |
| 182 | value = a | b; |
| 183 | break; |
| 184 | case 7: |
| 185 | value = a & b; |
| 186 | break; |
| 187 | } |
| 188 | } |
| 189 | *result = word ? sext(value, 32) : value; |
| 190 | return true; |
| 191 | } |
| 192 | |
| 193 | /* Memory fault details for one machine vector instruction. */ |
| 194 | struct vector_access { |
| 195 | struct machine *m; |
| 196 | struct machine_hart *h; |
| 197 | u64 addr; |
| 198 | u64 cause; |
| 199 | }; |
| 200 | |
| 201 | /* Transfer one vector element through the machine bus. */ |
| 202 | static bool machine_vector_access( |
| 203 | void *context, u64 address, u32 size, bool store, u64 *value |
| 204 | ) { |
| 205 | struct vector_access *access = context; |
| 206 | access->addr = address; |
| 207 | return machine_bus( |
| 208 | access->m, |
| 209 | access->h, |
| 210 | address, |
| 211 | size, |
| 212 | store ? ACCESS_STORE : 0, |
| 213 | value, |
| 214 | &access->cause |
| 215 | ); |
| 216 | } |
| 217 | |
| 218 | void machine_step(struct machine *m, struct machine_hart *h) { |
| 219 | h->wrote_instret = false; |
| 220 | u64 pending = machine_pending(m, h) & h->mie; |
| 221 | if (pending) |
| 222 | h->waiting = false; |
| 223 | if (pending && (h->privilege < 3 || (h->mstatus & MSTATUS_MIE))) { |
| 224 | u64 interrupt = (pending & MIP_MEIP) ? 11 |
| 225 | : (pending & MIP_MSIP) ? 3 |
| 226 | : 7; |
| 227 | machine_trap(m, h, MCAUSE_INTERRUPT | interrupt, 0); |
| 228 | return; |
| 229 | } |
| 230 | if (h->waiting) |
| 231 | return; |
| 232 | u64 raw = 0, cause = 0, address = h->pc; |
| 233 | if (!machine_bus(m, h, h->pc, 4, ACCESS_FETCH, &raw, &cause)) |
| 234 | goto fault; |
| 235 | u32 ins = (u32)raw; |
| 236 | if (m->config->trace) |
| 237 | fprintf( |
| 238 | stderr, |
| 239 | "hart=%u tick=%llu pc=0x%llx insn=%08x mode=%c\n", |
| 240 | h->id, |
| 241 | m->ticks, |
| 242 | h->pc, |
| 243 | ins, |
| 244 | h->privilege ? 'M' : 'U' |
| 245 | ); |
| 246 | u32 opcode = ins & 0x7f, rd = (ins >> 7) & 31; |
| 247 | u32 f3 = (ins >> 12) & 7, rs1 = (ins >> 15) & 31, rs2 = (ins >> 20) & 31; |
| 248 | u32 f7 = ins >> 25; |
| 249 | u64 a = h->regs[rs1], b = h->regs[rs2]; |
| 250 | u64 immediate = sext(ins >> 20, 12), result = 0, next = h->pc + 4; |
| 251 | bool write_rd = true; |
| 252 | enum bitmanip_op bm = bitmanip_decode(ins); |
| 253 | if (bm != BM_NONE) { |
| 254 | result = bitmanip_execute(bm, ins, a, b); |
| 255 | goto retire; |
| 256 | } |
| 257 | if (opcode == 0x57 || opcode == 7 || opcode == 39) { |
| 258 | if (!(h->mstatus & (3ULL << 9))) |
| 259 | goto illegal; |
| 260 | struct vector_access access = { m, h, 0, 0 }; |
| 261 | enum vector_result vr = vector_execute( |
| 262 | &h->vector, h->regs, ins, machine_vector_access, &access |
| 263 | ); |
| 264 | if (vr == VECTOR_ILLEGAL) |
| 265 | goto illegal; |
| 266 | h->mstatus |= 3ULL << 9; |
| 267 | if (vr == VECTOR_FAULT) { |
| 268 | cause = access.cause; |
| 269 | address = access.addr; |
| 270 | goto fault; |
| 271 | } |
| 272 | write_rd = false; |
| 273 | goto retire; |
| 274 | } |
| 275 | switch (opcode) { |
| 276 | case OP_LUI: |
| 277 | result = sext(ins & 0xfffff000U, 32); |
| 278 | break; |
| 279 | case OP_AUIPC: |
| 280 | result = h->pc + sext(ins & 0xfffff000U, 32); |
| 281 | break; |
| 282 | case OP_JAL: |
| 283 | immediate = ((ins >> 31) << 20) | (((ins >> 12) & 255) << 12) | |
| 284 | (((ins >> 20) & 1) << 11) | (((ins >> 21) & 1023) << 1); |
| 285 | next = h->pc + sext(immediate, 21); |
| 286 | result = h->pc + 4; |
| 287 | goto target; |
| 288 | case OP_JALR: |
| 289 | if (f3) |
| 290 | goto illegal; |
| 291 | next = (a + immediate) & ~1ULL; |
| 292 | result = h->pc + 4; |
| 293 | goto target; |
| 294 | case OP_BRANCH: { |
| 295 | bool taken; |
| 296 | write_rd = false; |
| 297 | switch (f3) { |
| 298 | case 0: |
| 299 | taken = a == b; |
| 300 | break; |
| 301 | case 1: |
| 302 | taken = a != b; |
| 303 | break; |
| 304 | case 4: |
| 305 | taken = (i64)a < (i64)b; |
| 306 | break; |
| 307 | case 5: |
| 308 | taken = (i64)a >= (i64)b; |
| 309 | break; |
| 310 | case 6: |
| 311 | taken = a < b; |
| 312 | break; |
| 313 | case 7: |
| 314 | taken = a >= b; |
| 315 | break; |
| 316 | default: |
| 317 | goto illegal; |
| 318 | } |
| 319 | if (taken) { |
| 320 | immediate = ((ins >> 31) << 12) | (((ins >> 7) & 1) << 11) | |
| 321 | (((ins >> 25) & 63) << 5) | (((ins >> 8) & 15) << 1); |
| 322 | next = h->pc + sext(immediate, 13); |
| 323 | } |
| 324 | goto target; |
| 325 | } |
| 326 | case OP_LOAD: { |
| 327 | if (f3 == 7) |
| 328 | goto illegal; |
| 329 | u32 width = 1U << (f3 & 3); |
| 330 | address = a + immediate; |
| 331 | if (!machine_bus(m, h, address, width, 0, &result, &cause)) |
| 332 | goto fault; |
| 333 | if (!(f3 & 4)) |
| 334 | result = sext(result, width * 8); |
| 335 | break; |
| 336 | } |
| 337 | case OP_STORE: { |
| 338 | if (f3 > 3) |
| 339 | goto illegal; |
| 340 | u32 width = 1U << f3; |
| 341 | address = a + sext(((ins >> 25) << 5) | ((ins >> 7) & 31), 12); |
| 342 | if (!machine_bus(m, h, address, width, ACCESS_STORE, &b, &cause)) |
| 343 | goto fault; |
| 344 | write_rd = false; |
| 345 | break; |
| 346 | } |
| 347 | case OP_IMM: |
| 348 | switch (f3) { |
| 349 | case 0: |
| 350 | result = a + immediate; |
| 351 | break; |
| 352 | case 2: |
| 353 | result = (i64)a < (i64)immediate; |
| 354 | break; |
| 355 | case 3: |
| 356 | result = a < immediate; |
| 357 | break; |
| 358 | case 4: |
| 359 | result = a ^ immediate; |
| 360 | break; |
| 361 | case 6: |
| 362 | result = a | immediate; |
| 363 | break; |
| 364 | case 7: |
| 365 | result = a & immediate; |
| 366 | break; |
| 367 | case 1: |
| 368 | if (ins >> 26) |
| 369 | goto illegal; |
| 370 | result = a << ((ins >> 20) & 63); |
| 371 | break; |
| 372 | case 5: |
| 373 | if ((ins >> 26) != 0 && (ins >> 26) != 0x10) |
| 374 | goto illegal; |
| 375 | result = (ins >> 26) ? (u64)((i64)a >> ((ins >> 20) & 63)) |
| 376 | : a >> ((ins >> 20) & 63); |
| 377 | break; |
| 378 | } |
| 379 | break; |
| 380 | case OP_IMM_32: |
| 381 | if (f3 == 0) |
| 382 | result = a + immediate; |
| 383 | else if (f3 == 1 && f7 == 0) |
| 384 | result = (u32)a << rs2; |
| 385 | else if (f3 == 5 && f7 == 0) |
| 386 | result = (u32)a >> rs2; |
| 387 | else if (f3 == 5 && f7 == 0x20) |
| 388 | result = (u32)((i32)a >> rs2); |
| 389 | else |
| 390 | goto illegal; |
| 391 | result = sext(result, 32); |
| 392 | break; |
| 393 | case OP_OP: |
| 394 | case OP_OP_32: |
| 395 | if (!register_op(ins, a, b, &result)) |
| 396 | goto illegal; |
| 397 | break; |
| 398 | case OP_AMO: { |
| 399 | if (f3 != 2 && f3 != 3) |
| 400 | goto illegal; |
| 401 | u32 operation = ins >> 27; |
| 402 | switch (operation) { |
| 403 | case 2: |
| 404 | if (rs2) |
| 405 | goto illegal; |
| 406 | break; |
| 407 | case 0: |
| 408 | case 1: |
| 409 | case 3: |
| 410 | case 4: |
| 411 | case 8: |
| 412 | case 12: |
| 413 | case 16: |
| 414 | case 20: |
| 415 | case 24: |
| 416 | case 28: |
| 417 | break; |
| 418 | default: |
| 419 | goto illegal; |
| 420 | } |
| 421 | address = a; |
| 422 | if (!atomic(m, h, ins, address, b, &result, &cause)) |
| 423 | goto fault; |
| 424 | break; |
| 425 | } |
| 426 | case OP_FENCE: |
| 427 | if (rs1 || rd) |
| 428 | goto illegal; |
| 429 | if (f3 == 0) { |
| 430 | u32 fm = ins >> 28; |
| 431 | if (fm != 0 && !(fm == 8 && ((ins >> 20) & 255) == 0x33)) |
| 432 | goto illegal; |
| 433 | } else if (f3 != 1 || (ins >> 20)) |
| 434 | goto illegal; |
| 435 | /* The bus is strongly ordered; fetch is uncached, including FENCE.I. */ |
| 436 | write_rd = false; |
| 437 | break; |
| 438 | case OP_SYSTEM: |
| 439 | write_rd = false; |
| 440 | if (f3) { |
| 441 | if (!machine_csr(m, h, ins)) |
| 442 | goto illegal; |
| 443 | } else if (ins == 0x00000073) { /* ecall */ |
| 444 | machine_trap( |
| 445 | m, h, h->privilege == 0 ? CAUSE_ECALL_U : CAUSE_ECALL_M, 0 |
| 446 | ); |
| 447 | return; |
| 448 | } else if (ins == 0x00100073) { /* ebreak */ |
| 449 | machine_trap(m, h, CAUSE_BREAKPOINT, h->pc); |
| 450 | return; |
| 451 | } else if (ins == 0x10500073) { /* wfi */ |
| 452 | h->waiting = true; |
| 453 | } else if (ins == 0x30200073) { /* mret */ |
| 454 | if (h->privilege != 3) |
| 455 | goto illegal; |
| 456 | h->privilege = (h->mstatus >> 11) & 3; |
| 457 | h->mstatus = (h->mstatus & ~(MSTATUS_MIE | MSTATUS_MPP)) | |
| 458 | ((h->mstatus & MSTATUS_MPIE) >> 4) | MSTATUS_MPIE; |
| 459 | next = h->mepc; |
| 460 | } else |
| 461 | goto illegal; |
| 462 | break; |
| 463 | default: |
| 464 | goto illegal; |
| 465 | } |
| 466 | goto retire; |
| 467 | |
| 468 | target: |
| 469 | if (next & 3) { |
| 470 | address = next; |
| 471 | cause = CAUSE_FETCH_MISALIGNED; |
| 472 | goto fault; |
| 473 | } |
| 474 | retire: |
| 475 | if (write_rd && rd) |
| 476 | h->regs[rd] = result; |
| 477 | h->pc = next; |
| 478 | if (!h->wrote_instret) |
| 479 | h->minstret++; |
| 480 | m->retired++; |
| 481 | return; |
| 482 | illegal: |
| 483 | machine_trap(m, h, CAUSE_ILLEGAL, ins); |
| 484 | return; |
| 485 | fault: |
| 486 | machine_trap(m, h, cause, address); |
| 487 | } |