/* Instruction execution for one machine hart. */ #include #include #include "bitmanip.h" #include "machine_internal.h" #include "muldiv.h" #include "riscv.h" /* RV64 wraparound arithmetic stays unsigned; extension never shifts by 64. */ static u64 sext(u64 value, u32 bits) { if (bits == 64) return value; u64 sign = 1ULL << (bits - 1); value &= (1ULL << bits) - 1; return (value ^ sign) - sign; } /* Perform an indivisible atomic operation in the sequential coherent bus model. * aq/rl are satisfied by total instruction order; AMOs never target devices. */ static bool atomic( struct machine *m, struct machine_hart *h, u32 ins, u64 address, u64 source, u64 *result, u64 *cause ) { u32 width = ((ins >> 12) & 7) == 2 ? 4 : 8; u32 operation = ins >> 27; u64 old = 0; if (operation != 2 && !machine_store_probe(m, h, address, width, cause)) { h->reserved = false; return false; } if (operation == 3) { bool success = h->reserved && h->reservation == address && h->reservation_size == width; h->reserved = false; if (success && !machine_bus( m, h, address, width, ACCESS_STORE | ACCESS_ATOMIC, &source, cause )) return false; *result = success ? 0 : 1; return true; } /* Only LR can fail here: an AMO has passed the store probe. */ if (!machine_bus(m, h, address, width, ACCESS_ATOMIC, &old, cause)) return false; *result = sext(old, width * 8); if (operation == 2) { h->reserved = true; h->reservation = address; h->reservation_size = width; return true; } u64 value = 0; if (width == 4) source = (u32)source; switch (operation) { case 0: value = old + source; break; case 1: value = source; break; case 4: value = old ^ source; break; case 8: value = old | source; break; case 12: value = old & source; break; case 16: value = (i64)sext(old, width * 8) < (i64)sext(source, width * 8) ? old : source; break; case 20: value = (i64)sext(old, width * 8) > (i64)sext(source, width * 8) ? old : source; break; case 24: value = old < source ? old : source; break; case 28: value = old > source ? old : source; break; } return machine_bus( m, h, address, width, ACCESS_STORE | ACCESS_ATOMIC, &value, cause ); } /* Compute an OP or OP-32 instruction into `result`. Returns false if the * instruction is illegal. */ static bool register_op(u32 ins, u64 a, u64 b, u64 *result) { bool word = (ins & 0x7f) == OP_OP_32; u32 f3 = (ins >> 12) & 7, f7 = ins >> 25; u64 value = 0; if (f7 == 1) { if (word) { if (f3 == 0) value = (u64)(u32)a * (u32)b; else if (f3 == 4 || f3 == 6) value = divs64(sext(a, 32), sext(b, 32), f3 == 6); else if (f3 == 5) value = (u32)b ? (u32)a / (u32)b : UINT32_MAX; else if (f3 == 7) value = (u32)b ? (u32)a % (u32)b : (u32)a; else return false; } else { switch (f3) { case 0: value = a * b; break; case 1: value = mulh64(a, b); break; case 2: value = mulhsu64(a, b); break; case 3: value = mulhu64(a, b); break; case 4: value = divs64(a, b, false); break; case 5: value = b ? a / b : UINT64_MAX; break; case 6: value = divs64(a, b, true); break; case 7: value = b ? a % b : a; break; } } } else { if (f7 != 0 && f7 != 0x20) return false; if (f7 == 0x20 && f3 != 0 && f3 != 5) return false; if (word && f3 != 0 && f3 != 1 && f3 != 5) return false; u32 shift = b & (word ? 31 : 63); switch (f3) { case 0: value = f7 ? a - b : a + b; break; case 1: value = a << shift; break; case 2: value = (i64)a < (i64)b; break; case 3: value = a < b; break; case 4: value = a ^ b; break; case 5: if (word) value = f7 ? (u32)((i32)a >> shift) : (u32)a >> shift; else value = f7 ? (u64)((i64)a >> shift) : a >> shift; break; case 6: value = a | b; break; case 7: value = a & b; break; } } *result = word ? sext(value, 32) : value; return true; } /* Memory fault details for one machine vector instruction. */ struct vector_access { struct machine *m; struct machine_hart *h; u64 addr; u64 cause; }; /* Transfer one vector element through the machine bus. */ static bool machine_vector_access( void *context, u64 address, u32 size, bool store, u64 *value ) { struct vector_access *access = context; access->addr = address; return machine_bus( access->m, access->h, address, size, store ? ACCESS_STORE : 0, value, &access->cause ); } void machine_step(struct machine *m, struct machine_hart *h) { h->wrote_instret = false; u64 pending = machine_pending(m, h) & h->mie; if (pending) h->waiting = false; if (pending && (h->privilege < 3 || (h->mstatus & MSTATUS_MIE))) { u64 interrupt = (pending & MIP_MEIP) ? 11 : (pending & MIP_MSIP) ? 3 : 7; machine_trap(m, h, MCAUSE_INTERRUPT | interrupt, 0); return; } if (h->waiting) return; u64 raw = 0, cause = 0, address = h->pc; if (!machine_bus(m, h, h->pc, 4, ACCESS_FETCH, &raw, &cause)) goto fault; u32 ins = (u32)raw; if (m->config->trace) fprintf( stderr, "hart=%u tick=%llu pc=0x%llx insn=%08x mode=%c\n", h->id, m->ticks, h->pc, ins, h->privilege ? 'M' : 'U' ); u32 opcode = ins & 0x7f, rd = (ins >> 7) & 31; u32 f3 = (ins >> 12) & 7, rs1 = (ins >> 15) & 31, rs2 = (ins >> 20) & 31; u32 f7 = ins >> 25; u64 a = h->regs[rs1], b = h->regs[rs2]; u64 immediate = sext(ins >> 20, 12), result = 0, next = h->pc + 4; bool write_rd = true; enum bitmanip_op bm = bitmanip_decode(ins); if (bm != BM_NONE) { result = bitmanip_execute(bm, ins, a, b); goto retire; } if (opcode == 0x57 || opcode == 7 || opcode == 39) { if (!(h->mstatus & (3ULL << 9))) goto illegal; struct vector_access access = { m, h, 0, 0 }; enum vector_result vr = vector_execute( &h->vector, h->regs, ins, machine_vector_access, &access ); if (vr == VECTOR_ILLEGAL) goto illegal; h->mstatus |= 3ULL << 9; if (vr == VECTOR_FAULT) { cause = access.cause; address = access.addr; goto fault; } write_rd = false; goto retire; } switch (opcode) { case OP_LUI: result = sext(ins & 0xfffff000U, 32); break; case OP_AUIPC: result = h->pc + sext(ins & 0xfffff000U, 32); break; case OP_JAL: immediate = ((ins >> 31) << 20) | (((ins >> 12) & 255) << 12) | (((ins >> 20) & 1) << 11) | (((ins >> 21) & 1023) << 1); next = h->pc + sext(immediate, 21); result = h->pc + 4; goto target; case OP_JALR: if (f3) goto illegal; next = (a + immediate) & ~1ULL; result = h->pc + 4; goto target; case OP_BRANCH: { bool taken; write_rd = false; switch (f3) { case 0: taken = a == b; break; case 1: taken = a != b; break; case 4: taken = (i64)a < (i64)b; break; case 5: taken = (i64)a >= (i64)b; break; case 6: taken = a < b; break; case 7: taken = a >= b; break; default: goto illegal; } if (taken) { immediate = ((ins >> 31) << 12) | (((ins >> 7) & 1) << 11) | (((ins >> 25) & 63) << 5) | (((ins >> 8) & 15) << 1); next = h->pc + sext(immediate, 13); } goto target; } case OP_LOAD: { if (f3 == 7) goto illegal; u32 width = 1U << (f3 & 3); address = a + immediate; if (!machine_bus(m, h, address, width, 0, &result, &cause)) goto fault; if (!(f3 & 4)) result = sext(result, width * 8); break; } case OP_STORE: { if (f3 > 3) goto illegal; u32 width = 1U << f3; address = a + sext(((ins >> 25) << 5) | ((ins >> 7) & 31), 12); if (!machine_bus(m, h, address, width, ACCESS_STORE, &b, &cause)) goto fault; write_rd = false; break; } case OP_IMM: switch (f3) { case 0: result = a + immediate; break; case 2: result = (i64)a < (i64)immediate; break; case 3: result = a < immediate; break; case 4: result = a ^ immediate; break; case 6: result = a | immediate; break; case 7: result = a & immediate; break; case 1: if (ins >> 26) goto illegal; result = a << ((ins >> 20) & 63); break; case 5: if ((ins >> 26) != 0 && (ins >> 26) != 0x10) goto illegal; result = (ins >> 26) ? (u64)((i64)a >> ((ins >> 20) & 63)) : a >> ((ins >> 20) & 63); break; } break; case OP_IMM_32: if (f3 == 0) result = a + immediate; else if (f3 == 1 && f7 == 0) result = (u32)a << rs2; else if (f3 == 5 && f7 == 0) result = (u32)a >> rs2; else if (f3 == 5 && f7 == 0x20) result = (u32)((i32)a >> rs2); else goto illegal; result = sext(result, 32); break; case OP_OP: case OP_OP_32: if (!register_op(ins, a, b, &result)) goto illegal; break; case OP_AMO: { if (f3 != 2 && f3 != 3) goto illegal; u32 operation = ins >> 27; switch (operation) { case 2: if (rs2) goto illegal; break; case 0: case 1: case 3: case 4: case 8: case 12: case 16: case 20: case 24: case 28: break; default: goto illegal; } address = a; if (!atomic(m, h, ins, address, b, &result, &cause)) goto fault; break; } case OP_FENCE: if (rs1 || rd) goto illegal; if (f3 == 0) { u32 fm = ins >> 28; if (fm != 0 && !(fm == 8 && ((ins >> 20) & 255) == 0x33)) goto illegal; } else if (f3 != 1 || (ins >> 20)) goto illegal; /* The bus is strongly ordered; fetch is uncached, including FENCE.I. */ write_rd = false; break; case OP_SYSTEM: write_rd = false; if (f3) { if (!machine_csr(m, h, ins)) goto illegal; } else if (ins == 0x00000073) { /* ecall */ machine_trap( m, h, h->privilege == 0 ? CAUSE_ECALL_U : CAUSE_ECALL_M, 0 ); return; } else if (ins == 0x00100073) { /* ebreak */ machine_trap(m, h, CAUSE_BREAKPOINT, h->pc); return; } else if (ins == 0x10500073) { /* wfi */ h->waiting = true; } else if (ins == 0x30200073) { /* mret */ if (h->privilege != 3) goto illegal; h->privilege = (h->mstatus >> 11) & 3; h->mstatus = (h->mstatus & ~(MSTATUS_MIE | MSTATUS_MPP)) | ((h->mstatus & MSTATUS_MPIE) >> 4) | MSTATUS_MPIE; next = h->mepc; } else goto illegal; break; default: goto illegal; } goto retire; target: if (next & 3) { address = next; cause = CAUSE_FETCH_MISALIGNED; goto fault; } retire: if (write_rd && rd) h->regs[rd] = result; h->pc = next; if (!h->wrote_instret) h->minstret++; m->retired++; return; illegal: machine_trap(m, h, CAUSE_ILLEGAL, ins); return; fault: machine_trap(m, h, cause, address); }