machine_cpu.c 13.6 KiB raw
1
/* Instruction execution for one machine hart. */
2
#include <stdint.h>
3
#include <stdio.h>
4
5
#include "bitmanip.h"
6
#include "machine_internal.h"
7
#include "muldiv.h"
8
#include "riscv.h"
9
10
/* RV64 wraparound arithmetic stays unsigned; extension never shifts by 64. */
11
static u64 sext(u64 value, u32 bits) {
12
    if (bits == 64)
13
        return value;
14
    u64 sign  = 1ULL << (bits - 1);
15
    value    &= (1ULL << bits) - 1;
16
    return (value ^ sign) - sign;
17
}
18
19
/* Perform an indivisible atomic operation in the sequential coherent bus model.
20
 * aq/rl are satisfied by total instruction order; AMOs never target devices. */
21
static bool atomic(
22
    struct machine      *m,
23
    struct machine_hart *h,
24
    u32                  ins,
25
    u64                  address,
26
    u64                  source,
27
    u64                 *result,
28
    u64                 *cause
29
) {
30
    u32 width     = ((ins >> 12) & 7) == 2 ? 4 : 8;
31
    u32 operation = ins >> 27;
32
    u64 old       = 0;
33
    if (operation != 2 && !machine_store_probe(m, h, address, width, cause)) {
34
        h->reserved = false;
35
        return false;
36
    }
37
    if (operation == 3) {
38
        bool success = h->reserved && h->reservation == address &&
39
                       h->reservation_size == width;
40
        h->reserved  = false;
41
        if (success && !machine_bus(
42
                           m,
43
                           h,
44
                           address,
45
                           width,
46
                           ACCESS_STORE | ACCESS_ATOMIC,
47
                           &source,
48
                           cause
49
                       ))
50
            return false;
51
        *result = success ? 0 : 1;
52
        return true;
53
    }
54
    /* Only LR can fail here: an AMO has passed the store probe. */
55
    if (!machine_bus(m, h, address, width, ACCESS_ATOMIC, &old, cause))
56
        return false;
57
    *result = sext(old, width * 8);
58
    if (operation == 2) {
59
        h->reserved         = true;
60
        h->reservation      = address;
61
        h->reservation_size = width;
62
        return true;
63
    }
64
    u64 value = 0;
65
    if (width == 4)
66
        source = (u32)source;
67
    switch (operation) {
68
    case 0:
69
        value = old + source;
70
        break;
71
    case 1:
72
        value = source;
73
        break;
74
    case 4:
75
        value = old ^ source;
76
        break;
77
    case 8:
78
        value = old | source;
79
        break;
80
    case 12:
81
        value = old & source;
82
        break;
83
    case 16:
84
        value = (i64)sext(old, width * 8) < (i64)sext(source, width * 8)
85
                    ? old
86
                    : source;
87
        break;
88
    case 20:
89
        value = (i64)sext(old, width * 8) > (i64)sext(source, width * 8)
90
                    ? old
91
                    : source;
92
        break;
93
    case 24:
94
        value = old < source ? old : source;
95
        break;
96
    case 28:
97
        value = old > source ? old : source;
98
        break;
99
    }
100
    return machine_bus(
101
        m, h, address, width, ACCESS_STORE | ACCESS_ATOMIC, &value, cause
102
    );
103
}
104
105
/* Compute an OP or OP-32 instruction into `result`. Returns false if the
106
 * instruction is illegal. */
107
static bool register_op(u32 ins, u64 a, u64 b, u64 *result) {
108
    bool word = (ins & 0x7f) == OP_OP_32;
109
    u32  f3 = (ins >> 12) & 7, f7 = ins >> 25;
110
    u64  value = 0;
111
    if (f7 == 1) {
112
        if (word) {
113
            if (f3 == 0)
114
                value = (u64)(u32)a * (u32)b;
115
            else if (f3 == 4 || f3 == 6)
116
                value = divs64(sext(a, 32), sext(b, 32), f3 == 6);
117
            else if (f3 == 5)
118
                value = (u32)b ? (u32)a / (u32)b : UINT32_MAX;
119
            else if (f3 == 7)
120
                value = (u32)b ? (u32)a % (u32)b : (u32)a;
121
            else
122
                return false;
123
        } else {
124
            switch (f3) {
125
            case 0:
126
                value = a * b;
127
                break;
128
            case 1:
129
                value = mulh64(a, b);
130
                break;
131
            case 2:
132
                value = mulhsu64(a, b);
133
                break;
134
            case 3:
135
                value = mulhu64(a, b);
136
                break;
137
            case 4:
138
                value = divs64(a, b, false);
139
                break;
140
            case 5:
141
                value = b ? a / b : UINT64_MAX;
142
                break;
143
            case 6:
144
                value = divs64(a, b, true);
145
                break;
146
            case 7:
147
                value = b ? a % b : a;
148
                break;
149
            }
150
        }
151
    } else {
152
        if (f7 != 0 && f7 != 0x20)
153
            return false;
154
        if (f7 == 0x20 && f3 != 0 && f3 != 5)
155
            return false;
156
        if (word && f3 != 0 && f3 != 1 && f3 != 5)
157
            return false;
158
        u32 shift = b & (word ? 31 : 63);
159
        switch (f3) {
160
        case 0:
161
            value = f7 ? a - b : a + b;
162
            break;
163
        case 1:
164
            value = a << shift;
165
            break;
166
        case 2:
167
            value = (i64)a < (i64)b;
168
            break;
169
        case 3:
170
            value = a < b;
171
            break;
172
        case 4:
173
            value = a ^ b;
174
            break;
175
        case 5:
176
            if (word)
177
                value = f7 ? (u32)((i32)a >> shift) : (u32)a >> shift;
178
            else
179
                value = f7 ? (u64)((i64)a >> shift) : a >> shift;
180
            break;
181
        case 6:
182
            value = a | b;
183
            break;
184
        case 7:
185
            value = a & b;
186
            break;
187
        }
188
    }
189
    *result = word ? sext(value, 32) : value;
190
    return true;
191
}
192
193
/* Memory fault details for one machine vector instruction. */
194
struct vector_access {
195
    struct machine      *m;
196
    struct machine_hart *h;
197
    u64                  addr;
198
    u64                  cause;
199
};
200
201
/* Transfer one vector element through the machine bus. */
202
static bool machine_vector_access(
203
    void *context, u64 address, u32 size, bool store, u64 *value
204
) {
205
    struct vector_access *access = context;
206
    access->addr                 = address;
207
    return machine_bus(
208
        access->m,
209
        access->h,
210
        address,
211
        size,
212
        store ? ACCESS_STORE : 0,
213
        value,
214
        &access->cause
215
    );
216
}
217
218
void machine_step(struct machine *m, struct machine_hart *h) {
219
    h->wrote_instret = false;
220
    u64 pending      = machine_pending(m, h) & h->mie;
221
    if (pending)
222
        h->waiting = false;
223
    if (pending && (h->privilege < 3 || (h->mstatus & MSTATUS_MIE))) {
224
        u64 interrupt = (pending & MIP_MEIP)   ? 11
225
                        : (pending & MIP_MSIP) ? 3
226
                                               : 7;
227
        machine_trap(m, h, MCAUSE_INTERRUPT | interrupt, 0);
228
        return;
229
    }
230
    if (h->waiting)
231
        return;
232
    u64 raw = 0, cause = 0, address = h->pc;
233
    if (!machine_bus(m, h, h->pc, 4, ACCESS_FETCH, &raw, &cause))
234
        goto fault;
235
    u32 ins = (u32)raw;
236
    if (m->config->trace)
237
        fprintf(
238
            stderr,
239
            "hart=%u tick=%llu pc=0x%llx insn=%08x mode=%c\n",
240
            h->id,
241
            m->ticks,
242
            h->pc,
243
            ins,
244
            h->privilege ? 'M' : 'U'
245
        );
246
    u32  opcode = ins & 0x7f, rd = (ins >> 7) & 31;
247
    u32  f3 = (ins >> 12) & 7, rs1 = (ins >> 15) & 31, rs2 = (ins >> 20) & 31;
248
    u32  f7 = ins >> 25;
249
    u64  a = h->regs[rs1], b = h->regs[rs2];
250
    u64  immediate = sext(ins >> 20, 12), result = 0, next = h->pc + 4;
251
    bool write_rd       = true;
252
    enum bitmanip_op bm = bitmanip_decode(ins);
253
    if (bm != BM_NONE) {
254
        result = bitmanip_execute(bm, ins, a, b);
255
        goto retire;
256
    }
257
    if (opcode == 0x57 || opcode == 7 || opcode == 39) {
258
        if (!(h->mstatus & (3ULL << 9)))
259
            goto illegal;
260
        struct vector_access access = { m, h, 0, 0 };
261
        enum vector_result   vr     = vector_execute(
262
            &h->vector, h->regs, ins, machine_vector_access, &access
263
        );
264
        if (vr == VECTOR_ILLEGAL)
265
            goto illegal;
266
        h->mstatus |= 3ULL << 9;
267
        if (vr == VECTOR_FAULT) {
268
            cause   = access.cause;
269
            address = access.addr;
270
            goto fault;
271
        }
272
        write_rd = false;
273
        goto retire;
274
    }
275
    switch (opcode) {
276
    case OP_LUI:
277
        result = sext(ins & 0xfffff000U, 32);
278
        break;
279
    case OP_AUIPC:
280
        result = h->pc + sext(ins & 0xfffff000U, 32);
281
        break;
282
    case OP_JAL:
283
        immediate = ((ins >> 31) << 20) | (((ins >> 12) & 255) << 12) |
284
                    (((ins >> 20) & 1) << 11) | (((ins >> 21) & 1023) << 1);
285
        next      = h->pc + sext(immediate, 21);
286
        result    = h->pc + 4;
287
        goto target;
288
    case OP_JALR:
289
        if (f3)
290
            goto illegal;
291
        next   = (a + immediate) & ~1ULL;
292
        result = h->pc + 4;
293
        goto target;
294
    case OP_BRANCH: {
295
        bool taken;
296
        write_rd = false;
297
        switch (f3) {
298
        case 0:
299
            taken = a == b;
300
            break;
301
        case 1:
302
            taken = a != b;
303
            break;
304
        case 4:
305
            taken = (i64)a < (i64)b;
306
            break;
307
        case 5:
308
            taken = (i64)a >= (i64)b;
309
            break;
310
        case 6:
311
            taken = a < b;
312
            break;
313
        case 7:
314
            taken = a >= b;
315
            break;
316
        default:
317
            goto illegal;
318
        }
319
        if (taken) {
320
            immediate = ((ins >> 31) << 12) | (((ins >> 7) & 1) << 11) |
321
                        (((ins >> 25) & 63) << 5) | (((ins >> 8) & 15) << 1);
322
            next      = h->pc + sext(immediate, 13);
323
        }
324
        goto target;
325
    }
326
    case OP_LOAD: {
327
        if (f3 == 7)
328
            goto illegal;
329
        u32 width = 1U << (f3 & 3);
330
        address   = a + immediate;
331
        if (!machine_bus(m, h, address, width, 0, &result, &cause))
332
            goto fault;
333
        if (!(f3 & 4))
334
            result = sext(result, width * 8);
335
        break;
336
    }
337
    case OP_STORE: {
338
        if (f3 > 3)
339
            goto illegal;
340
        u32 width = 1U << f3;
341
        address   = a + sext(((ins >> 25) << 5) | ((ins >> 7) & 31), 12);
342
        if (!machine_bus(m, h, address, width, ACCESS_STORE, &b, &cause))
343
            goto fault;
344
        write_rd = false;
345
        break;
346
    }
347
    case OP_IMM:
348
        switch (f3) {
349
        case 0:
350
            result = a + immediate;
351
            break;
352
        case 2:
353
            result = (i64)a < (i64)immediate;
354
            break;
355
        case 3:
356
            result = a < immediate;
357
            break;
358
        case 4:
359
            result = a ^ immediate;
360
            break;
361
        case 6:
362
            result = a | immediate;
363
            break;
364
        case 7:
365
            result = a & immediate;
366
            break;
367
        case 1:
368
            if (ins >> 26)
369
                goto illegal;
370
            result = a << ((ins >> 20) & 63);
371
            break;
372
        case 5:
373
            if ((ins >> 26) != 0 && (ins >> 26) != 0x10)
374
                goto illegal;
375
            result = (ins >> 26) ? (u64)((i64)a >> ((ins >> 20) & 63))
376
                                 : a >> ((ins >> 20) & 63);
377
            break;
378
        }
379
        break;
380
    case OP_IMM_32:
381
        if (f3 == 0)
382
            result = a + immediate;
383
        else if (f3 == 1 && f7 == 0)
384
            result = (u32)a << rs2;
385
        else if (f3 == 5 && f7 == 0)
386
            result = (u32)a >> rs2;
387
        else if (f3 == 5 && f7 == 0x20)
388
            result = (u32)((i32)a >> rs2);
389
        else
390
            goto illegal;
391
        result = sext(result, 32);
392
        break;
393
    case OP_OP:
394
    case OP_OP_32:
395
        if (!register_op(ins, a, b, &result))
396
            goto illegal;
397
        break;
398
    case OP_AMO: {
399
        if (f3 != 2 && f3 != 3)
400
            goto illegal;
401
        u32 operation = ins >> 27;
402
        switch (operation) {
403
        case 2:
404
            if (rs2)
405
                goto illegal;
406
            break;
407
        case 0:
408
        case 1:
409
        case 3:
410
        case 4:
411
        case 8:
412
        case 12:
413
        case 16:
414
        case 20:
415
        case 24:
416
        case 28:
417
            break;
418
        default:
419
            goto illegal;
420
        }
421
        address = a;
422
        if (!atomic(m, h, ins, address, b, &result, &cause))
423
            goto fault;
424
        break;
425
    }
426
    case OP_FENCE:
427
        if (rs1 || rd)
428
            goto illegal;
429
        if (f3 == 0) {
430
            u32 fm = ins >> 28;
431
            if (fm != 0 && !(fm == 8 && ((ins >> 20) & 255) == 0x33))
432
                goto illegal;
433
        } else if (f3 != 1 || (ins >> 20))
434
            goto illegal;
435
        /* The bus is strongly ordered; fetch is uncached, including FENCE.I. */
436
        write_rd = false;
437
        break;
438
    case OP_SYSTEM:
439
        write_rd = false;
440
        if (f3) {
441
            if (!machine_csr(m, h, ins))
442
                goto illegal;
443
        } else if (ins == 0x00000073) { /* ecall */
444
            machine_trap(
445
                m, h, h->privilege == 0 ? CAUSE_ECALL_U : CAUSE_ECALL_M, 0
446
            );
447
            return;
448
        } else if (ins == 0x00100073) { /* ebreak */
449
            machine_trap(m, h, CAUSE_BREAKPOINT, h->pc);
450
            return;
451
        } else if (ins == 0x10500073) { /* wfi */
452
            h->waiting = true;
453
        } else if (ins == 0x30200073) { /* mret */
454
            if (h->privilege != 3)
455
                goto illegal;
456
            h->privilege = (h->mstatus >> 11) & 3;
457
            h->mstatus   = (h->mstatus & ~(MSTATUS_MIE | MSTATUS_MPP)) |
458
                           ((h->mstatus & MSTATUS_MPIE) >> 4) | MSTATUS_MPIE;
459
            next         = h->mepc;
460
        } else
461
            goto illegal;
462
        break;
463
    default:
464
        goto illegal;
465
    }
466
    goto retire;
467
468
target:
469
    if (next & 3) {
470
        address = next;
471
        cause   = CAUSE_FETCH_MISALIGNED;
472
        goto fault;
473
    }
474
retire:
475
    if (write_rd && rd)
476
        h->regs[rd] = result;
477
    h->pc = next;
478
    if (!h->wrote_instret)
479
        h->minstret++;
480
    m->retired++;
481
    return;
482
illegal:
483
    machine_trap(m, h, CAUSE_ILLEGAL, ins);
484
    return;
485
fault:
486
    machine_trap(m, h, cause, address);
487
}