jit: Translate short vector copies directly

07bf905970972a412dbc00f37070c35a478c30251929f46ad89c085715e5a14f
Keep fixed-length vector setup and transfers in one block. Use
fixed-width host moves and reuse vector loads for adjacent stores.
Check each range against the hosted memory limit and preserve the
vector state and precise fault locations.

Change-Id: I4de3da047eee3539ab25cd76683401b26bd04a55
Alexis Sellier committed ago 1 parent ec6635ad
jit.c +171 -20
49 49
    u32  pos;      /* Current write position. */
50 50
    u32  capacity; /* Maximum bytes. */
51 51
    bool overflow; /* Set if we ran out of space. */
52 52
    bool vector;   /* Block uses four-lane 32-bit SIMD. */
53 53
    bool guard;    /* Emit the vector shape guard at block entry. */
54 +
    /* A short vector configuration written within this block. */
55 +
    u32 short_bytes;
56 +
    u32 short_type;
54 57
    /* The register file holds every guest register and host registers may be
55 58
     * clobbered: set from an instruction that addresses the register file
56 59
     * directly until the held registers are reloaded. */
57 60
    bool spilled;
58 61
    u32  insns; /* Instructions through the current operation. */
1422 1425
    emit_u8(e, 0x8d);
1423 1426
    emit_u8(e, modrm(2, dst, X_RAX));
1424 1427
    emit_u32(e, reg * VECTOR_BYTES);
1425 1428
}
1426 1429
1430 +
/* Set vl from rcx, set vtype, and clear vstart. */
1431 +
static void emit_vector_state(struct emitter *e, u32 type) {
1432 +
    emit_vector_base(e);
1433 +
    emit_rexw(e);
1434 +
    emit_u8(e, 0x89);
1435 +
    emit_u8(e, modrm(2, X_RCX, X_RAX));
1436 +
    emit_u32(e, offsetof(struct vector_state, vl));
1437 +
    emit_rexw(e);
1438 +
    emit_u8(e, 0xc7);
1439 +
    emit_u8(e, modrm(2, 0, X_RAX));
1440 +
    emit_u32(e, offsetof(struct vector_state, vtype));
1441 +
    emit_u32(e, type);
1442 +
    emit_rexw(e);
1443 +
    emit_u8(e, 0xc7);
1444 +
    emit_u8(e, modrm(2, 0, X_RAX));
1445 +
    emit_u32(e, offsetof(struct vector_state, vstart));
1446 +
    emit_u32(e, 0);
1447 +
}
1448 +
1427 1449
/* Select a static vector type and end the block after its state change. */
1428 1450
static bool emit_vector_config(struct emitter *e, u32 ins, u32 pc) {
1429 1451
    bool immediate = (ins >> 30) == 3;
1430 1452
    if ((!immediate && (ins >> 31)) || ((ins >> 12) & 7) != 7)
1431 1453
        return false;
1450 1472
        emit_u8(e, 0x0f);
1451 1473
        emit_u8(e, 0x47);
1452 1474
        emit_u8(e, 0xc1); /* cmova rax, rcx */
1453 1475
    }
1454 1476
    emit_store_guest(e, dst, X_RAX);
1455 -
    emit_rexw(e);
1456 -
    emit_u8(e, 0x89);
1457 -
    emit_u8(e, 0xc2);
1458 -
    emit_vector_base(e);
1459 -
    emit_rexw(e);
1460 -
    emit_u8(e, 0x89);
1461 -
    emit_u8(e, modrm(2, X_RDX, X_RAX));
1462 -
    emit_u32(e, offsetof(struct vector_state, vl));
1463 -
    emit_rexw(e);
1464 -
    emit_u8(e, 0xc7);
1465 -
    emit_u8(e, modrm(2, 0, X_RAX));
1466 -
    emit_u32(e, offsetof(struct vector_state, vtype));
1467 -
    emit_u32(e, type);
1468 -
    emit_rexw(e);
1469 -
    emit_u8(e, 0xc7);
1470 -
    emit_u8(e, modrm(2, 0, X_RAX));
1471 -
    emit_u32(e, offsetof(struct vector_state, vstart));
1472 -
    emit_u32(e, 0);
1477 +
    emit_mov_rr(e, X_RCX, X_RAX);
1478 +
    emit_vector_state(e, type);
1473 1479
    emit_chain_exit(e, pc + INSTR_SIZE);
1474 1480
    return true;
1475 1481
}
1476 1482
1477 1483
/* Fill active bytes of a register group from a scalar or signed immediate. */
1520 1526
    emit_u8(e, 0x39);
1521 1527
    emit_u8(e, 0xc8);
1522 1528
    emit_fault_if(e, pc, 0x87); /* ja */
1523 1529
}
1524 1530
1531 +
/* Decode a fixed element count from vsetivli or a preceding addi from zero
1532 +
 * in the current straight-line run. */
1533 +
static bool emit_short_vector_config(struct emitter *e, u32 ins, u32 pc) {
1534 +
    if ((ins & 0x707f) != 0x7057)
1535 +
        return false;
1536 +
    bool immediate = (ins >> 30) == 3;
1537 +
    if (!immediate && (ins >> 31))
1538 +
        return false;
1539 +
    u32 type = (ins >> 20) & (immediate ? 1023 : 2047);
1540 +
    if (type & ~0xdbu)
1541 +
        return false;
1542 +
    u32 src   = (ins >> 15) & 31;
1543 +
    u32 count = src;
1544 +
    if (!immediate) {
1545 +
        if (!src || e->run_start[e->runs - 1] == pc)
1546 +
            return false;
1547 +
        instr_t previous;
1548 +
        memcpy(&previous, e->mem + pc - INSTR_SIZE, sizeof(previous));
1549 +
        if ((previous.raw & 0x000fffff) != (src << 7 | 0x13))
1550 +
            return false;
1551 +
        count = get_i_imm(previous);
1552 +
    }
1553 +
    u32 lmul  = type & 3;
1554 +
    u32 width = (type >> 3) & 3;
1555 +
    u32 max   = (VECTOR_BYTES << lmul) >> width;
1556 +
    u32 bytes = count << width;
1557 +
    if (count == 0 || bytes > 64 || count > max || (bytes & 3) == 3)
1558 +
        return false;
1559 +
    u32  dst     = (ins >> 7) & 31;
1560 +
    bool changed = e->short_bytes != bytes || e->short_type != type;
1561 +
    if (changed || dst)
1562 +
        emit_mov_imm32_sx(e, X_RCX, count);
1563 +
    emit_store_guest(e, dst, X_RCX);
1564 +
    if (changed)
1565 +
        emit_vector_state(e, type);
1566 +
    e->short_bytes = bytes;
1567 +
    e->short_type  = type;
1568 +
    return true;
1569 +
}
1570 +
1571 +
/* Transfer exactly one 16-, 8-, or 4-byte chunk through an XMM register. */
1572 +
static void emit_short_chunk(
1573 +
    struct emitter *e, bool store, bool guest, u32 offset, u32 bytes, u32 xmm
1574 +
) {
1575 +
    emit_u8(e, bytes == 16 || (bytes == 8 && !store) ? 0xf3 : 0x66);
1576 +
    u8 opcode = store ? 0x7f : 0x6f;
1577 +
    if (bytes == 8)
1578 +
        opcode = store ? 0xd6 : 0x7e;
1579 +
    if (bytes == 4)
1580 +
        opcode = store ? 0x7e : 0x6e;
1581 +
    if (guest) {
1582 +
        emit_guest_memory(e, false, false, true, opcode, xmm, X_RAX, offset);
1583 +
        return;
1584 +
    }
1585 +
    emit_u8(e, 0x0f);
1586 +
    emit_u8(e, opcode);
1587 +
    emit_u8(e, modrm(2, xmm, X_RCX));
1588 +
    emit_u32(e, offset);
1589 +
}
1590 +
1591 +
/* Move the last byte or word through the source address register. */
1592 +
static void emit_short_tail(
1593 +
    struct emitter *e, bool store, u32 offset, u32 slot, u32 bytes
1594 +
) {
1595 +
    u8 load  = bytes == 2 ? 0xb7 : 0xb6;
1596 +
    u8 write = bytes == 2 ? 0x89 : 0x88;
1597 +
    if (store) {
1598 +
        emit_u8(e, 0x0f);
1599 +
        emit_u8(e, load);
1600 +
        emit_u8(e, modrm(2, X_RCX, X_RCX));
1601 +
        emit_u32(e, slot);
1602 +
        emit_guest_memory(
1603 +
            e, bytes == 2, false, false, write, X_RCX, X_RAX, offset
1604 +
        );
1605 +
    } else {
1606 +
        emit_guest_memory(e, false, false, true, load, X_RAX, X_RAX, offset);
1607 +
        if (bytes == 2)
1608 +
            emit_u8(e, 0x66);
1609 +
        emit_u8(e, write);
1610 +
        emit_u8(e, modrm(2, X_RAX, X_RCX));
1611 +
        emit_u32(e, slot);
1612 +
    }
1613 +
}
1614 +
1615 +
/* Transfer a known vector group between guest memory and the vector file.
1616 +
 * Each transfer retains its fault PC and checks the full range. */
1617 +
static bool emit_short_vector_memory(struct emitter *e, u32 ins, u32 pc) {
1618 +
    u32  width = (e->short_type >> 3) & 3;
1619 +
    u32  field = (width ? width + 4 : 0) << 12;
1620 +
    u32  op    = (ins & 0xfff0707f) ^ field;
1621 +
    bool store = op == 0x02000027;
1622 +
    if (!e->short_bytes || (op != 0x02000007 && !store))
1623 +
        return false;
1624 +
    u32 reg = (ins >> 7) & 31;
1625 +
    if (reg & ((1u << (e->short_type & 3)) - 1))
1626 +
        return false;
1627 +
1628 +
    emit_load_guest(e, X_RAX, (ins >> 15) & 31);
1629 +
    /* Hosted memory size is fixed for the lifetime of the code cache. */
1630 +
    u64 memory_size = e->jit->io.memory_size;
1631 +
    if (memory_size < e->short_bytes ||
1632 +
        memory_size - e->short_bytes > INT32_MAX) {
1633 +
        emit_vector_bound(e, e->short_bytes, pc);
1634 +
    } else {
1635 +
        emit_rexw(e);
1636 +
        emit_u8(e, 0x3d); /* cmp rax, imm32 */
1637 +
        emit_u32(e, memory_size - e->short_bytes);
1638 +
        emit_fault_if(e, pc, 0x87); /* ja */
1639 +
    }
1640 +
1641 +
    bool reuse = false;
1642 +
    if (store && !(e->short_bytes & 3) && e->run_start[e->runs - 1] != pc) {
1643 +
        u32 previous;
1644 +
        memcpy(&previous, e->mem + pc - INSTR_SIZE, sizeof(previous));
1645 +
        reuse = (previous & 0xfff07fff) == (0x02000007 | field | reg << 7);
1646 +
    }
1647 +
    if (!reuse) {
1648 +
        emit_u8(e, 0x49);
1649 +
        emit_u8(e, 0x8b);
1650 +
        emit_u8(e, modrm(1, X_RCX, HIO));
1651 +
        emit_u8(e, offsetof(struct jit_io, vector));
1652 +
    }
1653 +
    for (u32 offset = 0, xmm = 0; offset < e->short_bytes; xmm++) {
1654 +
        u32 left  = e->short_bytes - offset;
1655 +
        u32 bytes = left >= 16 ? 16 : left >= 8 ? 8 : left >= 4 ? 4 : left;
1656 +
        u32 slot  = reg * VECTOR_BYTES + offset;
1657 +
        if (bytes < 4) {
1658 +
            emit_short_tail(e, store, offset, slot, bytes);
1659 +
        } else {
1660 +
            if (!reuse)
1661 +
                emit_short_chunk(
1662 +
                    e, false, !store, store ? slot : offset, bytes, xmm
1663 +
                );
1664 +
            emit_short_chunk(e, true, store, store ? offset : slot, bytes, xmm);
1665 +
        }
1666 +
        offset += bytes;
1667 +
    }
1668 +
    return true;
1669 +
}
1670 +
1525 1671
/* Transfer active bytes with complete bounds checks before any guest write. */
1526 1672
static bool emit_group_memory(struct emitter *e, u32 ins, u32 pc) {
1527 1673
    u32  op = ins & 127, reg = (ins >> 7) & 31;
1528 1674
    u32  mode = (ins >> 26) & 3, index = (ins >> 20) & 31;
1529 1675
    u32  field = (ins >> 12) & 7, width = field == 6 ? 2 : 0;
2429 2575
        return true;
2430 2576
    if (opcode == 0x57 && emit_byte_splat(e, ins.raw, pc))
2431 2577
        return false;
2432 2578
    if ((opcode == 7 || opcode == 39) && emit_group_memory(e, ins.raw, pc))
2433 2579
        return false;
2434 -
    if ((opcode == 7 || opcode == 39) && emit_indexed_words(e, ins.raw, pc))
2580 +
    if (!e->short_bytes && (opcode == 7 || opcode == 39) &&
2581 +
        emit_indexed_words(e, ins.raw, pc))
2435 2582
        return false;
2436 2583
    if (opcode == 0x57 && emit_word_groups(e, ins.raw, pc))
2437 2584
        return false;
2438 -
    if (opcode == 0x57 && emit_vector(e, ins.raw))
2585 +
    if (!e->short_bytes && opcode == 0x57 && emit_vector(e, ins.raw))
2439 2586
        return false;
2440 2587
2441 2588
    switch (opcode) {
2442 2589
2443 2590
    case OP_LUI:
2800 2947
}
2801 2948
2802 2949
/* Translate one instruction, keeping the held guest registers coherent
2803 2950
 * around those whose emitters address the register file or clobber them. */
2804 2951
static bool translate_insn(struct emitter *e, instr_t ins, u32 pc) {
2952 +
    if (emit_short_vector_config(e, ins.raw, pc) ||
2953 +
        emit_short_vector_memory(e, ins.raw, pc))
2954 +
        return false;
2955 +
2805 2956
    /* Consecutive such instructions share one spill and reload. */
2806 2957
    if (needs_spill(ins)) {
2807 2958
        if (!e->spilled)
2808 2959
            emit_spill(e);
2809 2960
        return translate_body(e, ins, pc);
jit.h +5 -5
72 72
};
73 73
74 74
/* Exchange area between generated code and C.  Generated code keeps a pointer
75 75
 * to it in r14. */
76 76
struct jit_io {
77 -
    u32                  pc;          /* Next guest PC on exit. */
78 -
    i64                  budget;      /* Remaining guest instructions. */
79 -
    u8                  *patch;       /* Chain jump rel32 field. */
80 -
    struct vector_state *vector;      /* Architectural vector registers. */
81 -
    u64                  memory_size; /* Accessible hosted memory bytes. */
77 +
    u32                  pc;     /* Next guest PC on exit. */
78 +
    i64                  budget; /* Remaining guest instructions. */
79 +
    u8                  *patch;  /* Chain jump rel32 field. */
80 +
    struct vector_state *vector; /* Architectural vector registers. */
81 +
    u64 memory_size; /* Accessible bytes, fixed while code is cached. */
82 82
    /* LR reservation: the address in the low word and the access width in
83 83
     * the high word, or zero when none is held. */
84 84
    u64 reservation;
85 85
    /* Bounds of the host stack frames for translated calls. */
86 86
    u64  stack_base;
test/jit_vector.c +47 -0
54 54
    return vector_execute(v, x, ins, transfer, mem);
55 55
}
56 56
57 57
/* Native cases must complete without interpreter fallback. */
58 58
static bool require_native;
59 +
/* A short vector copy stays in the same translated block as its setup. */
60 +
static bool require_single_block;
59 61
/* Byte stride supplied to strided-store checks. */
60 62
static u64 byte_stride = 3;
61 63
62 64
/* Compare all state, memory, faults and instruction accounting for one block.
63 65
 */
139 141
    assert(memcmp(x, y, sizeof(x)) == 0);
140 142
    assert(memcmp(actual, expected, DATA) == 0);
141 143
    if ((count == 8 || (count == 7 && base == 128)) &&
142 144
        (type & ~0xc0u) == 0x11 && vl == 4 && start == 0)
143 145
        assert(jit.block_count == 1);
146 +
    if (require_single_block)
147 +
        assert(jit.block_count == 1);
144 148
    jit_destroy(&jit);
145 149
}
146 150
151 +
/* Short copies with a known length keep scalar and vector state precise. */
152 +
static void check_short_copy(void) {
153 +
    u32 load       = 0x02000007 | 10 << 15 | 8 << 7;
154 +
    u32 store      = 0x02000027 | 11 << 15 | 8 << 7;
155 +
    require_native = require_single_block = true;
156 +
    for (u32 size = 1; size <= 64; size++) {
157 +
        require_single_block = (size & 3) != 3;
158 +
        u32 config           = 0xc0007057 | 0xc3 << 20 | size << 15;
159 +
        u32 immediate[]      = { config, load, store };
160 +
        if (size < 32)
161 +
            check(immediate, 3, 0xd1, 4, 3, 0);
162 +
        u32 reg[] = { size << 20 | 5 << 7 | 0x13,
163 +
                      0xc3 << 20 | 5 << 15 | 0x7057,
164 +
                      load,
165 +
                      store };
166 +
        check(reg, 4, 0xd1, 4, 3, 0);
167 +
    }
168 +
    require_single_block = true;
169 +
    for (u32 width = 1; width <= 3; width++) {
170 +
        u32 field = (width + 4) << 12;
171 +
        for (u32 count = 2; count < 32 && count <= (64u >> width); count += 2) {
172 +
            u32 setup  = 0xcc307057 | width << 23 | count << 15;
173 +
            u32 copy[] = { setup, load | field, store | field };
174 +
            check(copy, 3, 0xd1, 4, 3, 128);
175 +
        }
176 +
    }
177 +
    u32 config  = 0xcc387057;
178 +
    u32 other[] = { config, load, store + (8 << 7) };
179 +
    check(other, 3, 0, 0, 0, 128);
180 +
    u32 repeated[] = { config, load, config | 12 << 7, store };
181 +
    check(repeated, 4, 0, 0, 0, 128);
182 +
    require_native = require_single_block = false;
183 +
    u32 fault[]                           = { config, load, store };
184 +
    check(fault, 3, 0, 0, 0, DATA - 8);
185 +
    byte_stride = DATA - 8;
186 +
    check(fault, 3, 0, 0, 0, 128);
187 +
    u32 wide_fault[] = { 0xcd917057, load | (7 << 12), store | (7 << 12) };
188 +
    check(wide_fault, 3, 0, 0, 0, 128);
189 +
    byte_stride = 3;
190 +
    check(wide_fault, 3, 0, 0, 0, DATA - 8);
191 +
}
192 +
147 193
/* Check byte transfers, fills, configuration, and fault boundaries. */
148 194
static void check_bulk(void) {
149 195
    u32 load       = 1u << 25 | 10 << 15 | 8 << 7 | 7;
150 196
    u32 store      = (load & ~127u) | 39;
151 197
    u32 stride     = store | 2u << 26 | 11 << 20;
325 371
    jit_destroy(&jit);
326 372
}
327 373
328 374
int main(void) {
329 375
    check_budget();
376 +
    check_short_copy();
330 377
    check_bulk();
331 378
    check_word_groups();
332 379
    u32 ops[] = {
333 380
        alu(0, 0, 8, 10, 12),   alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0),
334 381
        alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31),   alu(9, 3, 8, 8, 3),