jit: Translate short vector copies directly
07bf905970972a412dbc00f37070c35a478c30251929f46ad89c085715e5a14f
Keep fixed-length vector setup and transfers in one block. Use fixed-width host moves and reuse vector loads for adjacent stores. Check each range against the hosted memory limit and preserve the vector state and precise fault locations. Change-Id: I4de3da047eee3539ab25cd76683401b26bd04a55
1 parent
ec6635ad
jit.c
+171 -20
| 49 | 49 | u32 pos; /* Current write position. */ |
|
| 50 | 50 | u32 capacity; /* Maximum bytes. */ |
|
| 51 | 51 | bool overflow; /* Set if we ran out of space. */ |
|
| 52 | 52 | bool vector; /* Block uses four-lane 32-bit SIMD. */ |
|
| 53 | 53 | bool guard; /* Emit the vector shape guard at block entry. */ |
|
| 54 | + | /* A short vector configuration written within this block. */ |
|
| 55 | + | u32 short_bytes; |
|
| 56 | + | u32 short_type; |
|
| 54 | 57 | /* The register file holds every guest register and host registers may be |
|
| 55 | 58 | * clobbered: set from an instruction that addresses the register file |
|
| 56 | 59 | * directly until the held registers are reloaded. */ |
|
| 57 | 60 | bool spilled; |
|
| 58 | 61 | u32 insns; /* Instructions through the current operation. */ |
| 1422 | 1425 | emit_u8(e, 0x8d); |
|
| 1423 | 1426 | emit_u8(e, modrm(2, dst, X_RAX)); |
|
| 1424 | 1427 | emit_u32(e, reg * VECTOR_BYTES); |
|
| 1425 | 1428 | } |
|
| 1426 | 1429 | ||
| 1430 | + | /* Set vl from rcx, set vtype, and clear vstart. */ |
|
| 1431 | + | static void emit_vector_state(struct emitter *e, u32 type) { |
|
| 1432 | + | emit_vector_base(e); |
|
| 1433 | + | emit_rexw(e); |
|
| 1434 | + | emit_u8(e, 0x89); |
|
| 1435 | + | emit_u8(e, modrm(2, X_RCX, X_RAX)); |
|
| 1436 | + | emit_u32(e, offsetof(struct vector_state, vl)); |
|
| 1437 | + | emit_rexw(e); |
|
| 1438 | + | emit_u8(e, 0xc7); |
|
| 1439 | + | emit_u8(e, modrm(2, 0, X_RAX)); |
|
| 1440 | + | emit_u32(e, offsetof(struct vector_state, vtype)); |
|
| 1441 | + | emit_u32(e, type); |
|
| 1442 | + | emit_rexw(e); |
|
| 1443 | + | emit_u8(e, 0xc7); |
|
| 1444 | + | emit_u8(e, modrm(2, 0, X_RAX)); |
|
| 1445 | + | emit_u32(e, offsetof(struct vector_state, vstart)); |
|
| 1446 | + | emit_u32(e, 0); |
|
| 1447 | + | } |
|
| 1448 | + | ||
| 1427 | 1449 | /* Select a static vector type and end the block after its state change. */ |
|
| 1428 | 1450 | static bool emit_vector_config(struct emitter *e, u32 ins, u32 pc) { |
|
| 1429 | 1451 | bool immediate = (ins >> 30) == 3; |
|
| 1430 | 1452 | if ((!immediate && (ins >> 31)) || ((ins >> 12) & 7) != 7) |
|
| 1431 | 1453 | return false; |
| 1450 | 1472 | emit_u8(e, 0x0f); |
|
| 1451 | 1473 | emit_u8(e, 0x47); |
|
| 1452 | 1474 | emit_u8(e, 0xc1); /* cmova rax, rcx */ |
|
| 1453 | 1475 | } |
|
| 1454 | 1476 | emit_store_guest(e, dst, X_RAX); |
|
| 1455 | - | emit_rexw(e); |
|
| 1456 | - | emit_u8(e, 0x89); |
|
| 1457 | - | emit_u8(e, 0xc2); |
|
| 1458 | - | emit_vector_base(e); |
|
| 1459 | - | emit_rexw(e); |
|
| 1460 | - | emit_u8(e, 0x89); |
|
| 1461 | - | emit_u8(e, modrm(2, X_RDX, X_RAX)); |
|
| 1462 | - | emit_u32(e, offsetof(struct vector_state, vl)); |
|
| 1463 | - | emit_rexw(e); |
|
| 1464 | - | emit_u8(e, 0xc7); |
|
| 1465 | - | emit_u8(e, modrm(2, 0, X_RAX)); |
|
| 1466 | - | emit_u32(e, offsetof(struct vector_state, vtype)); |
|
| 1467 | - | emit_u32(e, type); |
|
| 1468 | - | emit_rexw(e); |
|
| 1469 | - | emit_u8(e, 0xc7); |
|
| 1470 | - | emit_u8(e, modrm(2, 0, X_RAX)); |
|
| 1471 | - | emit_u32(e, offsetof(struct vector_state, vstart)); |
|
| 1472 | - | emit_u32(e, 0); |
|
| 1477 | + | emit_mov_rr(e, X_RCX, X_RAX); |
|
| 1478 | + | emit_vector_state(e, type); |
|
| 1473 | 1479 | emit_chain_exit(e, pc + INSTR_SIZE); |
|
| 1474 | 1480 | return true; |
|
| 1475 | 1481 | } |
|
| 1476 | 1482 | ||
| 1477 | 1483 | /* Fill active bytes of a register group from a scalar or signed immediate. */ |
| 1520 | 1526 | emit_u8(e, 0x39); |
|
| 1521 | 1527 | emit_u8(e, 0xc8); |
|
| 1522 | 1528 | emit_fault_if(e, pc, 0x87); /* ja */ |
|
| 1523 | 1529 | } |
|
| 1524 | 1530 | ||
| 1531 | + | /* Decode a fixed element count from vsetivli or a preceding addi from zero |
|
| 1532 | + | * in the current straight-line run. */ |
|
| 1533 | + | static bool emit_short_vector_config(struct emitter *e, u32 ins, u32 pc) { |
|
| 1534 | + | if ((ins & 0x707f) != 0x7057) |
|
| 1535 | + | return false; |
|
| 1536 | + | bool immediate = (ins >> 30) == 3; |
|
| 1537 | + | if (!immediate && (ins >> 31)) |
|
| 1538 | + | return false; |
|
| 1539 | + | u32 type = (ins >> 20) & (immediate ? 1023 : 2047); |
|
| 1540 | + | if (type & ~0xdbu) |
|
| 1541 | + | return false; |
|
| 1542 | + | u32 src = (ins >> 15) & 31; |
|
| 1543 | + | u32 count = src; |
|
| 1544 | + | if (!immediate) { |
|
| 1545 | + | if (!src || e->run_start[e->runs - 1] == pc) |
|
| 1546 | + | return false; |
|
| 1547 | + | instr_t previous; |
|
| 1548 | + | memcpy(&previous, e->mem + pc - INSTR_SIZE, sizeof(previous)); |
|
| 1549 | + | if ((previous.raw & 0x000fffff) != (src << 7 | 0x13)) |
|
| 1550 | + | return false; |
|
| 1551 | + | count = get_i_imm(previous); |
|
| 1552 | + | } |
|
| 1553 | + | u32 lmul = type & 3; |
|
| 1554 | + | u32 width = (type >> 3) & 3; |
|
| 1555 | + | u32 max = (VECTOR_BYTES << lmul) >> width; |
|
| 1556 | + | u32 bytes = count << width; |
|
| 1557 | + | if (count == 0 || bytes > 64 || count > max || (bytes & 3) == 3) |
|
| 1558 | + | return false; |
|
| 1559 | + | u32 dst = (ins >> 7) & 31; |
|
| 1560 | + | bool changed = e->short_bytes != bytes || e->short_type != type; |
|
| 1561 | + | if (changed || dst) |
|
| 1562 | + | emit_mov_imm32_sx(e, X_RCX, count); |
|
| 1563 | + | emit_store_guest(e, dst, X_RCX); |
|
| 1564 | + | if (changed) |
|
| 1565 | + | emit_vector_state(e, type); |
|
| 1566 | + | e->short_bytes = bytes; |
|
| 1567 | + | e->short_type = type; |
|
| 1568 | + | return true; |
|
| 1569 | + | } |
|
| 1570 | + | ||
| 1571 | + | /* Transfer exactly one 16-, 8-, or 4-byte chunk through an XMM register. */ |
|
| 1572 | + | static void emit_short_chunk( |
|
| 1573 | + | struct emitter *e, bool store, bool guest, u32 offset, u32 bytes, u32 xmm |
|
| 1574 | + | ) { |
|
| 1575 | + | emit_u8(e, bytes == 16 || (bytes == 8 && !store) ? 0xf3 : 0x66); |
|
| 1576 | + | u8 opcode = store ? 0x7f : 0x6f; |
|
| 1577 | + | if (bytes == 8) |
|
| 1578 | + | opcode = store ? 0xd6 : 0x7e; |
|
| 1579 | + | if (bytes == 4) |
|
| 1580 | + | opcode = store ? 0x7e : 0x6e; |
|
| 1581 | + | if (guest) { |
|
| 1582 | + | emit_guest_memory(e, false, false, true, opcode, xmm, X_RAX, offset); |
|
| 1583 | + | return; |
|
| 1584 | + | } |
|
| 1585 | + | emit_u8(e, 0x0f); |
|
| 1586 | + | emit_u8(e, opcode); |
|
| 1587 | + | emit_u8(e, modrm(2, xmm, X_RCX)); |
|
| 1588 | + | emit_u32(e, offset); |
|
| 1589 | + | } |
|
| 1590 | + | ||
| 1591 | + | /* Move the last byte or word through the source address register. */ |
|
| 1592 | + | static void emit_short_tail( |
|
| 1593 | + | struct emitter *e, bool store, u32 offset, u32 slot, u32 bytes |
|
| 1594 | + | ) { |
|
| 1595 | + | u8 load = bytes == 2 ? 0xb7 : 0xb6; |
|
| 1596 | + | u8 write = bytes == 2 ? 0x89 : 0x88; |
|
| 1597 | + | if (store) { |
|
| 1598 | + | emit_u8(e, 0x0f); |
|
| 1599 | + | emit_u8(e, load); |
|
| 1600 | + | emit_u8(e, modrm(2, X_RCX, X_RCX)); |
|
| 1601 | + | emit_u32(e, slot); |
|
| 1602 | + | emit_guest_memory( |
|
| 1603 | + | e, bytes == 2, false, false, write, X_RCX, X_RAX, offset |
|
| 1604 | + | ); |
|
| 1605 | + | } else { |
|
| 1606 | + | emit_guest_memory(e, false, false, true, load, X_RAX, X_RAX, offset); |
|
| 1607 | + | if (bytes == 2) |
|
| 1608 | + | emit_u8(e, 0x66); |
|
| 1609 | + | emit_u8(e, write); |
|
| 1610 | + | emit_u8(e, modrm(2, X_RAX, X_RCX)); |
|
| 1611 | + | emit_u32(e, slot); |
|
| 1612 | + | } |
|
| 1613 | + | } |
|
| 1614 | + | ||
| 1615 | + | /* Transfer a known vector group between guest memory and the vector file. |
|
| 1616 | + | * Each transfer retains its fault PC and checks the full range. */ |
|
| 1617 | + | static bool emit_short_vector_memory(struct emitter *e, u32 ins, u32 pc) { |
|
| 1618 | + | u32 width = (e->short_type >> 3) & 3; |
|
| 1619 | + | u32 field = (width ? width + 4 : 0) << 12; |
|
| 1620 | + | u32 op = (ins & 0xfff0707f) ^ field; |
|
| 1621 | + | bool store = op == 0x02000027; |
|
| 1622 | + | if (!e->short_bytes || (op != 0x02000007 && !store)) |
|
| 1623 | + | return false; |
|
| 1624 | + | u32 reg = (ins >> 7) & 31; |
|
| 1625 | + | if (reg & ((1u << (e->short_type & 3)) - 1)) |
|
| 1626 | + | return false; |
|
| 1627 | + | ||
| 1628 | + | emit_load_guest(e, X_RAX, (ins >> 15) & 31); |
|
| 1629 | + | /* Hosted memory size is fixed for the lifetime of the code cache. */ |
|
| 1630 | + | u64 memory_size = e->jit->io.memory_size; |
|
| 1631 | + | if (memory_size < e->short_bytes || |
|
| 1632 | + | memory_size - e->short_bytes > INT32_MAX) { |
|
| 1633 | + | emit_vector_bound(e, e->short_bytes, pc); |
|
| 1634 | + | } else { |
|
| 1635 | + | emit_rexw(e); |
|
| 1636 | + | emit_u8(e, 0x3d); /* cmp rax, imm32 */ |
|
| 1637 | + | emit_u32(e, memory_size - e->short_bytes); |
|
| 1638 | + | emit_fault_if(e, pc, 0x87); /* ja */ |
|
| 1639 | + | } |
|
| 1640 | + | ||
| 1641 | + | bool reuse = false; |
|
| 1642 | + | if (store && !(e->short_bytes & 3) && e->run_start[e->runs - 1] != pc) { |
|
| 1643 | + | u32 previous; |
|
| 1644 | + | memcpy(&previous, e->mem + pc - INSTR_SIZE, sizeof(previous)); |
|
| 1645 | + | reuse = (previous & 0xfff07fff) == (0x02000007 | field | reg << 7); |
|
| 1646 | + | } |
|
| 1647 | + | if (!reuse) { |
|
| 1648 | + | emit_u8(e, 0x49); |
|
| 1649 | + | emit_u8(e, 0x8b); |
|
| 1650 | + | emit_u8(e, modrm(1, X_RCX, HIO)); |
|
| 1651 | + | emit_u8(e, offsetof(struct jit_io, vector)); |
|
| 1652 | + | } |
|
| 1653 | + | for (u32 offset = 0, xmm = 0; offset < e->short_bytes; xmm++) { |
|
| 1654 | + | u32 left = e->short_bytes - offset; |
|
| 1655 | + | u32 bytes = left >= 16 ? 16 : left >= 8 ? 8 : left >= 4 ? 4 : left; |
|
| 1656 | + | u32 slot = reg * VECTOR_BYTES + offset; |
|
| 1657 | + | if (bytes < 4) { |
|
| 1658 | + | emit_short_tail(e, store, offset, slot, bytes); |
|
| 1659 | + | } else { |
|
| 1660 | + | if (!reuse) |
|
| 1661 | + | emit_short_chunk( |
|
| 1662 | + | e, false, !store, store ? slot : offset, bytes, xmm |
|
| 1663 | + | ); |
|
| 1664 | + | emit_short_chunk(e, true, store, store ? offset : slot, bytes, xmm); |
|
| 1665 | + | } |
|
| 1666 | + | offset += bytes; |
|
| 1667 | + | } |
|
| 1668 | + | return true; |
|
| 1669 | + | } |
|
| 1670 | + | ||
| 1525 | 1671 | /* Transfer active bytes with complete bounds checks before any guest write. */ |
|
| 1526 | 1672 | static bool emit_group_memory(struct emitter *e, u32 ins, u32 pc) { |
|
| 1527 | 1673 | u32 op = ins & 127, reg = (ins >> 7) & 31; |
|
| 1528 | 1674 | u32 mode = (ins >> 26) & 3, index = (ins >> 20) & 31; |
|
| 1529 | 1675 | u32 field = (ins >> 12) & 7, width = field == 6 ? 2 : 0; |
| 2429 | 2575 | return true; |
|
| 2430 | 2576 | if (opcode == 0x57 && emit_byte_splat(e, ins.raw, pc)) |
|
| 2431 | 2577 | return false; |
|
| 2432 | 2578 | if ((opcode == 7 || opcode == 39) && emit_group_memory(e, ins.raw, pc)) |
|
| 2433 | 2579 | return false; |
|
| 2434 | - | if ((opcode == 7 || opcode == 39) && emit_indexed_words(e, ins.raw, pc)) |
|
| 2580 | + | if (!e->short_bytes && (opcode == 7 || opcode == 39) && |
|
| 2581 | + | emit_indexed_words(e, ins.raw, pc)) |
|
| 2435 | 2582 | return false; |
|
| 2436 | 2583 | if (opcode == 0x57 && emit_word_groups(e, ins.raw, pc)) |
|
| 2437 | 2584 | return false; |
|
| 2438 | - | if (opcode == 0x57 && emit_vector(e, ins.raw)) |
|
| 2585 | + | if (!e->short_bytes && opcode == 0x57 && emit_vector(e, ins.raw)) |
|
| 2439 | 2586 | return false; |
|
| 2440 | 2587 | ||
| 2441 | 2588 | switch (opcode) { |
|
| 2442 | 2589 | ||
| 2443 | 2590 | case OP_LUI: |
| 2800 | 2947 | } |
|
| 2801 | 2948 | ||
| 2802 | 2949 | /* Translate one instruction, keeping the held guest registers coherent |
|
| 2803 | 2950 | * around those whose emitters address the register file or clobber them. */ |
|
| 2804 | 2951 | static bool translate_insn(struct emitter *e, instr_t ins, u32 pc) { |
|
| 2952 | + | if (emit_short_vector_config(e, ins.raw, pc) || |
|
| 2953 | + | emit_short_vector_memory(e, ins.raw, pc)) |
|
| 2954 | + | return false; |
|
| 2955 | + | ||
| 2805 | 2956 | /* Consecutive such instructions share one spill and reload. */ |
|
| 2806 | 2957 | if (needs_spill(ins)) { |
|
| 2807 | 2958 | if (!e->spilled) |
|
| 2808 | 2959 | emit_spill(e); |
|
| 2809 | 2960 | return translate_body(e, ins, pc); |
jit.h
+5 -5
| 72 | 72 | }; |
|
| 73 | 73 | ||
| 74 | 74 | /* Exchange area between generated code and C. Generated code keeps a pointer |
|
| 75 | 75 | * to it in r14. */ |
|
| 76 | 76 | struct jit_io { |
|
| 77 | - | u32 pc; /* Next guest PC on exit. */ |
|
| 78 | - | i64 budget; /* Remaining guest instructions. */ |
|
| 79 | - | u8 *patch; /* Chain jump rel32 field. */ |
|
| 80 | - | struct vector_state *vector; /* Architectural vector registers. */ |
|
| 81 | - | u64 memory_size; /* Accessible hosted memory bytes. */ |
|
| 77 | + | u32 pc; /* Next guest PC on exit. */ |
|
| 78 | + | i64 budget; /* Remaining guest instructions. */ |
|
| 79 | + | u8 *patch; /* Chain jump rel32 field. */ |
|
| 80 | + | struct vector_state *vector; /* Architectural vector registers. */ |
|
| 81 | + | u64 memory_size; /* Accessible bytes, fixed while code is cached. */ |
|
| 82 | 82 | /* LR reservation: the address in the low word and the access width in |
|
| 83 | 83 | * the high word, or zero when none is held. */ |
|
| 84 | 84 | u64 reservation; |
|
| 85 | 85 | /* Bounds of the host stack frames for translated calls. */ |
|
| 86 | 86 | u64 stack_base; |
test/jit_vector.c
+47 -0
| 54 | 54 | return vector_execute(v, x, ins, transfer, mem); |
|
| 55 | 55 | } |
|
| 56 | 56 | ||
| 57 | 57 | /* Native cases must complete without interpreter fallback. */ |
|
| 58 | 58 | static bool require_native; |
|
| 59 | + | /* A short vector copy stays in the same translated block as its setup. */ |
|
| 60 | + | static bool require_single_block; |
|
| 59 | 61 | /* Byte stride supplied to strided-store checks. */ |
|
| 60 | 62 | static u64 byte_stride = 3; |
|
| 61 | 63 | ||
| 62 | 64 | /* Compare all state, memory, faults and instruction accounting for one block. |
|
| 63 | 65 | */ |
| 139 | 141 | assert(memcmp(x, y, sizeof(x)) == 0); |
|
| 140 | 142 | assert(memcmp(actual, expected, DATA) == 0); |
|
| 141 | 143 | if ((count == 8 || (count == 7 && base == 128)) && |
|
| 142 | 144 | (type & ~0xc0u) == 0x11 && vl == 4 && start == 0) |
|
| 143 | 145 | assert(jit.block_count == 1); |
|
| 146 | + | if (require_single_block) |
|
| 147 | + | assert(jit.block_count == 1); |
|
| 144 | 148 | jit_destroy(&jit); |
|
| 145 | 149 | } |
|
| 146 | 150 | ||
| 151 | + | /* Short copies with a known length keep scalar and vector state precise. */ |
|
| 152 | + | static void check_short_copy(void) { |
|
| 153 | + | u32 load = 0x02000007 | 10 << 15 | 8 << 7; |
|
| 154 | + | u32 store = 0x02000027 | 11 << 15 | 8 << 7; |
|
| 155 | + | require_native = require_single_block = true; |
|
| 156 | + | for (u32 size = 1; size <= 64; size++) { |
|
| 157 | + | require_single_block = (size & 3) != 3; |
|
| 158 | + | u32 config = 0xc0007057 | 0xc3 << 20 | size << 15; |
|
| 159 | + | u32 immediate[] = { config, load, store }; |
|
| 160 | + | if (size < 32) |
|
| 161 | + | check(immediate, 3, 0xd1, 4, 3, 0); |
|
| 162 | + | u32 reg[] = { size << 20 | 5 << 7 | 0x13, |
|
| 163 | + | 0xc3 << 20 | 5 << 15 | 0x7057, |
|
| 164 | + | load, |
|
| 165 | + | store }; |
|
| 166 | + | check(reg, 4, 0xd1, 4, 3, 0); |
|
| 167 | + | } |
|
| 168 | + | require_single_block = true; |
|
| 169 | + | for (u32 width = 1; width <= 3; width++) { |
|
| 170 | + | u32 field = (width + 4) << 12; |
|
| 171 | + | for (u32 count = 2; count < 32 && count <= (64u >> width); count += 2) { |
|
| 172 | + | u32 setup = 0xcc307057 | width << 23 | count << 15; |
|
| 173 | + | u32 copy[] = { setup, load | field, store | field }; |
|
| 174 | + | check(copy, 3, 0xd1, 4, 3, 128); |
|
| 175 | + | } |
|
| 176 | + | } |
|
| 177 | + | u32 config = 0xcc387057; |
|
| 178 | + | u32 other[] = { config, load, store + (8 << 7) }; |
|
| 179 | + | check(other, 3, 0, 0, 0, 128); |
|
| 180 | + | u32 repeated[] = { config, load, config | 12 << 7, store }; |
|
| 181 | + | check(repeated, 4, 0, 0, 0, 128); |
|
| 182 | + | require_native = require_single_block = false; |
|
| 183 | + | u32 fault[] = { config, load, store }; |
|
| 184 | + | check(fault, 3, 0, 0, 0, DATA - 8); |
|
| 185 | + | byte_stride = DATA - 8; |
|
| 186 | + | check(fault, 3, 0, 0, 0, 128); |
|
| 187 | + | u32 wide_fault[] = { 0xcd917057, load | (7 << 12), store | (7 << 12) }; |
|
| 188 | + | check(wide_fault, 3, 0, 0, 0, 128); |
|
| 189 | + | byte_stride = 3; |
|
| 190 | + | check(wide_fault, 3, 0, 0, 0, DATA - 8); |
|
| 191 | + | } |
|
| 192 | + | ||
| 147 | 193 | /* Check byte transfers, fills, configuration, and fault boundaries. */ |
|
| 148 | 194 | static void check_bulk(void) { |
|
| 149 | 195 | u32 load = 1u << 25 | 10 << 15 | 8 << 7 | 7; |
|
| 150 | 196 | u32 store = (load & ~127u) | 39; |
|
| 151 | 197 | u32 stride = store | 2u << 26 | 11 << 20; |
| 325 | 371 | jit_destroy(&jit); |
|
| 326 | 372 | } |
|
| 327 | 373 | ||
| 328 | 374 | int main(void) { |
|
| 329 | 375 | check_budget(); |
|
| 376 | + | check_short_copy(); |
|
| 330 | 377 | check_bulk(); |
|
| 331 | 378 | check_word_groups(); |
|
| 332 | 379 | u32 ops[] = { |
|
| 333 | 380 | alu(0, 0, 8, 10, 12), alu(11, 0, 14, 14, 8), alu(21, 3, 14, 14, 0), |
|
| 334 | 381 | alu(21, 3, 14, 14, 31), alu(0, 3, 8, 8, 31), alu(9, 3, 8, 8, 3), |