hash: Use vector BLAKE3 compression rounds

a1d50b0ecb0065d3eab0d316b08a9d9fd8042e26d5bf27234619d789fc7ae0f6
Run four compression columns in parallel with Zve64x and Zvbb. Keep
message schedules in read-only data and retain the streaming tree
construction, unaligned inputs, and 32-byte digest contract.

Link standard library assembly into hosted and native images. Keep
assembly inputs out of the source formatter. Compare partial-block
digests with scalar reference rounds alongside the official vectors,
streaming boundaries, and input alignment checks.

The matched 1 MiB workload falls from 38,646,576 to 16,417,837 guest
instructions.

Assisted-by: Codex:gpt-6
Change-Id: Ifb126c6569b917c8f8ec7b70ccf50a96b19fcd02
Alexis Sellier committed ago 1 parent 466ae14e
Makefile +1 -1
48 48
include kernel/Makefile
49 49
include test/Makefile
50 50
51 51
# Format the standard library and kernel sources in place. Set `FMT_SRC` to
52 52
# format other files.
53 -
FMT_SRC ?= $(STD_TEST_LIB) $(shell find kernel -name '*.rad' | sort)
53 +
FMT_SRC ?= $(filter %.rad,$(STD_TEST_LIB)) $(shell find kernel -name '*.rad' | sort)
54 54
55 55
fmt: $(RAD_BIN) $(INTEGRATION)
56 56
	@$(call integration,format,$(FMT_SRC))
57 57
58 58
seed:
kernel/Makefile +6 -2
16 16
    $(STD_TEST_LIB) std.lib.test $(RAD_BIN) kernel/Makefile
17 17
	@echo "radiance kernel tests => $@"
18 18
	@$(RADIANCE) -test $(STD_TEST) $(KERNEL) $(patsubst %,-mod %,$(KERNEL_TEST_SRC) \
19 19
	    $(KERNEL_ASM)) -entry kernel -o $@
20 20
21 +
# Combined assembly for the standard library native package.
22 +
$(BIN_DIR)/std.ras: $(filter %.ras,$(STD_LIB)) std.lib kernel/Makefile | $(BIN_DIR)
23 +
	@cat $(filter %.ras,$(STD_LIB)) > $@
24 +
21 25
# Native kernel image.
22 26
$(BIN_DIR)/kernel.ril $(BIN_DIR)/std.ril &: $(KERNEL_SRC) $(KERNEL_MANIFEST) $(STD_LIB) $(STD_MANIFEST) \
23 27
    $(RAD_BIN) kernel/Makefile
24 28
	@$(RADIANCE) $(STD) $(KERNEL) -entry kernel -ril $(BIN_DIR)
25 29
26 30
$(BIN_DIR)/kernel.build.rv64: kernel/tools/build.rad kernel/tools/build.manifest $(STD_LIB) $(STD_MANIFEST) $(RAD_BIN) \
27 -
    kernel/Makefile
31 +
    $(BIN_DIR)/std.ras kernel/Makefile
28 32
	@$(RADIANCE) $(STD) -pkg build -mod $< -entry build -o $@
29 33
30 34
$(BIN_DIR)/kernel.rv64: $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril \
31 35
    $(BIN_DIR)/kernel.build.rv64 $(KERNEL_ASM) kernel/Makefile
32 36
	@cat $(KERNEL_ASM) > $(BIN_DIR)/kernel.ras
33 -
	@$(HOSTED) $(BIN_DIR)/kernel.build.rv64 $(BIN_DIR)/std.ril \
37 +
	@$(HOSTED) $(BIN_DIR)/kernel.build.rv64 -std-asm $(BIN_DIR)/std.ras $(BIN_DIR)/std.ril \
34 38
	    $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.ras $@
kernel/tools/build.rad +37 -9
76 76
static DECODE: [u8; DECODE_CAPACITY] = [0; DECODE_CAPACITY];
77 77
/// Exact package bytes retained in the native catalog.
78 78
unsafe static INPUT: [[u8; INPUT_CAPACITY]; PACKAGES] = undefined;
79 79
/// Combined startup and boundary assembly source.
80 80
static SOURCE: [u8; SOURCE_CAPACITY] = [0; SOURCE_CAPACITY];
81 +
/// Standard library assembly source.
82 +
static LIBRARY_SOURCE: [u8; SOURCE_CAPACITY] = [0; SOURCE_CAPACITY];
83 +
/// Standard library assembly words.
84 +
static LIBRARY_TEXT: [u32; TEXT_CAPACITY] = [0; TEXT_CAPACITY];
81 85
/// Assembler workspace.
82 86
static ASSEMBLY: [u8; ASSEMBLY_CAPACITY] = [0; ASSEMBLY_CAPACITY];
83 87
/// Assembled startup words.
84 88
static TEXT: [u32; TEXT_CAPACITY] = [0; TEXT_CAPACITY];
85 89
/// Assembly identifiers retained through package linking.
240 244
    return &providers[..];
241 245
}
242 246
243 247
/// Link shared boot packages, retain their catalog, and instantiate the kernel graph.
244 248
@default unsafe fn main(env: *sys::Env) -> i32 {
245 -
    assert env.args.len >= 4 and env.args.len <= PACKAGES + 2;
246 -
    let count = env.args.len - 2;
249 +
    let withLibraryAssembly = env.args.len >= 2 and mem::eq(env.args[0], "-std-asm");
250 +
    let args = &env.args[2 ..] if withLibraryAssembly else &env.args[..];
251 +
    assert args.len >= 4 and args.len <= PACKAGES + 2;
252 +
    let count = args.len - 2;
247 253
    let decoder = heap::over(&mut DECODE[..]);
248 -
    let libraryBytesLength = try fs::readFile(env.args[0], &mut INPUT[0][..]) catch {
254 +
    let libraryBytesLength = try fs::readFile(args[0], &mut INPUT[0][..]) catch {
249 255
        panic "main: std RIL";
250 256
    };
251 257
    let libraryBytes = &INPUT[0][.. libraryBytesLength];
252 258
    let libraryInput = try! program::decode(libraryBytes, decoder, binary::Limits {
253 259
        registers: liveness::MAX_SSA_REGS, blocks: labels::MAX_BLOCKS_PER_FN
254 260
    });
255 -
    let kernelBytesLength = try fs::readFile(env.args[1], &mut INPUT[1][..]) catch {
261 +
    let kernelBytesLength = try fs::readFile(args[1], &mut INPUT[1][..]) catch {
256 262
        panic "main: kernel RIL";
257 263
    };
258 264
    let kernelBytes = &INPUT[1][.. kernelBytesLength];
259 265
    let kernelInput = try! program::decode(kernelBytes, decoder, binary::Limits {
260 266
        registers: liveness::MAX_SSA_REGS, blocks: labels::MAX_BLOCKS_PER_FN
262 268
    assert mem::eq(libraryInput.name, "std") and mem::eq(kernelInput.name, "kernel");
263 269
    assert libraryInput.dependencies.len == 0;
264 270
    for dependency in kernelInput.dependencies {
265 271
        assert mem::eq(dependency, "std");
266 272
    }
267 -
    let sourceLength = try fs::readFile(env.args[2], &mut SOURCE[..]) catch {
273 +
    let sourceLength = try fs::readFile(args[2], &mut SOURCE[..]) catch {
268 274
        panic "main: kernel assembly";
269 275
    };
270 276
    let source = &SOURCE[.. sourceLength];
271 277
    let assemblyArena = heap::over(&mut ASSEMBLY[..]);
272 278
    let empty: *mut [u8] = &mut [];
278 284
        empty,
279 285
        assemblyArena,
280 286
        &mut names,
281 287
        0
282 288
    ) catch error {
283 -
        asm::printError(error, &env.args[2][..], source);
289 +
        asm::printError(error, &args[2][..], source);
284 290
        return 1;
285 291
    };
292 +
    let mut libraryAssembly = asm::Program {
293 +
        text: &[], data: &[], symbols: &[], externalFixups: &[]
294 +
    };
295 +
    if withLibraryAssembly {
296 +
        let size = try fs::readFile(env.args[1], &mut LIBRARY_SOURCE[..]) catch {
297 +
            panic "main: standard library assembly";
298 +
        };
299 +
        let source = &LIBRARY_SOURCE[.. size];
300 +
        let empty: *mut [u8] = &mut [];
301 +
        set libraryAssembly = try asm::assemble(
302 +
            asm::scanner::SourceKind::String,
303 +
            source,
304 +
            &mut LIBRARY_TEXT[..],
305 +
            empty,
306 +
            assemblyArena,
307 +
            &mut names,
308 +
            0
309 +
        ) catch error {
310 +
            asm::printError(error, &env.args[1][..], source);
311 +
            return 1;
312 +
        };
313 +
    }
286 314
    let library = try compile(
287 315
        &libraryInput,
288 316
        0,
289 317
        CODE_ADDRESS
290 318
            + TRAMPOLINE_BYTES as u64,
291 319
        &[],
292 -
        &asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] },
320 +
        &libraryAssembly,
293 321
        rv64::StackCheck::Off
294 322
    ) catch error {
295 323
        report(error);
296 324
        return 1;
297 325
    };
311 339
    set catalogEntries[1] = catalog::Entry { source: kernelBytes, package: kernel };
312 340
    let mut codeEnd = kernel.codeAddress
313 341
        + kernel.code.len as u64 * rv64::INSTR_SIZE as u64;
314 342
    for slot in 2 .. count {
315 343
        let bytesLength = try fs::readFile(
316 -
            env.args[slot + 2], &mut INPUT[slot][..]
344 +
            args[slot + 2], &mut INPUT[slot][..]
317 345
        ) catch {
318 346
            panic "main: boot package RIL";
319 347
        };
320 348
        let bytes = &INPUT[slot][.. bytesLength];
321 349
        let mut input = try! program::decode(bytes, decoder, binary::Limits {
407 435
        code: image::Segment { address: CODE_ADDRESS, size: codeSize },
408 436
        roData: image::Segment { address: roAddress, size: roSize },
409 437
        rwData: image::Segment { address: rwAddress, size: rwSize },
410 438
        rwTail: 0,
411 439
    });
412 -
    let mut out = try! fs::create(env.args[3]);
440 +
    let mut out = try! fs::create(args[3]);
413 441
    out.write(&header[..]);
414 442
    out.write(@sliceOf(&NATIVE[0] as *u8, codeSize));
415 443
    out.write(&RO[.. roSize]);
416 444
    out.write(&RW[.. rwSize]);
417 445
    try! out.close();
lib/std/hash/blake3.rad +18 -158
20 20
/// Flag for a parent node of the hash tree.
21 21
constant PARENT: u32 = 4;
22 22
/// Flag for the root output of the hash tree.
23 23
constant ROOT: u32 = 8;
24 24
25 +
/// Byte offsets for the two column and two diagonal message vectors per round.
26 +
constant SCHEDULE: *[u32] = &[
27 +
    0, 8, 16, 24, 4, 12, 20, 28, 32, 40, 48, 56, 36, 44, 52, 60,
28 +
    8, 12, 28, 16, 24, 40, 0, 52, 4, 48, 36, 60, 44, 20, 56, 32,
29 +
    12, 40, 52, 28, 16, 48, 8, 56, 24, 36, 44, 32, 20, 0, 60, 4,
30 +
    40, 48, 56, 52, 28, 36, 12, 60, 16, 44, 20, 4, 0, 8, 32, 24,
31 +
    48, 36, 60, 56, 52, 44, 40, 32, 28, 20, 0, 24, 8, 12, 4, 16,
32 +
    36, 44, 32, 60, 56, 20, 48, 4, 52, 0, 8, 16, 12, 40, 24, 28,
33 +
    44, 20, 4, 32, 60, 0, 36, 24, 56, 8, 12, 28, 40, 48, 16, 52,
34 +
];
35 +
25 36
/// A 32-byte BLAKE3 digest.
26 37
export record Digest: Copy {
27 38
    /// Digest bytes in output order.
28 39
    bytes: [u8; DIGEST_BYTES],
29 40
}
74 85
        stack: [0; 432],
75 86
        depth: 0,
76 87
    };
77 88
}
78 89
79 -
/// Apply seven BLAKE3 rounds with wrapping word sums and three live state columns.
80 -
/// The working words and loop state fit the backend's block argument bound.
81 -
fn rounds(state: &mut [u32; 16], message: &[u32; 16]) {
82 -
    let mut v0 = state[0];
83 -
    let mut v1 = state[1];
84 -
    let mut v2 = state[2];
85 -
    let mut v3 = state[3];
86 -
    let mut v4 = state[4];
87 -
    let mut v5 = state[5];
88 -
    let mut v6 = state[6];
89 -
    let mut v7 = state[7];
90 -
    let mut v8 = state[8];
91 -
    let mut v9 = state[9];
92 -
    let mut v10 = state[10];
93 -
    let mut v11 = state[11];
94 -
    let mut m = *message;
95 -
    for round in 0 as u32 .. 7 {
96 -
        let mut v12 = state[12];
97 -
        let mut v13 = state[13];
98 -
        let mut v14 = state[14];
99 -
        let mut v15 = state[15];
100 -
        set v0 += v4 + m[0];
101 -
        set v12 ^= v0;
102 -
        set v12 = bits::rotateRight32(v12, 16);
103 -
        set v8 += v12;
104 -
        set v4 ^= v8;
105 -
        set v4 = bits::rotateRight32(v4, 12);
106 -
        set v0 += v4 + m[1];
107 -
        set v12 ^= v0;
108 -
        set v12 = bits::rotateRight32(v12, 8);
109 -
        set v8 += v12;
110 -
        set v4 ^= v8;
111 -
        set v4 = bits::rotateRight32(v4, 7);
112 -
        set v1 += v5 + m[2];
113 -
        set v13 ^= v1;
114 -
        set v13 = bits::rotateRight32(v13, 16);
115 -
        set v9 += v13;
116 -
        set v5 ^= v9;
117 -
        set v5 = bits::rotateRight32(v5, 12);
118 -
        set v1 += v5 + m[3];
119 -
        set v13 ^= v1;
120 -
        set v13 = bits::rotateRight32(v13, 8);
121 -
        set v9 += v13;
122 -
        set v5 ^= v9;
123 -
        set v5 = bits::rotateRight32(v5, 7);
124 -
        set v2 += v6 + m[4];
125 -
        set v14 ^= v2;
126 -
        set v14 = bits::rotateRight32(v14, 16);
127 -
        set v10 += v14;
128 -
        set v6 ^= v10;
129 -
        set v6 = bits::rotateRight32(v6, 12);
130 -
        set v2 += v6 + m[5];
131 -
        set v14 ^= v2;
132 -
        set v14 = bits::rotateRight32(v14, 8);
133 -
        set v10 += v14;
134 -
        set v6 ^= v10;
135 -
        set v6 = bits::rotateRight32(v6, 7);
136 -
        set v3 += v7 + m[6];
137 -
        set v15 ^= v3;
138 -
        set v15 = bits::rotateRight32(v15, 16);
139 -
        set v11 += v15;
140 -
        set v7 ^= v11;
141 -
        set v7 = bits::rotateRight32(v7, 12);
142 -
        set v3 += v7 + m[7];
143 -
        set v15 ^= v3;
144 -
        set v15 = bits::rotateRight32(v15, 8);
145 -
        set v11 += v15;
146 -
        set v7 ^= v11;
147 -
        set v7 = bits::rotateRight32(v7, 7);
148 -
        set v0 += v5 + m[8];
149 -
        set v15 ^= v0;
150 -
        set v15 = bits::rotateRight32(v15, 16);
151 -
        set v10 += v15;
152 -
        set v5 ^= v10;
153 -
        set v5 = bits::rotateRight32(v5, 12);
154 -
        set v0 += v5 + m[9];
155 -
        set v15 ^= v0;
156 -
        set v15 = bits::rotateRight32(v15, 8);
157 -
        set v10 += v15;
158 -
        set v5 ^= v10;
159 -
        set v5 = bits::rotateRight32(v5, 7);
160 -
        set v1 += v6 + m[10];
161 -
        set v12 ^= v1;
162 -
        set v12 = bits::rotateRight32(v12, 16);
163 -
        set v11 += v12;
164 -
        set v6 ^= v11;
165 -
        set v6 = bits::rotateRight32(v6, 12);
166 -
        set v1 += v6 + m[11];
167 -
        set v12 ^= v1;
168 -
        set v12 = bits::rotateRight32(v12, 8);
169 -
        set v11 += v12;
170 -
        set v6 ^= v11;
171 -
        set v6 = bits::rotateRight32(v6, 7);
172 -
        set v2 += v7 + m[12];
173 -
        set v13 ^= v2;
174 -
        set v13 = bits::rotateRight32(v13, 16);
175 -
        set v8 += v13;
176 -
        set v7 ^= v8;
177 -
        set v7 = bits::rotateRight32(v7, 12);
178 -
        set v2 += v7 + m[13];
179 -
        set v13 ^= v2;
180 -
        set v13 = bits::rotateRight32(v13, 8);
181 -
        set v8 += v13;
182 -
        set v7 ^= v8;
183 -
        set v7 = bits::rotateRight32(v7, 7);
184 -
        set v3 += v4 + m[14];
185 -
        set v14 ^= v3;
186 -
        set v14 = bits::rotateRight32(v14, 16);
187 -
        set v9 += v14;
188 -
        set v4 ^= v9;
189 -
        set v4 = bits::rotateRight32(v4, 12);
190 -
        set v3 += v4 + m[15];
191 -
        set v14 ^= v3;
192 -
        set v14 = bits::rotateRight32(v14, 8);
193 -
        set v9 += v14;
194 -
        set v4 ^= v9;
195 -
        set v4 = bits::rotateRight32(v4, 7);
196 -
        set state[12] = v12;
197 -
        set state[13] = v13;
198 -
        set state[14] = v14;
199 -
        set state[15] = v15;
200 -
        if round < 6 {
201 -
            // Permute the message words for the next round. The permutation
202 -
            // has two cycles of eight words.
203 -
            let first = m[0];
204 -
            set m[0] = m[2];
205 -
            set m[2] = m[3];
206 -
            set m[3] = m[10];
207 -
            set m[10] = m[12];
208 -
            set m[12] = m[9];
209 -
            set m[9] = m[11];
210 -
            set m[11] = m[5];
211 -
            set m[5] = first;
212 -
            let second = m[1];
213 -
            set m[1] = m[6];
214 -
            set m[6] = m[4];
215 -
            set m[4] = m[7];
216 -
            set m[7] = m[13];
217 -
            set m[13] = m[14];
218 -
            set m[14] = m[15];
219 -
            set m[15] = m[8];
220 -
            set m[8] = second;
221 -
        }
222 -
    }
223 -
    set state[0] = v0;
224 -
    set state[1] = v1;
225 -
    set state[2] = v2;
226 -
    set state[3] = v3;
227 -
    set state[4] = v4;
228 -
    set state[5] = v5;
229 -
    set state[6] = v6;
230 -
    set state[7] = v7;
231 -
    set state[8] = v8;
232 -
    set state[9] = v9;
233 -
    set state[10] = v10;
234 -
    set state[11] = v11;
235 -
}
90 +
/// Apply seven wrapping rounds to four rows of four 32-bit words.
91 +
/// The assembly routine reads 64 message bytes and updates 64 state bytes.
92 +
/// Each schedule offset must address one of the 16 message words.
93 +
unsafe fn rounds(state: &mut [u32; 16], message: &[u32; 16], schedule: *u32);
236 94
237 95
/// Compress a node and return its first eight output words.
238 96
fn compress(input: &Output) -> [u32; 8] {
239 97
    let mut state: [u32; 16] = [
240 98
        input.words[0], input.words[1], input.words[2], input.words[3],
241 99
        input.words[4], input.words[5], input.words[6], input.words[7],
242 100
        IV[0], IV[1], IV[2], IV[3], input.counter as u32,
243 101
        (input.counter >> 32) as u32, input.length, input.flags,
244 102
    ];
245 -
    rounds(&mut state, &input.block);
103 +
    unsafe {
104 +
        rounds(&mut state, &input.block, SCHEDULE.ptr);
105 +
    }
246 106
    return [
247 107
        state[0] ^ state[8], state[1] ^ state[9],
248 108
        state[2] ^ state[10], state[3] ^ state[11],
249 109
        state[4] ^ state[12], state[5] ^ state[13],
250 110
        state[6] ^ state[14], state[7] ^ state[15],
lib/std/hash/blake3.ras added +96 -0
1 +
//! Four-lane BLAKE3 rounds with Zve64x and Zvbb.
2 +
// a0 addresses 16 writable state words; a1 addresses 16 message words.
3 +
// a2 addresses 112 message offsets, four per vector load.
4 +
// m2 provides four 32-bit lanes when VLEN is at least 64 bits.
5 +
// Vector registers, vl and vtype are caller-saved.
6 +
// BLAKE3 compression rounds and mixing rotations.
7 +
.constant ROUNDS 7;
8 +
.constant ROTATE_FIRST 16;
9 +
.constant ROTATE_SECOND 12;
10 +
.constant ROTATE_THIRD 8;
11 +
.constant ROTATE_FOURTH 7;
12 +
.text;
13 +
.export @"std::hash::blake3::rounds";
14 +
@"std::hash::blake3::rounds"
15 +
    vsetivli %zero 4 e32 m2 ta ma;
16 +
    vle32.v %v8 (%a0);
17 +
    addi %t0 %a0 16;
18 +
    vle32.v %v10 (%t0);
19 +
    addi %t0 %t0 16;
20 +
    vle32.v %v12 (%t0);
21 +
    addi %t0 %t0 16;
22 +
    vle32.v %v14 (%t0);
23 +
    // Lane indices for left rotations by one, two and three words.
24 +
    vid.v %v2;
25 +
    vadd.vi %v4 %v2 2;
26 +
    vand.vi %v4 %v4 3;
27 +
    vadd.vi %v6 %v2 3;
28 +
    vand.vi %v6 %v6 3;
29 +
    vadd.vi %v2 %v2 1;
30 +
    vand.vi %v2 %v2 3;
31 +
    li %t1 ROUNDS;
32 +
@round
33 +
    // Mix columns, then align the rows for the diagonal phase.
34 +
    vle32.v %v24 (%a2);
35 +
    addi %a2 %a2 16;
36 +
    vluxei32.v %v26 (%a1) %v24;
37 +
    vadd.vv %v8 %v8 %v10;
38 +
    vadd.vv %v8 %v8 %v26;
39 +
    vxor.vv %v14 %v14 %v8;
40 +
    vror.vi %v14 %v14 ROTATE_FIRST;
41 +
    vadd.vv %v12 %v12 %v14;
42 +
    vxor.vv %v10 %v10 %v12;
43 +
    vror.vi %v10 %v10 ROTATE_SECOND;
44 +
    vle32.v %v24 (%a2);
45 +
    addi %a2 %a2 16;
46 +
    vluxei32.v %v26 (%a1) %v24;
47 +
    vadd.vv %v8 %v8 %v10;
48 +
    vadd.vv %v8 %v8 %v26;
49 +
    vxor.vv %v14 %v14 %v8;
50 +
    vror.vi %v14 %v14 ROTATE_THIRD;
51 +
    vadd.vv %v12 %v12 %v14;
52 +
    vxor.vv %v10 %v10 %v12;
53 +
    vror.vi %v10 %v10 ROTATE_FOURTH;
54 +
    vrgather.vv %v24 %v10 %v2;
55 +
    vmv.v.v %v10 %v24;
56 +
    vrgather.vv %v24 %v12 %v4;
57 +
    vmv.v.v %v12 %v24;
58 +
    vrgather.vv %v24 %v14 %v6;
59 +
    vmv.v.v %v14 %v24;
60 +
    vle32.v %v24 (%a2);
61 +
    addi %a2 %a2 16;
62 +
    vluxei32.v %v26 (%a1) %v24;
63 +
    vadd.vv %v8 %v8 %v10;
64 +
    vadd.vv %v8 %v8 %v26;
65 +
    vxor.vv %v14 %v14 %v8;
66 +
    vror.vi %v14 %v14 ROTATE_FIRST;
67 +
    vadd.vv %v12 %v12 %v14;
68 +
    vxor.vv %v10 %v10 %v12;
69 +
    vror.vi %v10 %v10 ROTATE_SECOND;
70 +
    vle32.v %v24 (%a2);
71 +
    addi %a2 %a2 16;
72 +
    vluxei32.v %v26 (%a1) %v24;
73 +
    vadd.vv %v8 %v8 %v10;
74 +
    vadd.vv %v8 %v8 %v26;
75 +
    vxor.vv %v14 %v14 %v8;
76 +
    vror.vi %v14 %v14 ROTATE_THIRD;
77 +
    vadd.vv %v12 %v12 %v14;
78 +
    vxor.vv %v10 %v10 %v12;
79 +
    vror.vi %v10 %v10 ROTATE_FOURTH;
80 +
    // Restore row order before the next column phase.
81 +
    vrgather.vv %v24 %v10 %v6;
82 +
    vmv.v.v %v10 %v24;
83 +
    vrgather.vv %v24 %v12 %v4;
84 +
    vmv.v.v %v12 %v24;
85 +
    vrgather.vv %v24 %v14 %v2;
86 +
    vmv.v.v %v14 %v24;
87 +
    addi %t1 %t1 -1;
88 +
    bnez %t1 @round;
89 +
    vse32.v %v8 (%a0);
90 +
    addi %t0 %a0 16;
91 +
    vse32.v %v10 (%t0);
92 +
    addi %t0 %t0 16;
93 +
    vse32.v %v12 (%t0);
94 +
    addi %t0 %t0 16;
95 +
    vse32.v %v14 (%t0);
96 +
    ret;
lib/std/hash/blake3/tests.rad +60 -0
1 1
//! BLAKE3 official unkeyed vectors and streaming boundaries.
2 2
3 +
use std::bits;
3 4
use std::fmt;
4 5
use std::hash::blake3;
5 6
use std::mem;
6 7
use std::testing;
7 8
202 203
            let actual = blake3::digest(&shifted[offset .. offset + length]);
203 204
            assert mem::eq(&actual.bytes[..], &expected.bytes[..]);
204 205
        }
205 206
    }
206 207
}
208 +
209 +
/// Mix one scalar column with wrapping sums and fixed rotations.
210 +
fn mix(state: &mut [u32; 16], a: u32, b: u32, c: u32, d: u32, x: u32, y: u32) {
211 +
    set state[a] += state[b] + x;
212 +
    set state[d] = bits::rotateRight32(state[d] ^ state[a], 16);
213 +
    set state[c] += state[d];
214 +
    set state[b] = bits::rotateRight32(state[b] ^ state[c], 12);
215 +
    set state[a] += state[b] + y;
216 +
    set state[d] = bits::rotateRight32(state[d] ^ state[a], 8);
217 +
    set state[c] += state[d];
218 +
    set state[b] = bits::rotateRight32(state[b] ^ state[c], 7);
219 +
}
220 +
221 +
/// Compare single-block digests with scalar rounds and every partial length.
222 +
@test fn compressionBlocks() throws (testing::TestError) {
223 +
    let permutation: [u32; 16] = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8];
224 +
    let mut seed: u32 = 0x12345678;
225 +
    for length in 0 .. 65 {
226 +
        let mut input: [u8; 64] = [0; 64];
227 +
        let mut words: [u32; 16] = [0; 16];
228 +
        for i in 0 .. length {
229 +
            set seed = seed * 1664525 + 1013904223;
230 +
            set input[i] = (seed >> 24) as u8;
231 +
        }
232 +
        for i in 0 .. 16 {
233 +
            set words[i] = mem::load32Le(&input[..], i * 4);
234 +
        }
235 +
        let mut state: [u32; 16] = [
236 +
            0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
237 +
            0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19,
238 +
            0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
239 +
            0, 0, length, 11,
240 +
        ];
241 +
        for _ in 0 .. 7 {
242 +
            for i in 0 .. 4 {
243 +
                mix(&mut state, i, i + 4, i + 8, i + 12, words[i * 2], words[i * 2 + 1]);
244 +
            }
245 +
            for i in 0 .. 4 {
246 +
                mix(
247 +
                    &mut state,
248 +
                    i,
249 +
                    (i + 1) % 4 + 4,
250 +
                    (i + 2) % 4 + 8,
251 +
                    (i + 3) % 4 + 12,
252 +
                    words[i * 2 + 8],
253 +
                    words[i * 2 + 9]
254 +
                );
255 +
            }
256 +
            let previous = words;
257 +
            for i in 0 .. 16 {
258 +
                set words[i] = previous[permutation[i]];
259 +
            }
260 +
        }
261 +
        let actual = blake3::digest(&input[.. length]);
262 +
        for i in 0 .. 8 {
263 +
            assert mem::load32Le(&actual.bytes[..], i * 4) == (state[i] ^ state[i + 8]);
264 +
        }
265 +
    }
266 +
}
std.lib +1 -0
74 74
lib/std/hash/blake3.rad
75 75
lib/std/lang/il/link.rad
76 76
lib/std/lang/il/binary/linkage.rad
77 77
lib/std/lang/il/verify/linking.rad
78 78
lib/std/arch/rv64/vector.rad
79 +
lib/std/hash/blake3.ras
test/integration/kernel.rad +5 -4
3 3
//! embeds as byte arrays.
4 4
5 5
use std::fs;
6 6
use std::mem;
7 7
use std::process;
8 -
use std::sys;
9 -
use super::harness;
10 -
use super::harness::Harness;
11 -
use super::harness::Expect;
12 8
use std::process::Outcome;
13 9
use std::process::Output;
14 10
use std::process::Target;
11 +
use std::sys;
12 +
use super::harness;
13 +
use super::harness::Expect;
14 +
use super::harness::Harness;
15 15
16 16
/// Directory of the kernel modules and assembly.
17 17
constant KERNEL_DIR: *[u8] = "kernel/kernel";
18 18
/// Module value of the kernel ABI, for the fixture packages.
19 19
export constant ABI: *[u8] = "abi=kernel/kernel/abi.rad";
77 77
    image: *[u8],
78 78
    boot: &[*[u8]],
79 79
    want: Outcome,
80 80
    expects: &[Expect]
81 81
) -> bool {
82 +
    h.add(&["-std-asm", "bin/std.ras"]);
82 83
    h.add(&[h.join(&[ril, "/std.ril"]), h.join(&[ril, "/kernel.ril"]), assembly, image]);
83 84
    h.add(boot);
84 85
    let output = Output::File(h.join(&[image, ".log"]));
85 86
    let tool = h.cmd(
86 87
        "bin/kernel.build.rv64", h.take(), Target::Inherit, harness::COMPILE_STEPS, output