hash: Use vector BLAKE3 compression rounds
a1d50b0ecb0065d3eab0d316b08a9d9fd8042e26d5bf27234619d789fc7ae0f6
Run four compression columns in parallel with Zve64x and Zvbb. Keep message schedules in read-only data and retain the streaming tree construction, unaligned inputs, and 32-byte digest contract. Link standard library assembly into hosted and native images. Keep assembly inputs out of the source formatter. Compare partial-block digests with scalar reference rounds alongside the official vectors, streaming boundaries, and input alignment checks. The matched 1 MiB workload falls from 38,646,576 to 16,417,837 guest instructions. Assisted-by: Codex:gpt-6 Change-Id: Ifb126c6569b917c8f8ec7b70ccf50a96b19fcd02
1 parent
466ae14e
Makefile
+1 -1
| 48 | 48 | include kernel/Makefile |
|
| 49 | 49 | include test/Makefile |
|
| 50 | 50 | ||
| 51 | 51 | # Format the standard library and kernel sources in place. Set `FMT_SRC` to |
|
| 52 | 52 | # format other files. |
|
| 53 | - | FMT_SRC ?= $(STD_TEST_LIB) $(shell find kernel -name '*.rad' | sort) |
|
| 53 | + | FMT_SRC ?= $(filter %.rad,$(STD_TEST_LIB)) $(shell find kernel -name '*.rad' | sort) |
|
| 54 | 54 | ||
| 55 | 55 | fmt: $(RAD_BIN) $(INTEGRATION) |
|
| 56 | 56 | @$(call integration,format,$(FMT_SRC)) |
|
| 57 | 57 | ||
| 58 | 58 | seed: |
kernel/Makefile
+6 -2
| 16 | 16 | $(STD_TEST_LIB) std.lib.test $(RAD_BIN) kernel/Makefile |
|
| 17 | 17 | @echo "radiance kernel tests => $@" |
|
| 18 | 18 | @$(RADIANCE) -test $(STD_TEST) $(KERNEL) $(patsubst %,-mod %,$(KERNEL_TEST_SRC) \ |
|
| 19 | 19 | $(KERNEL_ASM)) -entry kernel -o $@ |
|
| 20 | 20 | ||
| 21 | + | # Combined assembly for the standard library native package. |
|
| 22 | + | $(BIN_DIR)/std.ras: $(filter %.ras,$(STD_LIB)) std.lib kernel/Makefile | $(BIN_DIR) |
|
| 23 | + | @cat $(filter %.ras,$(STD_LIB)) > $@ |
|
| 24 | + | ||
| 21 | 25 | # Native kernel image. |
|
| 22 | 26 | $(BIN_DIR)/kernel.ril $(BIN_DIR)/std.ril &: $(KERNEL_SRC) $(KERNEL_MANIFEST) $(STD_LIB) $(STD_MANIFEST) \ |
|
| 23 | 27 | $(RAD_BIN) kernel/Makefile |
|
| 24 | 28 | @$(RADIANCE) $(STD) $(KERNEL) -entry kernel -ril $(BIN_DIR) |
|
| 25 | 29 | ||
| 26 | 30 | $(BIN_DIR)/kernel.build.rv64: kernel/tools/build.rad kernel/tools/build.manifest $(STD_LIB) $(STD_MANIFEST) $(RAD_BIN) \ |
|
| 27 | - | kernel/Makefile |
|
| 31 | + | $(BIN_DIR)/std.ras kernel/Makefile |
|
| 28 | 32 | @$(RADIANCE) $(STD) -pkg build -mod $< -entry build -o $@ |
|
| 29 | 33 | ||
| 30 | 34 | $(BIN_DIR)/kernel.rv64: $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril \ |
|
| 31 | 35 | $(BIN_DIR)/kernel.build.rv64 $(KERNEL_ASM) kernel/Makefile |
|
| 32 | 36 | @cat $(KERNEL_ASM) > $(BIN_DIR)/kernel.ras |
|
| 33 | - | @$(HOSTED) $(BIN_DIR)/kernel.build.rv64 $(BIN_DIR)/std.ril \ |
|
| 37 | + | @$(HOSTED) $(BIN_DIR)/kernel.build.rv64 -std-asm $(BIN_DIR)/std.ras $(BIN_DIR)/std.ril \ |
|
| 34 | 38 | $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.ras $@ |
kernel/tools/build.rad
+37 -9
| 76 | 76 | static DECODE: [u8; DECODE_CAPACITY] = [0; DECODE_CAPACITY]; |
|
| 77 | 77 | /// Exact package bytes retained in the native catalog. |
|
| 78 | 78 | unsafe static INPUT: [[u8; INPUT_CAPACITY]; PACKAGES] = undefined; |
|
| 79 | 79 | /// Combined startup and boundary assembly source. |
|
| 80 | 80 | static SOURCE: [u8; SOURCE_CAPACITY] = [0; SOURCE_CAPACITY]; |
|
| 81 | + | /// Standard library assembly source. |
|
| 82 | + | static LIBRARY_SOURCE: [u8; SOURCE_CAPACITY] = [0; SOURCE_CAPACITY]; |
|
| 83 | + | /// Standard library assembly words. |
|
| 84 | + | static LIBRARY_TEXT: [u32; TEXT_CAPACITY] = [0; TEXT_CAPACITY]; |
|
| 81 | 85 | /// Assembler workspace. |
|
| 82 | 86 | static ASSEMBLY: [u8; ASSEMBLY_CAPACITY] = [0; ASSEMBLY_CAPACITY]; |
|
| 83 | 87 | /// Assembled startup words. |
|
| 84 | 88 | static TEXT: [u32; TEXT_CAPACITY] = [0; TEXT_CAPACITY]; |
|
| 85 | 89 | /// Assembly identifiers retained through package linking. |
| 240 | 244 | return &providers[..]; |
|
| 241 | 245 | } |
|
| 242 | 246 | ||
| 243 | 247 | /// Link shared boot packages, retain their catalog, and instantiate the kernel graph. |
|
| 244 | 248 | @default unsafe fn main(env: *sys::Env) -> i32 { |
|
| 245 | - | assert env.args.len >= 4 and env.args.len <= PACKAGES + 2; |
|
| 246 | - | let count = env.args.len - 2; |
|
| 249 | + | let withLibraryAssembly = env.args.len >= 2 and mem::eq(env.args[0], "-std-asm"); |
|
| 250 | + | let args = &env.args[2 ..] if withLibraryAssembly else &env.args[..]; |
|
| 251 | + | assert args.len >= 4 and args.len <= PACKAGES + 2; |
|
| 252 | + | let count = args.len - 2; |
|
| 247 | 253 | let decoder = heap::over(&mut DECODE[..]); |
|
| 248 | - | let libraryBytesLength = try fs::readFile(env.args[0], &mut INPUT[0][..]) catch { |
|
| 254 | + | let libraryBytesLength = try fs::readFile(args[0], &mut INPUT[0][..]) catch { |
|
| 249 | 255 | panic "main: std RIL"; |
|
| 250 | 256 | }; |
|
| 251 | 257 | let libraryBytes = &INPUT[0][.. libraryBytesLength]; |
|
| 252 | 258 | let libraryInput = try! program::decode(libraryBytes, decoder, binary::Limits { |
|
| 253 | 259 | registers: liveness::MAX_SSA_REGS, blocks: labels::MAX_BLOCKS_PER_FN |
|
| 254 | 260 | }); |
|
| 255 | - | let kernelBytesLength = try fs::readFile(env.args[1], &mut INPUT[1][..]) catch { |
|
| 261 | + | let kernelBytesLength = try fs::readFile(args[1], &mut INPUT[1][..]) catch { |
|
| 256 | 262 | panic "main: kernel RIL"; |
|
| 257 | 263 | }; |
|
| 258 | 264 | let kernelBytes = &INPUT[1][.. kernelBytesLength]; |
|
| 259 | 265 | let kernelInput = try! program::decode(kernelBytes, decoder, binary::Limits { |
|
| 260 | 266 | registers: liveness::MAX_SSA_REGS, blocks: labels::MAX_BLOCKS_PER_FN |
| 262 | 268 | assert mem::eq(libraryInput.name, "std") and mem::eq(kernelInput.name, "kernel"); |
|
| 263 | 269 | assert libraryInput.dependencies.len == 0; |
|
| 264 | 270 | for dependency in kernelInput.dependencies { |
|
| 265 | 271 | assert mem::eq(dependency, "std"); |
|
| 266 | 272 | } |
|
| 267 | - | let sourceLength = try fs::readFile(env.args[2], &mut SOURCE[..]) catch { |
|
| 273 | + | let sourceLength = try fs::readFile(args[2], &mut SOURCE[..]) catch { |
|
| 268 | 274 | panic "main: kernel assembly"; |
|
| 269 | 275 | }; |
|
| 270 | 276 | let source = &SOURCE[.. sourceLength]; |
|
| 271 | 277 | let assemblyArena = heap::over(&mut ASSEMBLY[..]); |
|
| 272 | 278 | let empty: *mut [u8] = &mut []; |
| 278 | 284 | empty, |
|
| 279 | 285 | assemblyArena, |
|
| 280 | 286 | &mut names, |
|
| 281 | 287 | 0 |
|
| 282 | 288 | ) catch error { |
|
| 283 | - | asm::printError(error, &env.args[2][..], source); |
|
| 289 | + | asm::printError(error, &args[2][..], source); |
|
| 284 | 290 | return 1; |
|
| 285 | 291 | }; |
|
| 292 | + | let mut libraryAssembly = asm::Program { |
|
| 293 | + | text: &[], data: &[], symbols: &[], externalFixups: &[] |
|
| 294 | + | }; |
|
| 295 | + | if withLibraryAssembly { |
|
| 296 | + | let size = try fs::readFile(env.args[1], &mut LIBRARY_SOURCE[..]) catch { |
|
| 297 | + | panic "main: standard library assembly"; |
|
| 298 | + | }; |
|
| 299 | + | let source = &LIBRARY_SOURCE[.. size]; |
|
| 300 | + | let empty: *mut [u8] = &mut []; |
|
| 301 | + | set libraryAssembly = try asm::assemble( |
|
| 302 | + | asm::scanner::SourceKind::String, |
|
| 303 | + | source, |
|
| 304 | + | &mut LIBRARY_TEXT[..], |
|
| 305 | + | empty, |
|
| 306 | + | assemblyArena, |
|
| 307 | + | &mut names, |
|
| 308 | + | 0 |
|
| 309 | + | ) catch error { |
|
| 310 | + | asm::printError(error, &env.args[1][..], source); |
|
| 311 | + | return 1; |
|
| 312 | + | }; |
|
| 313 | + | } |
|
| 286 | 314 | let library = try compile( |
|
| 287 | 315 | &libraryInput, |
|
| 288 | 316 | 0, |
|
| 289 | 317 | CODE_ADDRESS |
|
| 290 | 318 | + TRAMPOLINE_BYTES as u64, |
|
| 291 | 319 | &[], |
|
| 292 | - | &asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }, |
|
| 320 | + | &libraryAssembly, |
|
| 293 | 321 | rv64::StackCheck::Off |
|
| 294 | 322 | ) catch error { |
|
| 295 | 323 | report(error); |
|
| 296 | 324 | return 1; |
|
| 297 | 325 | }; |
| 311 | 339 | set catalogEntries[1] = catalog::Entry { source: kernelBytes, package: kernel }; |
|
| 312 | 340 | let mut codeEnd = kernel.codeAddress |
|
| 313 | 341 | + kernel.code.len as u64 * rv64::INSTR_SIZE as u64; |
|
| 314 | 342 | for slot in 2 .. count { |
|
| 315 | 343 | let bytesLength = try fs::readFile( |
|
| 316 | - | env.args[slot + 2], &mut INPUT[slot][..] |
|
| 344 | + | args[slot + 2], &mut INPUT[slot][..] |
|
| 317 | 345 | ) catch { |
|
| 318 | 346 | panic "main: boot package RIL"; |
|
| 319 | 347 | }; |
|
| 320 | 348 | let bytes = &INPUT[slot][.. bytesLength]; |
|
| 321 | 349 | let mut input = try! program::decode(bytes, decoder, binary::Limits { |
| 407 | 435 | code: image::Segment { address: CODE_ADDRESS, size: codeSize }, |
|
| 408 | 436 | roData: image::Segment { address: roAddress, size: roSize }, |
|
| 409 | 437 | rwData: image::Segment { address: rwAddress, size: rwSize }, |
|
| 410 | 438 | rwTail: 0, |
|
| 411 | 439 | }); |
|
| 412 | - | let mut out = try! fs::create(env.args[3]); |
|
| 440 | + | let mut out = try! fs::create(args[3]); |
|
| 413 | 441 | out.write(&header[..]); |
|
| 414 | 442 | out.write(@sliceOf(&NATIVE[0] as *u8, codeSize)); |
|
| 415 | 443 | out.write(&RO[.. roSize]); |
|
| 416 | 444 | out.write(&RW[.. rwSize]); |
|
| 417 | 445 | try! out.close(); |
lib/std/hash/blake3.rad
+18 -158
| 20 | 20 | /// Flag for a parent node of the hash tree. |
|
| 21 | 21 | constant PARENT: u32 = 4; |
|
| 22 | 22 | /// Flag for the root output of the hash tree. |
|
| 23 | 23 | constant ROOT: u32 = 8; |
|
| 24 | 24 | ||
| 25 | + | /// Byte offsets for the two column and two diagonal message vectors per round. |
|
| 26 | + | constant SCHEDULE: *[u32] = &[ |
|
| 27 | + | 0, 8, 16, 24, 4, 12, 20, 28, 32, 40, 48, 56, 36, 44, 52, 60, |
|
| 28 | + | 8, 12, 28, 16, 24, 40, 0, 52, 4, 48, 36, 60, 44, 20, 56, 32, |
|
| 29 | + | 12, 40, 52, 28, 16, 48, 8, 56, 24, 36, 44, 32, 20, 0, 60, 4, |
|
| 30 | + | 40, 48, 56, 52, 28, 36, 12, 60, 16, 44, 20, 4, 0, 8, 32, 24, |
|
| 31 | + | 48, 36, 60, 56, 52, 44, 40, 32, 28, 20, 0, 24, 8, 12, 4, 16, |
|
| 32 | + | 36, 44, 32, 60, 56, 20, 48, 4, 52, 0, 8, 16, 12, 40, 24, 28, |
|
| 33 | + | 44, 20, 4, 32, 60, 0, 36, 24, 56, 8, 12, 28, 40, 48, 16, 52, |
|
| 34 | + | ]; |
|
| 35 | + | ||
| 25 | 36 | /// A 32-byte BLAKE3 digest. |
|
| 26 | 37 | export record Digest: Copy { |
|
| 27 | 38 | /// Digest bytes in output order. |
|
| 28 | 39 | bytes: [u8; DIGEST_BYTES], |
|
| 29 | 40 | } |
| 74 | 85 | stack: [0; 432], |
|
| 75 | 86 | depth: 0, |
|
| 76 | 87 | }; |
|
| 77 | 88 | } |
|
| 78 | 89 | ||
| 79 | - | /// Apply seven BLAKE3 rounds with wrapping word sums and three live state columns. |
|
| 80 | - | /// The working words and loop state fit the backend's block argument bound. |
|
| 81 | - | fn rounds(state: &mut [u32; 16], message: &[u32; 16]) { |
|
| 82 | - | let mut v0 = state[0]; |
|
| 83 | - | let mut v1 = state[1]; |
|
| 84 | - | let mut v2 = state[2]; |
|
| 85 | - | let mut v3 = state[3]; |
|
| 86 | - | let mut v4 = state[4]; |
|
| 87 | - | let mut v5 = state[5]; |
|
| 88 | - | let mut v6 = state[6]; |
|
| 89 | - | let mut v7 = state[7]; |
|
| 90 | - | let mut v8 = state[8]; |
|
| 91 | - | let mut v9 = state[9]; |
|
| 92 | - | let mut v10 = state[10]; |
|
| 93 | - | let mut v11 = state[11]; |
|
| 94 | - | let mut m = *message; |
|
| 95 | - | for round in 0 as u32 .. 7 { |
|
| 96 | - | let mut v12 = state[12]; |
|
| 97 | - | let mut v13 = state[13]; |
|
| 98 | - | let mut v14 = state[14]; |
|
| 99 | - | let mut v15 = state[15]; |
|
| 100 | - | set v0 += v4 + m[0]; |
|
| 101 | - | set v12 ^= v0; |
|
| 102 | - | set v12 = bits::rotateRight32(v12, 16); |
|
| 103 | - | set v8 += v12; |
|
| 104 | - | set v4 ^= v8; |
|
| 105 | - | set v4 = bits::rotateRight32(v4, 12); |
|
| 106 | - | set v0 += v4 + m[1]; |
|
| 107 | - | set v12 ^= v0; |
|
| 108 | - | set v12 = bits::rotateRight32(v12, 8); |
|
| 109 | - | set v8 += v12; |
|
| 110 | - | set v4 ^= v8; |
|
| 111 | - | set v4 = bits::rotateRight32(v4, 7); |
|
| 112 | - | set v1 += v5 + m[2]; |
|
| 113 | - | set v13 ^= v1; |
|
| 114 | - | set v13 = bits::rotateRight32(v13, 16); |
|
| 115 | - | set v9 += v13; |
|
| 116 | - | set v5 ^= v9; |
|
| 117 | - | set v5 = bits::rotateRight32(v5, 12); |
|
| 118 | - | set v1 += v5 + m[3]; |
|
| 119 | - | set v13 ^= v1; |
|
| 120 | - | set v13 = bits::rotateRight32(v13, 8); |
|
| 121 | - | set v9 += v13; |
|
| 122 | - | set v5 ^= v9; |
|
| 123 | - | set v5 = bits::rotateRight32(v5, 7); |
|
| 124 | - | set v2 += v6 + m[4]; |
|
| 125 | - | set v14 ^= v2; |
|
| 126 | - | set v14 = bits::rotateRight32(v14, 16); |
|
| 127 | - | set v10 += v14; |
|
| 128 | - | set v6 ^= v10; |
|
| 129 | - | set v6 = bits::rotateRight32(v6, 12); |
|
| 130 | - | set v2 += v6 + m[5]; |
|
| 131 | - | set v14 ^= v2; |
|
| 132 | - | set v14 = bits::rotateRight32(v14, 8); |
|
| 133 | - | set v10 += v14; |
|
| 134 | - | set v6 ^= v10; |
|
| 135 | - | set v6 = bits::rotateRight32(v6, 7); |
|
| 136 | - | set v3 += v7 + m[6]; |
|
| 137 | - | set v15 ^= v3; |
|
| 138 | - | set v15 = bits::rotateRight32(v15, 16); |
|
| 139 | - | set v11 += v15; |
|
| 140 | - | set v7 ^= v11; |
|
| 141 | - | set v7 = bits::rotateRight32(v7, 12); |
|
| 142 | - | set v3 += v7 + m[7]; |
|
| 143 | - | set v15 ^= v3; |
|
| 144 | - | set v15 = bits::rotateRight32(v15, 8); |
|
| 145 | - | set v11 += v15; |
|
| 146 | - | set v7 ^= v11; |
|
| 147 | - | set v7 = bits::rotateRight32(v7, 7); |
|
| 148 | - | set v0 += v5 + m[8]; |
|
| 149 | - | set v15 ^= v0; |
|
| 150 | - | set v15 = bits::rotateRight32(v15, 16); |
|
| 151 | - | set v10 += v15; |
|
| 152 | - | set v5 ^= v10; |
|
| 153 | - | set v5 = bits::rotateRight32(v5, 12); |
|
| 154 | - | set v0 += v5 + m[9]; |
|
| 155 | - | set v15 ^= v0; |
|
| 156 | - | set v15 = bits::rotateRight32(v15, 8); |
|
| 157 | - | set v10 += v15; |
|
| 158 | - | set v5 ^= v10; |
|
| 159 | - | set v5 = bits::rotateRight32(v5, 7); |
|
| 160 | - | set v1 += v6 + m[10]; |
|
| 161 | - | set v12 ^= v1; |
|
| 162 | - | set v12 = bits::rotateRight32(v12, 16); |
|
| 163 | - | set v11 += v12; |
|
| 164 | - | set v6 ^= v11; |
|
| 165 | - | set v6 = bits::rotateRight32(v6, 12); |
|
| 166 | - | set v1 += v6 + m[11]; |
|
| 167 | - | set v12 ^= v1; |
|
| 168 | - | set v12 = bits::rotateRight32(v12, 8); |
|
| 169 | - | set v11 += v12; |
|
| 170 | - | set v6 ^= v11; |
|
| 171 | - | set v6 = bits::rotateRight32(v6, 7); |
|
| 172 | - | set v2 += v7 + m[12]; |
|
| 173 | - | set v13 ^= v2; |
|
| 174 | - | set v13 = bits::rotateRight32(v13, 16); |
|
| 175 | - | set v8 += v13; |
|
| 176 | - | set v7 ^= v8; |
|
| 177 | - | set v7 = bits::rotateRight32(v7, 12); |
|
| 178 | - | set v2 += v7 + m[13]; |
|
| 179 | - | set v13 ^= v2; |
|
| 180 | - | set v13 = bits::rotateRight32(v13, 8); |
|
| 181 | - | set v8 += v13; |
|
| 182 | - | set v7 ^= v8; |
|
| 183 | - | set v7 = bits::rotateRight32(v7, 7); |
|
| 184 | - | set v3 += v4 + m[14]; |
|
| 185 | - | set v14 ^= v3; |
|
| 186 | - | set v14 = bits::rotateRight32(v14, 16); |
|
| 187 | - | set v9 += v14; |
|
| 188 | - | set v4 ^= v9; |
|
| 189 | - | set v4 = bits::rotateRight32(v4, 12); |
|
| 190 | - | set v3 += v4 + m[15]; |
|
| 191 | - | set v14 ^= v3; |
|
| 192 | - | set v14 = bits::rotateRight32(v14, 8); |
|
| 193 | - | set v9 += v14; |
|
| 194 | - | set v4 ^= v9; |
|
| 195 | - | set v4 = bits::rotateRight32(v4, 7); |
|
| 196 | - | set state[12] = v12; |
|
| 197 | - | set state[13] = v13; |
|
| 198 | - | set state[14] = v14; |
|
| 199 | - | set state[15] = v15; |
|
| 200 | - | if round < 6 { |
|
| 201 | - | // Permute the message words for the next round. The permutation |
|
| 202 | - | // has two cycles of eight words. |
|
| 203 | - | let first = m[0]; |
|
| 204 | - | set m[0] = m[2]; |
|
| 205 | - | set m[2] = m[3]; |
|
| 206 | - | set m[3] = m[10]; |
|
| 207 | - | set m[10] = m[12]; |
|
| 208 | - | set m[12] = m[9]; |
|
| 209 | - | set m[9] = m[11]; |
|
| 210 | - | set m[11] = m[5]; |
|
| 211 | - | set m[5] = first; |
|
| 212 | - | let second = m[1]; |
|
| 213 | - | set m[1] = m[6]; |
|
| 214 | - | set m[6] = m[4]; |
|
| 215 | - | set m[4] = m[7]; |
|
| 216 | - | set m[7] = m[13]; |
|
| 217 | - | set m[13] = m[14]; |
|
| 218 | - | set m[14] = m[15]; |
|
| 219 | - | set m[15] = m[8]; |
|
| 220 | - | set m[8] = second; |
|
| 221 | - | } |
|
| 222 | - | } |
|
| 223 | - | set state[0] = v0; |
|
| 224 | - | set state[1] = v1; |
|
| 225 | - | set state[2] = v2; |
|
| 226 | - | set state[3] = v3; |
|
| 227 | - | set state[4] = v4; |
|
| 228 | - | set state[5] = v5; |
|
| 229 | - | set state[6] = v6; |
|
| 230 | - | set state[7] = v7; |
|
| 231 | - | set state[8] = v8; |
|
| 232 | - | set state[9] = v9; |
|
| 233 | - | set state[10] = v10; |
|
| 234 | - | set state[11] = v11; |
|
| 235 | - | } |
|
| 90 | + | /// Apply seven wrapping rounds to four rows of four 32-bit words. |
|
| 91 | + | /// The assembly routine reads 64 message bytes and updates 64 state bytes. |
|
| 92 | + | /// Each schedule offset must address one of the 16 message words. |
|
| 93 | + | unsafe fn rounds(state: &mut [u32; 16], message: &[u32; 16], schedule: *u32); |
|
| 236 | 94 | ||
| 237 | 95 | /// Compress a node and return its first eight output words. |
|
| 238 | 96 | fn compress(input: &Output) -> [u32; 8] { |
|
| 239 | 97 | let mut state: [u32; 16] = [ |
|
| 240 | 98 | input.words[0], input.words[1], input.words[2], input.words[3], |
|
| 241 | 99 | input.words[4], input.words[5], input.words[6], input.words[7], |
|
| 242 | 100 | IV[0], IV[1], IV[2], IV[3], input.counter as u32, |
|
| 243 | 101 | (input.counter >> 32) as u32, input.length, input.flags, |
|
| 244 | 102 | ]; |
|
| 245 | - | rounds(&mut state, &input.block); |
|
| 103 | + | unsafe { |
|
| 104 | + | rounds(&mut state, &input.block, SCHEDULE.ptr); |
|
| 105 | + | } |
|
| 246 | 106 | return [ |
|
| 247 | 107 | state[0] ^ state[8], state[1] ^ state[9], |
|
| 248 | 108 | state[2] ^ state[10], state[3] ^ state[11], |
|
| 249 | 109 | state[4] ^ state[12], state[5] ^ state[13], |
|
| 250 | 110 | state[6] ^ state[14], state[7] ^ state[15], |
lib/std/hash/blake3.ras
added
+96 -0
| 1 | + | //! Four-lane BLAKE3 rounds with Zve64x and Zvbb. |
|
| 2 | + | // a0 addresses 16 writable state words; a1 addresses 16 message words. |
|
| 3 | + | // a2 addresses 112 message offsets, four per vector load. |
|
| 4 | + | // m2 provides four 32-bit lanes when VLEN is at least 64 bits. |
|
| 5 | + | // Vector registers, vl and vtype are caller-saved. |
|
| 6 | + | // BLAKE3 compression rounds and mixing rotations. |
|
| 7 | + | .constant ROUNDS 7; |
|
| 8 | + | .constant ROTATE_FIRST 16; |
|
| 9 | + | .constant ROTATE_SECOND 12; |
|
| 10 | + | .constant ROTATE_THIRD 8; |
|
| 11 | + | .constant ROTATE_FOURTH 7; |
|
| 12 | + | .text; |
|
| 13 | + | .export @"std::hash::blake3::rounds"; |
|
| 14 | + | @"std::hash::blake3::rounds" |
|
| 15 | + | vsetivli %zero 4 e32 m2 ta ma; |
|
| 16 | + | vle32.v %v8 (%a0); |
|
| 17 | + | addi %t0 %a0 16; |
|
| 18 | + | vle32.v %v10 (%t0); |
|
| 19 | + | addi %t0 %t0 16; |
|
| 20 | + | vle32.v %v12 (%t0); |
|
| 21 | + | addi %t0 %t0 16; |
|
| 22 | + | vle32.v %v14 (%t0); |
|
| 23 | + | // Lane indices for left rotations by one, two and three words. |
|
| 24 | + | vid.v %v2; |
|
| 25 | + | vadd.vi %v4 %v2 2; |
|
| 26 | + | vand.vi %v4 %v4 3; |
|
| 27 | + | vadd.vi %v6 %v2 3; |
|
| 28 | + | vand.vi %v6 %v6 3; |
|
| 29 | + | vadd.vi %v2 %v2 1; |
|
| 30 | + | vand.vi %v2 %v2 3; |
|
| 31 | + | li %t1 ROUNDS; |
|
| 32 | + | @round |
|
| 33 | + | // Mix columns, then align the rows for the diagonal phase. |
|
| 34 | + | vle32.v %v24 (%a2); |
|
| 35 | + | addi %a2 %a2 16; |
|
| 36 | + | vluxei32.v %v26 (%a1) %v24; |
|
| 37 | + | vadd.vv %v8 %v8 %v10; |
|
| 38 | + | vadd.vv %v8 %v8 %v26; |
|
| 39 | + | vxor.vv %v14 %v14 %v8; |
|
| 40 | + | vror.vi %v14 %v14 ROTATE_FIRST; |
|
| 41 | + | vadd.vv %v12 %v12 %v14; |
|
| 42 | + | vxor.vv %v10 %v10 %v12; |
|
| 43 | + | vror.vi %v10 %v10 ROTATE_SECOND; |
|
| 44 | + | vle32.v %v24 (%a2); |
|
| 45 | + | addi %a2 %a2 16; |
|
| 46 | + | vluxei32.v %v26 (%a1) %v24; |
|
| 47 | + | vadd.vv %v8 %v8 %v10; |
|
| 48 | + | vadd.vv %v8 %v8 %v26; |
|
| 49 | + | vxor.vv %v14 %v14 %v8; |
|
| 50 | + | vror.vi %v14 %v14 ROTATE_THIRD; |
|
| 51 | + | vadd.vv %v12 %v12 %v14; |
|
| 52 | + | vxor.vv %v10 %v10 %v12; |
|
| 53 | + | vror.vi %v10 %v10 ROTATE_FOURTH; |
|
| 54 | + | vrgather.vv %v24 %v10 %v2; |
|
| 55 | + | vmv.v.v %v10 %v24; |
|
| 56 | + | vrgather.vv %v24 %v12 %v4; |
|
| 57 | + | vmv.v.v %v12 %v24; |
|
| 58 | + | vrgather.vv %v24 %v14 %v6; |
|
| 59 | + | vmv.v.v %v14 %v24; |
|
| 60 | + | vle32.v %v24 (%a2); |
|
| 61 | + | addi %a2 %a2 16; |
|
| 62 | + | vluxei32.v %v26 (%a1) %v24; |
|
| 63 | + | vadd.vv %v8 %v8 %v10; |
|
| 64 | + | vadd.vv %v8 %v8 %v26; |
|
| 65 | + | vxor.vv %v14 %v14 %v8; |
|
| 66 | + | vror.vi %v14 %v14 ROTATE_FIRST; |
|
| 67 | + | vadd.vv %v12 %v12 %v14; |
|
| 68 | + | vxor.vv %v10 %v10 %v12; |
|
| 69 | + | vror.vi %v10 %v10 ROTATE_SECOND; |
|
| 70 | + | vle32.v %v24 (%a2); |
|
| 71 | + | addi %a2 %a2 16; |
|
| 72 | + | vluxei32.v %v26 (%a1) %v24; |
|
| 73 | + | vadd.vv %v8 %v8 %v10; |
|
| 74 | + | vadd.vv %v8 %v8 %v26; |
|
| 75 | + | vxor.vv %v14 %v14 %v8; |
|
| 76 | + | vror.vi %v14 %v14 ROTATE_THIRD; |
|
| 77 | + | vadd.vv %v12 %v12 %v14; |
|
| 78 | + | vxor.vv %v10 %v10 %v12; |
|
| 79 | + | vror.vi %v10 %v10 ROTATE_FOURTH; |
|
| 80 | + | // Restore row order before the next column phase. |
|
| 81 | + | vrgather.vv %v24 %v10 %v6; |
|
| 82 | + | vmv.v.v %v10 %v24; |
|
| 83 | + | vrgather.vv %v24 %v12 %v4; |
|
| 84 | + | vmv.v.v %v12 %v24; |
|
| 85 | + | vrgather.vv %v24 %v14 %v2; |
|
| 86 | + | vmv.v.v %v14 %v24; |
|
| 87 | + | addi %t1 %t1 -1; |
|
| 88 | + | bnez %t1 @round; |
|
| 89 | + | vse32.v %v8 (%a0); |
|
| 90 | + | addi %t0 %a0 16; |
|
| 91 | + | vse32.v %v10 (%t0); |
|
| 92 | + | addi %t0 %t0 16; |
|
| 93 | + | vse32.v %v12 (%t0); |
|
| 94 | + | addi %t0 %t0 16; |
|
| 95 | + | vse32.v %v14 (%t0); |
|
| 96 | + | ret; |
lib/std/hash/blake3/tests.rad
+60 -0
| 1 | 1 | //! BLAKE3 official unkeyed vectors and streaming boundaries. |
|
| 2 | 2 | ||
| 3 | + | use std::bits; |
|
| 3 | 4 | use std::fmt; |
|
| 4 | 5 | use std::hash::blake3; |
|
| 5 | 6 | use std::mem; |
|
| 6 | 7 | use std::testing; |
|
| 7 | 8 |
| 202 | 203 | let actual = blake3::digest(&shifted[offset .. offset + length]); |
|
| 203 | 204 | assert mem::eq(&actual.bytes[..], &expected.bytes[..]); |
|
| 204 | 205 | } |
|
| 205 | 206 | } |
|
| 206 | 207 | } |
|
| 208 | + | ||
| 209 | + | /// Mix one scalar column with wrapping sums and fixed rotations. |
|
| 210 | + | fn mix(state: &mut [u32; 16], a: u32, b: u32, c: u32, d: u32, x: u32, y: u32) { |
|
| 211 | + | set state[a] += state[b] + x; |
|
| 212 | + | set state[d] = bits::rotateRight32(state[d] ^ state[a], 16); |
|
| 213 | + | set state[c] += state[d]; |
|
| 214 | + | set state[b] = bits::rotateRight32(state[b] ^ state[c], 12); |
|
| 215 | + | set state[a] += state[b] + y; |
|
| 216 | + | set state[d] = bits::rotateRight32(state[d] ^ state[a], 8); |
|
| 217 | + | set state[c] += state[d]; |
|
| 218 | + | set state[b] = bits::rotateRight32(state[b] ^ state[c], 7); |
|
| 219 | + | } |
|
| 220 | + | ||
| 221 | + | /// Compare single-block digests with scalar rounds and every partial length. |
|
| 222 | + | @test fn compressionBlocks() throws (testing::TestError) { |
|
| 223 | + | let permutation: [u32; 16] = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8]; |
|
| 224 | + | let mut seed: u32 = 0x12345678; |
|
| 225 | + | for length in 0 .. 65 { |
|
| 226 | + | let mut input: [u8; 64] = [0; 64]; |
|
| 227 | + | let mut words: [u32; 16] = [0; 16]; |
|
| 228 | + | for i in 0 .. length { |
|
| 229 | + | set seed = seed * 1664525 + 1013904223; |
|
| 230 | + | set input[i] = (seed >> 24) as u8; |
|
| 231 | + | } |
|
| 232 | + | for i in 0 .. 16 { |
|
| 233 | + | set words[i] = mem::load32Le(&input[..], i * 4); |
|
| 234 | + | } |
|
| 235 | + | let mut state: [u32; 16] = [ |
|
| 236 | + | 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, |
|
| 237 | + | 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, |
|
| 238 | + | 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, |
|
| 239 | + | 0, 0, length, 11, |
|
| 240 | + | ]; |
|
| 241 | + | for _ in 0 .. 7 { |
|
| 242 | + | for i in 0 .. 4 { |
|
| 243 | + | mix(&mut state, i, i + 4, i + 8, i + 12, words[i * 2], words[i * 2 + 1]); |
|
| 244 | + | } |
|
| 245 | + | for i in 0 .. 4 { |
|
| 246 | + | mix( |
|
| 247 | + | &mut state, |
|
| 248 | + | i, |
|
| 249 | + | (i + 1) % 4 + 4, |
|
| 250 | + | (i + 2) % 4 + 8, |
|
| 251 | + | (i + 3) % 4 + 12, |
|
| 252 | + | words[i * 2 + 8], |
|
| 253 | + | words[i * 2 + 9] |
|
| 254 | + | ); |
|
| 255 | + | } |
|
| 256 | + | let previous = words; |
|
| 257 | + | for i in 0 .. 16 { |
|
| 258 | + | set words[i] = previous[permutation[i]]; |
|
| 259 | + | } |
|
| 260 | + | } |
|
| 261 | + | let actual = blake3::digest(&input[.. length]); |
|
| 262 | + | for i in 0 .. 8 { |
|
| 263 | + | assert mem::load32Le(&actual.bytes[..], i * 4) == (state[i] ^ state[i + 8]); |
|
| 264 | + | } |
|
| 265 | + | } |
|
| 266 | + | } |
std.lib
+1 -0
| 74 | 74 | lib/std/hash/blake3.rad |
|
| 75 | 75 | lib/std/lang/il/link.rad |
|
| 76 | 76 | lib/std/lang/il/binary/linkage.rad |
|
| 77 | 77 | lib/std/lang/il/verify/linking.rad |
|
| 78 | 78 | lib/std/arch/rv64/vector.rad |
|
| 79 | + | lib/std/hash/blake3.ras |
test/integration/kernel.rad
+5 -4
| 3 | 3 | //! embeds as byte arrays. |
|
| 4 | 4 | ||
| 5 | 5 | use std::fs; |
|
| 6 | 6 | use std::mem; |
|
| 7 | 7 | use std::process; |
|
| 8 | - | use std::sys; |
|
| 9 | - | use super::harness; |
|
| 10 | - | use super::harness::Harness; |
|
| 11 | - | use super::harness::Expect; |
|
| 12 | 8 | use std::process::Outcome; |
|
| 13 | 9 | use std::process::Output; |
|
| 14 | 10 | use std::process::Target; |
|
| 11 | + | use std::sys; |
|
| 12 | + | use super::harness; |
|
| 13 | + | use super::harness::Expect; |
|
| 14 | + | use super::harness::Harness; |
|
| 15 | 15 | ||
| 16 | 16 | /// Directory of the kernel modules and assembly. |
|
| 17 | 17 | constant KERNEL_DIR: *[u8] = "kernel/kernel"; |
|
| 18 | 18 | /// Module value of the kernel ABI, for the fixture packages. |
|
| 19 | 19 | export constant ABI: *[u8] = "abi=kernel/kernel/abi.rad"; |
| 77 | 77 | image: *[u8], |
|
| 78 | 78 | boot: &[*[u8]], |
|
| 79 | 79 | want: Outcome, |
|
| 80 | 80 | expects: &[Expect] |
|
| 81 | 81 | ) -> bool { |
|
| 82 | + | h.add(&["-std-asm", "bin/std.ras"]); |
|
| 82 | 83 | h.add(&[h.join(&[ril, "/std.ril"]), h.join(&[ril, "/kernel.ril"]), assembly, image]); |
|
| 83 | 84 | h.add(boot); |
|
| 84 | 85 | let output = Output::File(h.join(&[image, ".log"])); |
|
| 85 | 86 | let tool = h.cmd( |
|
| 86 | 87 | "bin/kernel.build.rv64", h.take(), Target::Inherit, harness::COMPILE_STEPS, output |