cache: Keep existing entries instead of repairing
c51f0fc0f9916b91f7db85d7995dd1ac2c4ec351c739ab555b9f3863a476610e
Publishing a cache entry renames a private directory over the entry. When an entry already existed, publication probed it, kept it if it was valid, and otherwise moved the damaged tree aside and retried the rename, which needed a bounded retry loop and a quarantine directory. Atomic publication only leaves damaged entries after outside changes. Keep any existing entry and discard the new one. A valid entry was published by a concurrent writer of the same request; a damaged one stays a miss until `radiance clean` removes it. Cleaning now removes every name in the cache root that starts with a key. Assisted-by: Claude:opus-5.5 Change-Id: I25495f335cdfc7e9db8cb20cf9e67f4762209c57
1 parent
a6c5cbd1
compiler/radiance/build.rad
+1 -3
| 792 | 792 | }; |
|
| 793 | 793 | resetFrontend(ctx, graph, work.main); |
|
| 794 | 794 | if let root = ctx.cacheDirectory { |
|
| 795 | 795 | // Function storage has no live IL after its sink consumed each function. |
|
| 796 | 796 | let cacheKey = request else panic "buildPackages: missing cache key"; |
|
| 797 | - | try cache::publish( |
|
| 798 | - | root, cacheKey, &built, work.main, work.functions |
|
| 799 | - | ) catch { |
|
| 797 | + | try cache::publish(root, cacheKey, &built, work.functions) catch { |
|
| 800 | 798 | throw super::error(&["cannot publish package cache", built.name]); |
|
| 801 | 799 | }; |
|
| 802 | 800 | unsafe { |
|
| 803 | 801 | heap::reset(work.functions); |
|
| 804 | 802 | } |
compiler/radiance/cache.rad
+41 -92
| 54 | 54 | } |
|
| 55 | 55 | ||
| 56 | 56 | /// Reserve a private directory with an atomic creation operation. |
|
| 57 | 57 | fn temporary(base: *[u8], arena: &heap::Heap) -> *[u8] throws (sys::Error) { |
|
| 58 | 58 | for i in 0 .. TEMPORARY_SLOTS { |
|
| 59 | - | let mut path: *[u8] = ""; |
|
| 60 | - | { |
|
| 61 | - | let mut digits: [u8; fmt::U32_STR_LEN] = undefined; |
|
| 62 | - | let text = fmt::formatU32(i, &mut digits[..]); |
|
| 63 | - | let owned = try alloc::copyBytes(arena, text) catch { |
|
| 64 | - | throw sys::Error::TooLarge; |
|
| 65 | - | }; |
|
| 66 | - | set path = try artifact::join(arena, &[base, TEMPORARY_MARKER, &owned[..]]); |
|
| 67 | - | } |
|
| 59 | + | let mut digits: [u8; fmt::U32_STR_LEN] = undefined; |
|
| 60 | + | let number = try alloc::copyBytes(arena, fmt::formatU32(i, &mut digits[..])) catch { |
|
| 61 | + | throw sys::Error::TooLarge; |
|
| 62 | + | }; |
|
| 63 | + | let path = try artifact::join(arena, &[base, TEMPORARY_MARKER, &number[..]]); |
|
| 68 | 64 | try fs::makeDir(path) catch error { |
|
| 69 | 65 | if error == sys::Error::Exists { |
|
| 70 | 66 | continue; |
|
| 71 | 67 | } |
|
| 72 | 68 | throw error; |
| 74 | 70 | return path; |
|
| 75 | 71 | } |
|
| 76 | 72 | throw sys::Error::Limit; |
|
| 77 | 73 | } |
|
| 78 | 74 | ||
| 75 | + | /// Test for a lowercase request key or a reservation made by `temporary`. |
|
| 76 | + | fn owned(name: *[u8]) -> bool { |
|
| 77 | + | if name.len < KEY_LENGTH { |
|
| 78 | + | return false; |
|
| 79 | + | } |
|
| 80 | + | for byte in &name[.. KEY_LENGTH] { |
|
| 81 | + | if not (char::isDigit(byte) or (byte >= 'a' and byte <= 'f')) { |
|
| 82 | + | return false; |
|
| 83 | + | } |
|
| 84 | + | } |
|
| 85 | + | let suffix = &name[KEY_LENGTH ..]; |
|
| 86 | + | if suffix.len == 0 { |
|
| 87 | + | return true; |
|
| 88 | + | } |
|
| 89 | + | if not mem::startsWith(suffix, TEMPORARY_MARKER) { |
|
| 90 | + | return false; |
|
| 91 | + | } |
|
| 92 | + | let digits = &suffix[TEMPORARY_MARKER.len ..]; |
|
| 93 | + | if digits.len > 1 and digits[0] == '0' { |
|
| 94 | + | return false; |
|
| 95 | + | } |
|
| 96 | + | let slot = try fmt::parseUint(digits, 10) catch { |
|
| 97 | + | return false; |
|
| 98 | + | }; |
|
| 99 | + | return slot < TEMPORARY_SLOTS as u64; |
|
| 100 | + | } |
|
| 101 | + | ||
| 79 | 102 | /// Remove completed and temporary cache entries below `root`. |
|
| 80 | 103 | /// Missing caches are empty. Other files in the cache root are retained. |
|
| 81 | 104 | export fn clean(root: *[u8], arena: &heap::Heap) throws (sys::Error) { |
|
| 82 | 105 | let info = try fs::stat(root) else return; |
|
| 83 | 106 | if info.kind <> fs::Kind::Directory { |
|
| 84 | 107 | throw sys::Error::Invalid; |
|
| 85 | 108 | } |
|
| 86 | 109 | let listing = try artifact::list(root, arena); |
|
| 87 | 110 | let mut names = mem::split(listing, 0); |
|
| 88 | 111 | while let name = names.next() { |
|
| 89 | - | if name.len < KEY_LENGTH { |
|
| 90 | - | continue; |
|
| 91 | - | } |
|
| 92 | - | let mut owned = true; |
|
| 93 | - | for byte in &name[.. KEY_LENGTH] { |
|
| 94 | - | set owned = owned and char::isHexDigit(byte); |
|
| 95 | - | } |
|
| 96 | - | if name.len > KEY_LENGTH { |
|
| 97 | - | let suffix = &name[KEY_LENGTH ..]; |
|
| 98 | - | if not mem::startsWith(suffix, TEMPORARY_MARKER) |
|
| 99 | - | or suffix.len == TEMPORARY_MARKER.len |
|
| 100 | - | { |
|
| 101 | - | continue; |
|
| 102 | - | } |
|
| 103 | - | for byte in &suffix[TEMPORARY_MARKER.len ..] { |
|
| 104 | - | set owned = owned and char::isDigit(byte); |
|
| 105 | - | } |
|
| 106 | - | } |
|
| 107 | - | if not owned { |
|
| 112 | + | if not owned(name) { |
|
| 108 | 113 | continue; |
|
| 109 | 114 | } |
|
| 110 | 115 | let saved = heap::mark(arena); |
|
| 111 | 116 | try fs::removeTree(try artifact::join(arena, &[root, "/", name]), arena); |
|
| 112 | 117 | // Entry paths and recursive listings have no users after removal. |
| 115 | 120 | } |
|
| 116 | 121 | } |
|
| 117 | 122 | } |
|
| 118 | 123 | ||
| 119 | 124 | /// Write a private complete set for request `key`, then make it visible with one |
|
| 120 | - | /// directory rename. |
|
| 121 | - | export fn publish( |
|
| 122 | - | root: *[u8], |
|
| 123 | - | key: binary::Id, |
|
| 124 | - | package: &artifact::Package, |
|
| 125 | - | scratch: *heap::Heap, |
|
| 126 | - | arena: &heap::Heap |
|
| 127 | - | ) |
|
| 125 | + | /// directory rename. An existing entry is kept: either a concurrent writer |
|
| 126 | + | /// published the same request, or the entry is damaged and stays a miss until |
|
| 127 | + | /// the cache is cleaned. |
|
| 128 | + | export fn publish(root: *[u8], key: binary::Id, package: &artifact::Package, arena: &heap::Heap) |
|
| 128 | 129 | throws (sys::Error) |
|
| 129 | 130 | { |
|
| 130 | 131 | let path = try entry(root, key, arena); |
|
| 131 | 132 | try artifact::directories(try artifact::join(arena, &[root, "/"])); |
|
| 132 | 133 | let stage = try temporary(path, arena); |
|
| 133 | 134 | try artifact::write(package, stage, arena) catch error { |
|
| 134 | 135 | try fs::removeTree(stage, arena) catch {}; |
|
| 135 | 136 | throw error; |
|
| 136 | 137 | }; |
|
| 137 | - | try install(root, key, path, stage, package, scratch, arena) catch error { |
|
| 138 | + | try fs::rename(stage, path) catch error { |
|
| 138 | 139 | try fs::removeTree(stage, arena) catch {}; |
|
| 140 | + | if try fs::stat(path) <> nil { |
|
| 141 | + | return; |
|
| 142 | + | } |
|
| 139 | 143 | throw error; |
|
| 140 | 144 | }; |
|
| 141 | 145 | } |
|
| 142 | - | ||
| 143 | - | /// Install one complete set, retrying a concurrent repair or publication. |
|
| 144 | - | fn install( |
|
| 145 | - | root: *[u8], |
|
| 146 | - | key: binary::Id, |
|
| 147 | - | path: *[u8], |
|
| 148 | - | stage: *[u8], |
|
| 149 | - | package: &artifact::Package, |
|
| 150 | - | scratch: *heap::Heap, |
|
| 151 | - | arena: &heap::Heap |
|
| 152 | - | ) throws (sys::Error) { |
|
| 153 | - | for _ in 0 .. TEMPORARY_SLOTS { |
|
| 154 | - | try fs::rename(stage, path) catch error { |
|
| 155 | - | if error <> sys::Error::NotEmpty |
|
| 156 | - | and error <> sys::Error::Exists |
|
| 157 | - | and error <> sys::Error::Invalid |
|
| 158 | - | { |
|
| 159 | - | throw error; |
|
| 160 | - | } |
|
| 161 | - | if try fs::stat(path) == nil { |
|
| 162 | - | continue; |
|
| 163 | - | } |
|
| 164 | - | let existing = try lookup(root, package.name, key, scratch, arena); |
|
| 165 | - | if existing <> nil { |
|
| 166 | - | try fs::removeTree(stage, arena); |
|
| 167 | - | return; |
|
| 168 | - | } |
|
| 169 | - | // Quarantine the damaged set before installing its complete replacement. |
|
| 170 | - | let damaged = try temporary(path, arena); |
|
| 171 | - | let old = try artifact::join(arena, &[damaged, "/old"]); |
|
| 172 | - | try fs::rename(path, old) catch failure { |
|
| 173 | - | try fs::remove(damaged) catch {}; |
|
| 174 | - | if failure == sys::Error::NotFound { |
|
| 175 | - | continue; |
|
| 176 | - | } |
|
| 177 | - | throw failure; |
|
| 178 | - | }; |
|
| 179 | - | // Unknown damaged trees remain available for inspection. remove does |
|
| 180 | - | // not follow symbolic links, and only empty directories are removed. |
|
| 181 | - | try fs::remove(old) catch failure { |
|
| 182 | - | if failure <> sys::Error::NotEmpty { |
|
| 183 | - | throw failure; |
|
| 184 | - | } |
|
| 185 | - | }; |
|
| 186 | - | try fs::remove(damaged) catch failure { |
|
| 187 | - | if failure <> sys::Error::NotEmpty { |
|
| 188 | - | throw failure; |
|
| 189 | - | } |
|
| 190 | - | }; |
|
| 191 | - | continue; |
|
| 192 | - | }; |
|
| 193 | - | return; |
|
| 194 | - | } |
|
| 195 | - | throw sys::Error::Limit; |
|
| 196 | - | } |
test/integration/packages.rad
+14 -0
| 326 | 326 | ]); |
|
| 327 | 327 | try harness::makeDirs(partial); |
|
| 328 | 328 | try fs::writeFile(h.join(&[partial, "/partial"]), "partial cache entry"); |
|
| 329 | 329 | let unrelated = h.join(&[cache, "/keep.txt"]); |
|
| 330 | 330 | try fs::writeFile(unrelated, "unrelated file"); |
|
| 331 | + | let prefix = h.join(&[ |
|
| 332 | + | cache, "/0000000000000000000000000000000000000000000000000000000000000000" |
|
| 333 | + | ]); |
|
| 334 | + | let suffixes = [".tar", ".tmp.", ".tmp.00", ".tmp.4096"]; |
|
| 335 | + | for suffix in &suffixes[..] { |
|
| 336 | + | try fs::writeFile(h.join(&[prefix, suffix]), "keep"); |
|
| 337 | + | } |
|
| 331 | 338 | h.add(&["clean", "-cache", cache]); |
|
| 332 | 339 | if not h.compile("clean the selected package cache", h.out(&["clean.log"])) { |
|
| 333 | 340 | return; |
|
| 334 | 341 | } |
|
| 342 | + | for suffix in &suffixes[..] { |
|
| 343 | + | h.expect( |
|
| 344 | + | "retain files outside the publication names", |
|
| 345 | + | h.join(&[prefix, suffix]), |
|
| 346 | + | harness::Expect::Present |
|
| 347 | + | ); |
|
| 348 | + | } |
|
| 335 | 349 | h.expect("remove completed cache entries", entries[0], harness::Expect::Absent); |
|
| 336 | 350 | h.expect("remove interrupted cache entries", partial, harness::Expect::Absent); |
|
| 337 | 351 | h.expect("retain unrelated cache-root files", unrelated, harness::Expect::Present); |
|
| 338 | 352 | h.expect( |
|
| 339 | 353 | "retain exported packages", |