compiler: Build the compiler from its manifest

70292a01288a6d91c7c1885875ebe8571c0aa423caa08a5ee1810f4b24c6f7a7
The seed now reads package manifests, but the compiler build still
passed it `-pkg` and `-mod` flags with explicit module lists, and the
kernel build tool found the standard library manifest by a fixed path.

Build the compiler with `SEED_OPTS := compiler/radiance`, so the seed
and every bootstrap stage read `compiler/radiance/radiance.manifest`,
and make `COMPILER_SRC` a wildcard that only serves as a Make
prerequisite. Move the library directory search into
`std::lang::manifest`, where the compiler and the kernel build tool
share it; the tool now finds `std` in the kernel's library directories.

The package flags, `std.lib` and `seed/dependencies` remain for the
integration checks that still describe package graphs on the command
line.

Assisted-by: Claude:opus-5.5
Change-Id: I721d17fe6f43d27dcf74ff67cb03473e1c27a276
Alexis Sellier committed ago 1 parent 0df4c1db
AGENTS.md +2 -1
139 139
- Tests: read nearby fixtures and `test/integration/fixtures.rad` for supported
140 140
  checks. `.ril` files check IL; `//! returns: N` requests an executable check.
141 141
  Success fixtures use `@default fn main`, assertions, and a zero return.
142 142
  Inspect generated IL before accepting a new snapshot.
143 143
- Test production std behavior in std unit suites. Modules are listed in
144 -
  `lib/std/std.manifest`, and again for the seed in `std.lib` and `std.lib.test`.
144 +
  `lib/std/std.manifest`, and again in `std.lib` and `std.lib.test` for the
145 +
  integration checks that still describe packages with flags.
145 146
- Backend debugging: locate the first incorrect stage, from AST and resolution
146 147
  through IL, assembly, and execution. Use Make recipes with the existing
147 148
  `RADIANCE` settings for diagnostic compilation.
148 149
- Bootstrap changes: read `seed/README`. The seed must compile checked-in source.
149 150
  Update it only when needed, through `timeout 600 make seed` or the documented
Makefile +16 -18
1 1
# Radiance build file.
2 2
3 3
# Standard library manifest and sources. `std.lib` and `std.lib.test` list
4 -
# the same modules as the manifest for the seed, which predates manifests.
4 +
# the same modules as the manifest for the integration checks that still
5 +
# describe packages with flags.
5 6
STD_MANIFEST := lib/std/std.manifest
6 7
STD_LIB      := $(shell cat std.lib)
7 8
STD_TEST_LIB := $(STD_LIB) $(shell cat std.lib.test)
8 9
9 10
# Compiler output.
23 24
default: $(RAD_BIN)
24 25
25 26
# Compiler build
26 27
27 28
SEED              := seed/radiance.rv64
28 -
COMPILER_SRC      := compiler/radiance/radiance.rad compiler/radiance/codegen.rad \
29 -
    compiler/radiance/testgen.rad compiler/radiance/link.rad \
30 -
    compiler/radiance/artifact.rad compiler/radiance/metadata.rad \
31 -
    compiler/radiance/build.rad compiler/radiance/cache.rad compiler/radiance/locations.rad
32 -
# Package flags for the seed, which predates manifests. Its dependency lists
33 -
# are in `seed/dependencies`. Module files are in the package directory, so
34 -
# each module after the root is mapped to its logical path.
35 -
# `$(call modules,DIR,ROOT,FILES)` gives the `-mod` flags of the files FILES
36 -
# of the package in DIR whose root module is ROOT.
29 +
COMPILER_SRC      := $(wildcard compiler/radiance/*.rad) compiler/radiance/radiance.manifest
30 +
# The compiler package, which the seed builds with its dependencies.
31 +
SEED_OPTS         := compiler/radiance
32 +
33 +
$(RAD_BIN): Makefile $(SEED) $(STD_LIB) $(STD_MANIFEST) $(COMPILER_SRC) | $(BIN_DIR)
34 +
	@echo "radiance $(SEED) => $@"
35 +
	@$(HOSTED) $(SEED) $(SEED_OPTS) -o $@
36 +
37 +
# Package flags for integration checks that still describe packages on the
38 +
# command line. Their dependency lists are in `seed/dependencies`. Module
39 +
# files are in the package directory, so each module after the root is
40 +
# mapped to its logical path. `$(call modules,DIR,ROOT,FILES)` gives the
41 +
# `-mod` flags of the files FILES of the package in DIR whose root module
42 +
# is ROOT.
37 43
modules = -mod $(2) $(foreach f,$(filter-out $(2),$(filter %.rad,$(3))), \
38 44
    -mod $(patsubst $(1)/%.rad,%,$(f))=$(f)) $(patsubst %,-mod %,$(filter %.ras,$(3)))
39 45
STD               := -pkg std -manifest seed/dependencies/none \
40 46
    $(call modules,lib/std,lib/std/std.rad,$(STD_LIB))
41 -
SEED_OPTS         := $(STD) -pkg radiance -manifest seed/dependencies/std \
42 -
    $(call modules,compiler/radiance,compiler/radiance/radiance.rad,$(COMPILER_SRC)) \
43 -
    -entry radiance
44 -
45 -
$(RAD_BIN): Makefile $(SEED) std.lib $(STD_LIB) $(COMPILER_SRC) seed/dependencies/none \
46 -
    seed/dependencies/std | $(BIN_DIR)
47 -
	@echo "radiance $(SEED) => $@"
48 -
	@$(HOSTED) $(SEED) $(SEED_OPTS) -o $@
49 47
50 48
# Compiler inputs shared by the normal build and bootstrap stages.
51 49
compiler-options:
52 50
	@printf '%s\n' '$(SEED_OPTS)'
53 51
compiler/radiance/radiance.rad +20 -82
138 138
    requirements: *[manifest::Dependency],
139 139
    /// Declared dependency names in manifest order.
140 140
    dependencies: *[*[u8]],
141 141
}
142 142
143 -
/// Name of the library directories searched from the entry package upward.
144 -
constant LIBRARY: *[u8] = "lib";
145 143
/// Extension of a root module given in place of a package directory.
146 144
constant MODULE_EXT: *[u8] = ".rad";
147 145
148 -
/// A package found by name.
149 -
export union Found: Copy {
150 -
    /// Source package with this manifest.
151 -
    Source(*[u8]),
152 -
    /// Compiled package in this directory.
153 -
    Compiled(*[u8]),
154 -
}
155 -
156 146
/// Provider of one dependency, selected with `-dep`.
157 147
export record Provider: Copy {
158 148
    /// Dependency name.
159 149
    name: *[u8],
160 150
    /// The selected package.
161 -
    found: Found,
151 +
    found: manifest::Found,
162 152
}
163 153
164 154
/// How to discover package inputs in a manifest graph.
165 155
export record ManifestSearch: Copy {
166 -
    /// Library directories from `-lib`, searched in order before the
167 -
    /// [`LIBRARY`] directories of the entry package and its parents.
156 +
    /// Library directories from `-lib`, searched in order before the `lib`
157 +
    /// directories of the entry package and its parents.
168 158
    libraries: *[*[u8]],
169 159
    /// Providers from `-dep`, which override the library search.
170 160
    providers: *[Provider],
171 161
    /// Whether test modules are added.
172 162
    tests: bool,
436 426
        throw error(&["source arena exhausted"]);
437 427
    };
438 428
    return &result[..];
439 429
}
440 430
441 -
/// Check whether a file or directory exists at `path`.
442 -
fn exists(path: *[u8]) -> bool throws (Error) {
443 -
    let found = try fs::stat(path) catch {
444 -
        throw error(&["cannot read", path]);
445 -
    };
446 -
    return found <> nil;
447 -
}
448 -
449 431
/// Return the library directories searched for the dependencies of the
450 -
/// package in `directory`: `-lib` directories in order, then [`LIBRARY`] in
451 -
/// `directory` and in each of its parents, up to the current directory.
432 +
/// package in `directory`, after the `-lib` directories `libraries`.
452 433
fn libraryPath(directory: *[u8], libraries: *[*[u8]], arena: &heap::Heap) -> *[*[u8]]
453 434
    throws (Error)
454 435
{
455 -
    let mut result: *mut [*[u8]] = &mut [];
456 -
    try result.extend(libraries, arena) catch {
457 -
        throw error(&["source arena exhausted"]);
436 +
    return try manifest::libraries(directory, libraries, arena) catch {
437 +
        throw error(&["cannot search library directories for", directory]);
458 438
    };
459 -
    let mut current = try locate("", directory, arena);
460 -
    loop {
461 -
        let candidate = try artifact::join(arena, &[current, "/", LIBRARY]) catch {
462 -
            throw error(&["source arena exhausted"]);
463 -
        };
464 -
        let library = try locate("", candidate, arena);
465 -
        try result.append(library, arena) catch {
466 -
            throw error(&["source arena exhausted"]);
467 -
        };
468 -
        if mem::eq(current, ".")
469 -
            or mem::eq(current, "/")
470 -
            or mem::eq(current, "..")
471 -
            or mem::endsWith(current, "/..")
472 -
        {
473 -
            break;
474 -
        }
475 -
        if let slash = mem::rfind(current, "/") {
476 -
            set current = "/" if slash == 0 else &current[.. slash];
477 -
        } else {
478 -
            set current = ".";
479 -
        }
480 -
    }
481 -
    return &result[..];
482 439
}
483 440
484 -
/// Find package `name` in the directory `directory`: the source package with
485 -
/// the manifest `directory/name.manifest`, or the compiled package
486 -
/// `directory/name.ril.bin`.
487 -
fn findPackage(directory: *[u8], name: *[u8], arena: &heap::Heap) -> ?Found throws (Error) {
488 -
    let file = try manifest::file(directory, name, arena) catch {
489 -
        throw error(&["path too long:", directory]);
490 -
    };
491 -
    if try exists(file) {
492 -
        return Found::Source(file);
493 -
    }
494 -
    let image = try artifact::join(arena, &[directory, "/", name, ".ril.bin"]) catch {
495 -
        throw error(&["source arena exhausted"]);
441 +
/// Find package `name` in the library directories `libraries`.
442 +
fn findLibrary(name: *[u8], libraries: *[*[u8]], arena: &heap::Heap) -> manifest::Found
443 +
    throws (Error)
444 +
{
445 +
    let found = try manifest::lookup(name, libraries, arena) catch {
446 +
        throw error(&["cannot search library directories for", name]);
496 447
    };
497 -
    if try exists(image) {
498 -
        return Found::Compiled(directory);
499 -
    }
500 -
    return nil;
501 -
}
502 -
503 -
/// Find package `name` in the library directories `libraries`, as the
504 -
/// package directory `name` of the first one that holds it.
505 -
fn findLibrary(name: *[u8], libraries: *[*[u8]], arena: &heap::Heap) -> Found throws (Error) {
506 -
    for library in libraries {
507 -
        let joined = try artifact::join(arena, &[library, "/", name]) catch {
508 -
            throw error(&["source arena exhausted"]);
509 -
        };
510 -
        if let found = try findPackage(try locate("", joined, arena), name, arena) {
511 -
            return found;
512 -
        }
448 +
    if let package = found {
449 +
        return package;
513 450
    }
514 451
    io::printError("radiance: library directories searched:");
515 452
    for library in libraries {
516 453
        io::printError(" ");
517 454
        io::printError(library);
529 466
    let name = &arg[.. at];
530 467
    if name.len == 0 or at + 1 >= arg.len {
531 468
        throw error(&["`-dep` requires <name>=<directory>"]);
532 469
    }
533 470
    let directory = try locate("", &arg[at + 1 ..], arena);
534 -
    let found = try findPackage(directory, name, arena) else {
535 -
        throw error(&["`-dep`:", directory, "holds no package", name]);
471 +
    let package = try manifest::found(directory, name, arena) catch {
472 +
        throw error(&["cannot read", directory]);
536 473
    };
474 +
    let found = package else throw error(&["`-dep`:", directory, "holds no package", name]);
537 475
    let mut result: *mut [Provider] = &mut [];
538 476
    try result.extend(providers, arena) catch {
539 477
        throw error(&["source arena exhausted"]);
540 478
    };
541 479
    try result.append(Provider { name, found }, arena) catch {
675 613
    native: build::NativeInputs,
676 614
    used: &mut [bool; package::MAX_PACKAGES],
677 615
    syntax: &mut parser::Parser,
678 616
    arena: &heap::Heap
679 617
) -> PackageInput throws (Error) {
680 -
    let mut provider: ?Found = nil;
618 +
    let mut provider: ?manifest::Found = nil;
681 619
    for given, i in search.providers {
682 620
        if mem::eq(given.name, name) {
683 621
            set provider = given.found;
684 622
            set used[i] = true;
685 623
        }
686 624
    }
687 625
    let found = provider else try findLibrary(name, search.libraries, arena);
688 626
    match found {
689 -
        case Found::Source(file) => {
627 +
        case manifest::Found::Source(file) => {
690 628
            return try loadManifest(
691 629
                file, name, search.tests, native, syntax, arena
692 630
            );
693 631
        }
694 -
        case Found::Compiled(directory) => {
632 +
        case manifest::Found::Compiled(directory) => {
695 633
            let mut input = packageInput(name);
696 634
            set input.artifactDirectory = directory;
697 635
            return input;
698 636
        }
699 637
    }
kernel/tools/build.rad +9 -6
256 256
        set output[i] = shared::Symbol { name: imported.exported, target, startable: false };
257 257
    }
258 258
    return &providers[..];
259 259
}
260 260
261 -
/// Library directory that holds the standard library package.
262 -
constant LIBRARY: *[u8] = "lib/std";
263 -
264 261
/// Read the manifest at `path`, from which its relative paths are resolved.
265 262
fn readManifest(path: *[u8], arena: &heap::Heap) -> ManifestFile {
266 263
    let text = try fs::readFileAlloc(path, arena) catch {
267 264
        panic "readManifest: cannot read manifest";
268 265
    };
329 326
///
330 327
/// `[-std-asm <std.ras>] <std.ril.bin> <kernel.ril.bin> <kernel.ras> <output> [<boot.ril.bin>..]`
331 328
/// `-package <kernel directory> <std.ril.bin> <kernel.ril.bin> <output> [<boot.ril.bin>..]`
332 329
///
333 330
/// With `-package`, the assembly is what the manifests of the kernel package
334 -
/// and of the standard library package in [`LIBRARY`] list.
331 +
/// and of its `std` dependency list. `std` is found in the library
332 +
/// directories of the kernel package, as the compiler finds it.
335 333
unsafe fn arguments(args: *[*[u8]]) -> Arguments {
336 334
    if args.len >= 2 and mem::eq(args[0], "-package") {
337 335
        let rest = &args[2 ..];
338 336
        assert rest.len >= 3 and rest.len <= PACKAGES + 1;
339 337
        let manifests = heap::over(&mut MANIFESTS[..]);
340 338
        let kernelPath = try manifest::find(args[1], manifests) catch {
341 339
            panic "arguments: no kernel package manifest";
342 340
        };
343 -
        let libraryPath = try manifest::file(LIBRARY, "std", manifests) catch {
344 -
            panic "arguments: path too long";
341 +
        let libraries = try manifest::libraries(args[1], &[], manifests) catch {
342 +
            panic "arguments: cannot search library directories";
343 +
        };
344 +
        let found = try manifest::lookup("std", libraries, manifests) catch {
345 +
            panic "arguments: cannot search library directories";
345 346
        };
347 +
        let case manifest::Found::Source(libraryPath) = found
348 +
            else panic "arguments: no `std` source package";
346 349
        let kernelFile = readManifest(kernelPath, manifests);
347 350
        let libraryFile = readManifest(libraryPath, manifests);
348 351
        let mut packages: *mut [*[u8]] = &mut [];
349 352
        try! packages.extend(&rest[.. 2], manifests);
350 353
        try! packages.extend(&rest[3 ..], manifests);
lib/std/lang/manifest.rad +85 -0
43 43
use std::lang::parser;
44 44
use std::lang::scanner;
45 45
46 46
/// File name extension of a manifest.
47 47
export constant EXTENSION: *[u8] = ".manifest";
48 +
/// Name of the library directories searched from a package upward.
49 +
export constant LIBRARY: *[u8] = "lib";
48 50
/// Bytes of directory listing storage that [`find`] uses.
49 51
constant LIST_BYTES: u32 = 65536;
50 52
51 53
/// Largest number of path components that [`resolve`] keeps.
52 54
constant MAX_COMPONENTS: u32 = 64;
53 55
56 +
/// A package found by name.
57 +
export union Found: Copy {
58 +
    /// Source package with the manifest at this path.
59 +
    Source(*[u8]),
60 +
    /// Compiled package in this directory.
61 +
    Compiled(*[u8]),
62 +
}
63 +
54 64
/// A package that another package imports.
55 65
export record Dependency: Copy {
56 66
    /// Package name.
57 67
    name: *[u8],
58 68
    /// Oldest compatible version, `MAJOR.MINOR.PATCH`.
195 205
    };
196 206
    let text: *[u8] = &joined[..];
197 207
    return try resolve(&text[.. directory.len + 1], &text[directory.len + 1 ..], arena);
198 208
}
199 209
210 +
/// Return the library directories searched for the dependencies of the
211 +
/// package in `directory`: the directories `first` in order, then [`LIBRARY`]
212 +
/// in `directory` and in each of its parents, up to the current directory.
213 +
export fn libraries(directory: *[u8], first: *[*[u8]], arena: &heap::Heap) -> *[*[u8]]
214 +
    throws (Error)
215 +
{
216 +
    let mut result: *mut [*[u8]] = &mut [];
217 +
    try result.extend(first, arena) catch {
218 +
        throw Error::TooLarge;
219 +
    };
220 +
    let mut current = try resolve("", directory, arena);
221 +
    loop {
222 +
        let parts: [*[u8]; 3] = [current, "/", LIBRARY];
223 +
        let joined = try alloc::concatBytes(arena, &parts[..]) catch {
224 +
            throw Error::TooLarge;
225 +
        };
226 +
        try result.append(try resolve("", &joined[..], arena), arena) catch {
227 +
            throw Error::TooLarge;
228 +
        };
229 +
        if mem::eq(current, ".")
230 +
            or mem::eq(current, "/")
231 +
            or mem::eq(current, "..")
232 +
            or mem::endsWith(current, "/..")
233 +
        {
234 +
            break;
235 +
        }
236 +
        if let slash = mem::rfind(current, "/") {
237 +
            set current = "/" if slash == 0 else &current[.. slash];
238 +
        } else {
239 +
            set current = ".";
240 +
        }
241 +
    }
242 +
    return &result[..];
243 +
}
244 +
245 +
/// Find package `name` in `directory`: the source package with the manifest
246 +
/// `directory/name.manifest`, or the compiled package `directory/name.ril.bin`.
247 +
export fn found(directory: *[u8], name: *[u8], arena: &heap::Heap) -> ?Found throws (Error) {
248 +
    let manifest = try file(directory, name, arena);
249 +
    if try exists(manifest) {
250 +
        return Found::Source(manifest);
251 +
    }
252 +
    let parts: [*[u8]; 4] = [directory, "/", name, ".ril.bin"];
253 +
    let image = try alloc::concatBytes(arena, &parts[..]) catch {
254 +
        throw Error::TooLarge;
255 +
    };
256 +
    if try exists(&image[..]) {
257 +
        return Found::Compiled(directory);
258 +
    }
259 +
    return nil;
260 +
}
261 +
262 +
/// Find package `name` as the package directory `name` of the first of the
263 +
/// library directories `libraries` that holds it.
264 +
export fn lookup(name: *[u8], libraries: *[*[u8]], arena: &heap::Heap) -> ?Found throws (Error) {
265 +
    for library in libraries {
266 +
        let parts: [*[u8]; 3] = [library, "/", name];
267 +
        let joined = try alloc::concatBytes(arena, &parts[..]) catch {
268 +
            throw Error::TooLarge;
269 +
        };
270 +
        if let package = try found(try resolve("", &joined[..], arena), name, arena) {
271 +
            return package;
272 +
        }
273 +
    }
274 +
    return nil;
275 +
}
276 +
277 +
/// Check whether a file or directory exists at `path`.
278 +
fn exists(path: *[u8]) -> bool throws (Error) {
279 +
    let entry = try fs::stat(path) catch {
280 +
        throw Error::Unreadable;
281 +
    };
282 +
    return entry <> nil;
283 +
}
284 +
200 285
/// Return the location of the one manifest in `directory`.
201 286
export fn find(directory: *[u8], arena: &heap::Heap) -> *[u8] throws (Error) {
202 287
    let listing = try fs::listDirAlloc(directory, LIST_BYTES, arena) catch failure as sys::Error {
203 288
        throw Error::Unreadable;
204 289
    } catch failure as alloc::AllocError {
lib/std/lang/manifest/tests.rad +20 -0
171 171
        };
172 172
        try testing::expect(mem::eq(path, item[2]));
173 173
    }
174 174
}
175 175
176 +
@test unsafe fn testLibraries() throws (testing::TestError) {
177 +
    let arena = heap::over(&mut RESULTS[..]);
178 +
    let nested = try super::libraries("compiler/radiance", &["extra"], arena) catch {
179 +
        throw testing::TestError::Failed;
180 +
    };
181 +
    try expectNames(nested, &["extra", "compiler/radiance/lib", "compiler/lib", "lib"]);
182 +
    let current = try super::libraries(".", &[], arena) catch {
183 +
        throw testing::TestError::Failed;
184 +
    };
185 +
    try expectNames(current, &["lib"]);
186 +
    let above = try super::libraries("../app", &[], arena) catch {
187 +
        throw testing::TestError::Failed;
188 +
    };
189 +
    try expectNames(above, &["../app/lib", "../lib"]);
190 +
    let absolute = try super::libraries("/srv/app", &[], arena) catch {
191 +
        throw testing::TestError::Failed;
192 +
    };
193 +
    try expectNames(absolute, &["/srv/app/lib", "/srv/lib", "/lib"]);
194 +
}
195 +
176 196
@test fn testCompatible() throws (testing::TestError) {
177 197
    assert not super::compatible("1.0.0", "18446744073709551617.0.0");
178 198
    assert super::compatible("1.18446744073709551615.0", "1.18446744073709551616.0");
179 199
    assert super::compatible("1.2.3", "1.2.3");
180 200
    assert super::compatible("1.2.3", "1.2.4");