radiance · Radiance self-hosting compiler

compiler/ kernel/ lib/ examples/ std/ arch/ char/ collections/ lang/ alloc/ ast/ gen/ il/ module/ parser/ resolver/ scanner/ alloc.rad 7.1 KiB ast.rad 26.7 KiB gen.rad 513 B il.rad 19.5 KiB lower.rad 308.1 KiB module.rad 14.9 KiB package.rad 1.3 KiB parser.rad 89.5 KiB resolver.rad 439.6 KiB scanner.rad 18.0 KiB sexpr.rad 6.4 KiB strings.rad 2.2 KiB types.rad 1.6 KiB sys/ arch.rad 68 B char.rad 855 B collections.rad 39 B fmt.rad 8.3 KiB intrinsics.rad 467 B io.rad 1.7 KiB lang.rad 276 B mem.rad 2.3 KiB sys.rad 179 B testing.rad 2.4 KiB tests.rad 15.7 KiB vec.rad 3.2 KiB std.rad 281 B scripts/ seed/ sublime/ test/ vim/ .gitignore 336 B .gitsigners 112 B CONTRIBUTING 2.1 KiB LICENSE 1.1 KiB Makefile 9.2 KiB README 2.5 KiB STYLE 2.5 KiB std.lib 1.5 KiB std.lib.test 662 B

lib/std/lang/scanner.rad 18.0 KiB raw

//! Lexical scanner for the Radiance programming language.
//!
//! This module implements a hand-written scanner that tokenizes Radiance
//! source code into a stream of tokens for consumption by the parser.
@test mod tests;

use std::char;
use std::mem;
use std::lang::strings;

/// Token kinds representing all lexical elements in Radiance.
///
/// This enum covers operators, keywords, literals, and structural
/// elements used by the parser to build the AST.
export union TokenKind: Copy {
    /// Special end of file token generated when the input is exhausted.
    Eof,
    /// Special invalid token.
    Invalid,

    LParen,     // (
    RParen,     // )
    LBrace,     // {
    RBrace,     // }
    LBracket,   // [
    RBracket,   // ]
    Comma,      // ,
    Dot,        // .
    DotDot,     // ..
    Minus,      // -
    Plus,       // +
    Colon,      // :
    ColonColon, // ::
    Semicolon,  // ;
    Slash,      // /
    Star,       // *
    Percent,    // %
    Amp,        // &
    Pipe,       // |
    Caret,      // ^
    Tilde,      // ~
    Underscore, // _
    Question,   // ?
    Bang,       // !
    LtGt,       // <>
    Equal,      // =
    EqualEqual, // ==
    Gt,         // >
    GtEqual,    // >=
    Lt,         // <
    LtEqual,    // <=
    LtLt,       // <<
    GtGt,       // >>
    Arrow,      // ->
    FatArrow,   // =>

    // Compound assignment operators.
    PlusEqual,    // +=
    MinusEqual,   // -=
    StarEqual,    // *=
    SlashEqual,   // /=
    PercentEqual, // %=
    AmpEqual,     // &=
    PipeEqual,    // |=
    CaretEqual,   // ^=
    LtLtEqual,    // <<=
    GtGtEqual,    // >>=

    // Boolean operators.
    Not, And, Or,

    /// Eg. `fnord`
    Ident,
    /// Eg. `@default`
    AtIdent,
    /// Region name with an apostrophe prefix.
    Region,
    /// The `log` keyword.
    Log,

    // Literals.
    String,     // "fnord"
    Char,       // 'f'
    Number,     // 42
    True,       // true
    False,      // false
    Nil,        // nil
    Undefined,  // undefined

    // Control flow tokens.
    If, Else, Return, Break,
    Continue, While, For, In,
    Loop, Match, Case, Try, Catch,
    Throw, Throws, Panic, Assert,

    // Variable binding tokens.
    Let, Mut, Set, Constant, Align,

    // Module-related tokens.
    Mod, Use, Super,

    // Type or function attributes.
    Export, Static,

    // Trait-related tokens.
    Trait, Instance,

    // Type-related tokens.
    I8, I16, I32, I64, U8, U16, U32, U64,
    Opaque, Fn, Bool, Union, Record, As, Unsafe, Where
}

/// A reserved keyword.
record Keyword: Copy {
    /// Keyword string.
    name: *[u8],
    /// Corresponding token.
    tok: TokenKind,
}

/// Sorted keyword table for binary search.
constant KEYWORDS: [Keyword; 53] = [
    { name: "align", tok: TokenKind::Align },
    { name: "and", tok: TokenKind::And },
    { name: "as", tok: TokenKind::As },
    { name: "assert", tok: TokenKind::Assert },
    { name: "bool", tok: TokenKind::Bool },
    { name: "break", tok: TokenKind::Break },
    { name: "case", tok: TokenKind::Case },
    { name: "catch", tok: TokenKind::Catch },
    { name: "constant", tok: TokenKind::Constant },
    { name: "continue", tok: TokenKind::Continue },
    { name: "else", tok: TokenKind::Else },
    { name: "export", tok: TokenKind::Export },
    { name: "false", tok: TokenKind::False },
    { name: "fn", tok: TokenKind::Fn },
    { name: "for", tok: TokenKind::For },
    { name: "i16", tok: TokenKind::I16 },
    { name: "i32", tok: TokenKind::I32 },
    { name: "i64", tok: TokenKind::I64 },
    { name: "i8", tok: TokenKind::I8 },
    { name: "if", tok: TokenKind::If },
    { name: "in", tok: TokenKind::In },
    { name: "instance", tok: TokenKind::Instance },
    { name: "let", tok: TokenKind::Let },
    { name: "log", tok: TokenKind::Log },
    { name: "loop", tok: TokenKind::Loop },
    { name: "match", tok: TokenKind::Match },
    { name: "mod", tok: TokenKind::Mod },
    { name: "mut", tok: TokenKind::Mut },
    { name: "nil", tok: TokenKind::Nil },
    { name: "not", tok: TokenKind::Not },
    { name: "opaque", tok: TokenKind::Opaque },
    { name: "or", tok: TokenKind::Or },
    { name: "panic", tok: TokenKind::Panic },
    { name: "record", tok: TokenKind::Record },
    { name: "return", tok: TokenKind::Return },
    { name: "set", tok: TokenKind::Set },
    { name: "static", tok: TokenKind::Static },
    { name: "super", tok: TokenKind::Super },
    { name: "throw", tok: TokenKind::Throw },
    { name: "throws", tok: TokenKind::Throws },
    { name: "trait", tok: TokenKind::Trait },
    { name: "true", tok: TokenKind::True },
    { name: "try", tok: TokenKind::Try },
    { name: "u16", tok: TokenKind::U16 },
    { name: "u32", tok: TokenKind::U32 },
    { name: "u64", tok: TokenKind::U64 },
    { name: "u8", tok: TokenKind::U8 },
    { name: "undefined", tok: TokenKind::Undefined },
    { name: "union", tok: TokenKind::Union },
    { name: "unsafe", tok: TokenKind::Unsafe },
    { name: "use", tok: TokenKind::Use },
    { name: "where", tok: TokenKind::Where },
    { name: "while", tok: TokenKind::While },
];

/// Describes where source code originated from.
export union SourceLoc: Copy {
    /// Source loaded from a file at the given path.
    File(*[u8]),
    /// Source provided as an inline string (no file path).
    String,
}

/// Lexical scanner state for tokenizing Radiance source code.
///
/// Maintains position information and source buffer reference.
export record Scanner: Copy {
    /// Origin of the source being scanned.
    sourceLoc: SourceLoc,
    /// Source buffer.
    source: *[u8],
    /// Offset of current token into buffer.
    token: u32,
    /// Offset of current character being scanned.
    cursor: u32,
    /// Interned string pool. It must remain valid while scanning.
    pool: *unsafe mut strings::Pool,
}

/// Individual token with kind, source text, and position.
///
/// Represents a single lexical element extracted from source,
/// including its original text and byte offset for error reporting.
export record Token: Copy {
    /// Token kind.
    kind: TokenKind,
    /// Token source string.
    source: *[u8],
    /// Byte offset of `source` in input buffer.
    offset: u32,
}

/// Source code location with line/column information.
///
/// Used for error reporting and debugging.
export record Location: Copy {
    /// Origin of the source.
    source: SourceLoc,
    /// Line number.
    line: u16,
    /// Column number.
    col: u16,
}

/// Create a new scanner object.
export unsafe fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: *unsafe mut strings::Pool) -> Scanner {
    // Intern built-in functions and attributes.
    strings::intern(pool, "@sizeOf");
    strings::intern(pool, "@alignOf");
    strings::intern(pool, "@sliceOf");
    strings::intern(pool, "@default");
    strings::intern(pool, "@intrinsic");
    strings::intern(pool, "@test");
    // Intern built-in slice methods.
    strings::intern(pool, "append");
    strings::intern(pool, "delete");

    return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
}

/// Check if we've reached the end of input.
export fn isEof(s: &Scanner) -> bool {
    return s.cursor >= s.source.len;
}

/// Get the current character, if any.
export fn current(s: &Scanner) -> ?u8 {
    if isEof(s) {
        return nil;
    }
    return s.source[s.cursor];
}

/// Peek at the next character without advancing the scanner.
fn peek(s: &Scanner) -> ?u8 {
    if s.cursor + 1 >= s.source.len {
        return nil;
    }
    return s.source[s.cursor + 1];
}

/// Advance scanner and return the character that was consumed.
fn advance(s: &mut Scanner) -> u8 {
    let ch = s.source[s.cursor];
    set s.cursor += 1;
    return ch;
}

/// Consume the expected character if it matches the current position.
fn consume(s: &mut Scanner, expected: u8) -> bool {
    if let c = current(s); c == expected {
        advance(s);
        return true;
    }
    return false;
}

/// Create a token from the current scanner state.
fn tok(s: &Scanner, kind: TokenKind) -> Token {
    return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
}

/// Create an invalid token with the given message.
export fn invalid(offset: u32, message: *[u8]) -> Token {
    return Token { kind: TokenKind::Invalid, source: message, offset };
}

/// Skip whitespace characters and line comments.
fn skipWhitespace(s: &mut Scanner) {
    while let ch = current(s) {
        match ch {
            case ' ', '\n', '\r', '\t' => advance(s),
            case '/' => {
                if let c = peek(s); c == '/' {
                    while let ch = current(s); ch <> '\n' {
                        advance(s);
                    }
                } else {
                    return;
                }
            }
            else => return,
        }
    }
}

/// Scan numeric literal (decimal, hex, or binary).
fn scanNumber(s: &mut Scanner) -> Token {
    // Check for hex literal (`0x` or `0X` prefix).
    if s.source[s.cursor - 1] == '0' {
        if let ch = current(s); ch == 'x' or ch == 'X' {
            advance(s);
            // Must have at least one hex digit after `0x`.
            if let ch = current(s); not char::isHexDigit(ch) {
                return invalid(s.token, "invalid hex literal");
            }
            while let ch = current(s); char::isHexDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
        // Check for binary literal (`0b` or `0B` prefix).
        if let ch = current(s); ch == 'b' or ch == 'B' {
            advance(s);
            // Must have at least one binary digit after `0b`.
            if let ch = current(s); not char::isBinDigit(ch) {
                return invalid(s.token, "invalid binary literal");
            }
            while let ch = current(s); char::isBinDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
    }
    // Regular decimal number.
    while let ch = current(s); char::isDigit(ch) {
        advance(s);
    }
    // Look for decimal part.
    if let ch = current(s); ch == '.' {
        if let p = peek(s); char::isDigit(p) {
            advance(s); // Consume the "."
            while let ch = current(s); char::isDigit(ch) {
                advance(s);
            }
        }
    }
    return tok(s, TokenKind::Number);
}

fn scanDelimited(s: &mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
    while let ch = current(s); ch <> delim {
        if not char::isPrint(ch) {
            return invalid(s.token, "invalid character");
        }
        if consume(s, '\\') { // Consume escapes
            if isEof(s) {
                return nil;
            }
        }
        advance(s);
    }
    if not consume(s, delim) {
        return nil;
    }
    return tok(s, kind);
}

/// Scan string literal enclosed in double quotes.
fn scanString(s: &mut Scanner) -> Token {
    if let tok = scanDelimited(s, '"', TokenKind::String) {
        return tok;
    }
    return invalid(s.token, "unterminated string");
}

/// Scan an apostrophe-prefixed region name or a quoted character literal.
fn scanChar(s: &mut Scanner) -> Token {
    if let ch = current(s); char::isAlpha(ch) or ch == '_' {
        while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
            advance(s);
        }
        if consume(s, '\'') {
            return tok(s, TokenKind::Char);
        }
        return tok(s, TokenKind::Region);
    }
    if let tok = scanDelimited(s, '\'', TokenKind::Char) {
        return tok;
    }
    return invalid(s.token, "unterminated character");
}

/// Scan a keyword or an identifier.
fn keywordOrIdent(src: *[u8]) -> TokenKind {
    let mut left: u32 = 0;
    let mut right: u32 = KEYWORDS.len;

    while left < right {
        let mid = left + ((right - left) / 2);
        let kw = &KEYWORDS[mid];
        let cmp = mem::cmp(src, kw.name);

        match cmp {
            case -1 => set right = mid,
            case 1 => set left = mid + 1,
            else => return kw.tok,
        }
    }
    return TokenKind::Ident;
}

/// Scan an identifier, keyword, or label.
unsafe fn scanIdentifier(s: &mut Scanner) -> Token {
    while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
        advance(s);
    }
    let ident = &s.source[s.token..s.cursor];
    let kind = keywordOrIdent(ident);

    // Only intern actual identifiers, not keywords.
    if kind == TokenKind::Ident {
        return Token { kind, source: strings::intern(s.pool, ident), offset: s.token };
    }
    return tok(s, kind);
}

/// Scan the next token. The retained string pool must be valid and writable.
export unsafe fn next(s: &mut Scanner) -> Token {
    skipWhitespace(s);  // Skip any whitespace between tokens.
    set s.token = s.cursor; // Token starts at current position.

    if isEof(s) {
        return tok(s, TokenKind::Eof);
    }
    let c: u8 = advance(s);

    if char::isDigit(c) {
        return scanNumber(s);
    }
    if char::isAlpha(c) {
        return scanIdentifier(s);
    }
    match c {
        case '\'' => return scanChar(s),
        case '"'  => return scanString(s),
        case '('  => return tok(s, TokenKind::LParen),
        case ')'  => return tok(s, TokenKind::RParen),
        case '{'  => return tok(s, TokenKind::LBrace),
        case '}'  => return tok(s, TokenKind::RBrace),
        case '['  => return tok(s, TokenKind::LBracket),
        case ']'  => return tok(s, TokenKind::RBracket),
        case ';'  => return tok(s, TokenKind::Semicolon),
        case ','  => return tok(s, TokenKind::Comma),
        case '.'  => {
            if consume(s, '.') {
                return tok(s, TokenKind::DotDot);
            }
            return tok(s, TokenKind::Dot);
        }
        case ':'  => {
            if consume(s, ':') {
                return tok(s, TokenKind::ColonColon);
            }
            return tok(s, TokenKind::Colon);
        }
        case '-'  => {
            if consume(s, '>') {
                return tok(s, TokenKind::Arrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::MinusEqual);
            }
            return tok(s, TokenKind::Minus);
        }
        case '+' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PlusEqual);
            }
            return tok(s, TokenKind::Plus);
        }
        case '/' => {
            if consume(s, '=') {
                return tok(s, TokenKind::SlashEqual);
            }
            return tok(s, TokenKind::Slash);
        }
        case '*' => {
            if consume(s, '=') {
                return tok(s, TokenKind::StarEqual);
            }
            return tok(s, TokenKind::Star);
        }
        case '%' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PercentEqual);
            }
            return tok(s, TokenKind::Percent);
        }
        case '&' => {
            if consume(s, '=') {
                return tok(s, TokenKind::AmpEqual);
            }
            return tok(s, TokenKind::Amp);
        }
        case '?' => return tok(s, TokenKind::Question),
        case '|' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PipeEqual);
            }
            return tok(s, TokenKind::Pipe);
        }
        case '^' => {
            if consume(s, '=') {
                return tok(s, TokenKind::CaretEqual);
            }
            return tok(s, TokenKind::Caret);
        }
        case '~' => return tok(s, TokenKind::Tilde),
        case '!' => return tok(s, TokenKind::Bang),
        case '=' => {
            if consume(s, '>') {
                return tok(s, TokenKind::FatArrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::EqualEqual);
            }
            return tok(s, TokenKind::Equal);
        }
        case '<' => {
            if consume(s, '>') {
                return tok(s, TokenKind::LtGt);
            }
            if consume(s, '<') {
                if consume(s, '=') {
                    return tok(s, TokenKind::LtLtEqual);
                }
                return tok(s, TokenKind::LtLt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::LtEqual);
            }
            return tok(s, TokenKind::Lt);
        }
        case '>' => {
            if consume(s, '>') {
                if consume(s, '=') {
                    return tok(s, TokenKind::GtGtEqual);
                }
                return tok(s, TokenKind::GtGt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::GtEqual);
            }
            return tok(s, TokenKind::Gt);
        }
        case '@' => {
            // Scan `@identifier` as a single token.
            let ch = current(s) else {
                return invalid(s.token, "expected identifier after `@`");
            };
            if not char::isAlpha(ch) and ch <> '_' {
                return invalid(s.token, "expected identifier after `@`");
            }
            while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
                advance(s);
            }
            let name = &s.source[s.token..s.cursor];
            return Token {
                kind: TokenKind::AtIdent,
                source: strings::intern(s.pool, name),
                offset: s.token,
            };
        }
        case '_' => {
            if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
                // This is part of an identifier like `_foo` or `__start`
                return scanIdentifier(s);
            }
            return tok(s, TokenKind::Underscore);
        }
        else => return invalid(s.token, "unexpected character"),
    }
}

/// Get the source code location from a byte offset.
export fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
    let mut l: u16 = 1;
    let mut c: u16 = 1;

    if offset >= source.len {
        return nil;
    }
    for ch in &source[..offset] {
        if ch == '\n' {
            set c = 1;
            set l += 1;
        } else {
            set c += 1;
        }
    }
    return Location { source: sourceLoc, line: l, col: c };
}

1	//! Lexical scanner for the Radiance programming language.
2	//!
3	//! This module implements a hand-written scanner that tokenizes Radiance
4	//! source code into a stream of tokens for consumption by the parser.
5	@test mod tests;
6
7	use std::char;
8	use std::mem;
9	use std::lang::strings;
10
11	/// Token kinds representing all lexical elements in Radiance.
12	///
13	/// This enum covers operators, keywords, literals, and structural
14	/// elements used by the parser to build the AST.
15	export union TokenKind: Copy {
16	/// Special end of file token generated when the input is exhausted.
17	Eof,
18	/// Special invalid token.
19	Invalid,
20
21	LParen, // (
22	RParen, // )
23	LBrace, // {
24	RBrace, // }
25	LBracket, // [
26	RBracket, // ]
27	Comma, // ,
28	Dot, // .
29	DotDot, // ..
30	Minus, // -
31	Plus, // +
32	Colon, // :
33	ColonColon, // ::
34	Semicolon, // ;
35	Slash, // /
36	Star, // *
37	Percent, // %
38	Amp, // &
39	Pipe, // \|
40	Caret, // ^
41	Tilde, // ~
42	Underscore, // _
43	Question, // ?
44	Bang, // !
45	LtGt, // <>
46	Equal, // =
47	EqualEqual, // ==
48	Gt, // >
49	GtEqual, // >=
50	Lt, // <
51	LtEqual, // <=
52	LtLt, // <<
53	GtGt, // >>
54	Arrow, // ->
55	FatArrow, // =>
56
57	// Compound assignment operators.
58	PlusEqual, // +=
59	MinusEqual, // -=
60	StarEqual, // *=
61	SlashEqual, // /=
62	PercentEqual, // %=
63	AmpEqual, // &=
64	PipeEqual, // \|=
65	CaretEqual, // ^=
66	LtLtEqual, // <<=
67	GtGtEqual, // >>=
68
69	// Boolean operators.
70	Not, And, Or,
71
72	/// Eg. `fnord`
73	Ident,
74	/// Eg. `@default`
75	AtIdent,
76	/// Region name with an apostrophe prefix.
77	Region,
78	/// The `log` keyword.
79	Log,
80
81	// Literals.
82	String, // "fnord"
83	Char, // 'f'
84	Number, // 42
85	True, // true
86	False, // false
87	Nil, // nil
88	Undefined, // undefined
89
90	// Control flow tokens.
91	If, Else, Return, Break,
92	Continue, While, For, In,
93	Loop, Match, Case, Try, Catch,
94	Throw, Throws, Panic, Assert,
95
96	// Variable binding tokens.
97	Let, Mut, Set, Constant, Align,
98
99	// Module-related tokens.
100	Mod, Use, Super,
101
102	// Type or function attributes.
103	Export, Static,
104
105	// Trait-related tokens.
106	Trait, Instance,
107
108	// Type-related tokens.
109	I8, I16, I32, I64, U8, U16, U32, U64,
110	Opaque, Fn, Bool, Union, Record, As, Unsafe, Where
111	}
112
113	/// A reserved keyword.
114	record Keyword: Copy {
115	/// Keyword string.
116	name: *[u8],
117	/// Corresponding token.
118	tok: TokenKind,
119	}
120
121	/// Sorted keyword table for binary search.
122	constant KEYWORDS: [Keyword; 53] = [
123	{ name: "align", tok: TokenKind::Align },
124	{ name: "and", tok: TokenKind::And },
125	{ name: "as", tok: TokenKind::As },
126	{ name: "assert", tok: TokenKind::Assert },
127	{ name: "bool", tok: TokenKind::Bool },
128	{ name: "break", tok: TokenKind::Break },
129	{ name: "case", tok: TokenKind::Case },
130	{ name: "catch", tok: TokenKind::Catch },
131	{ name: "constant", tok: TokenKind::Constant },
132	{ name: "continue", tok: TokenKind::Continue },
133	{ name: "else", tok: TokenKind::Else },
134	{ name: "export", tok: TokenKind::Export },
135	{ name: "false", tok: TokenKind::False },
136	{ name: "fn", tok: TokenKind::Fn },
137	{ name: "for", tok: TokenKind::For },
138	{ name: "i16", tok: TokenKind::I16 },
139	{ name: "i32", tok: TokenKind::I32 },
140	{ name: "i64", tok: TokenKind::I64 },
141	{ name: "i8", tok: TokenKind::I8 },
142	{ name: "if", tok: TokenKind::If },
143	{ name: "in", tok: TokenKind::In },
144	{ name: "instance", tok: TokenKind::Instance },
145	{ name: "let", tok: TokenKind::Let },
146	{ name: "log", tok: TokenKind::Log },
147	{ name: "loop", tok: TokenKind::Loop },
148	{ name: "match", tok: TokenKind::Match },
149	{ name: "mod", tok: TokenKind::Mod },
150	{ name: "mut", tok: TokenKind::Mut },
151	{ name: "nil", tok: TokenKind::Nil },
152	{ name: "not", tok: TokenKind::Not },
153	{ name: "opaque", tok: TokenKind::Opaque },
154	{ name: "or", tok: TokenKind::Or },
155	{ name: "panic", tok: TokenKind::Panic },
156	{ name: "record", tok: TokenKind::Record },
157	{ name: "return", tok: TokenKind::Return },
158	{ name: "set", tok: TokenKind::Set },
159	{ name: "static", tok: TokenKind::Static },
160	{ name: "super", tok: TokenKind::Super },
161	{ name: "throw", tok: TokenKind::Throw },
162	{ name: "throws", tok: TokenKind::Throws },
163	{ name: "trait", tok: TokenKind::Trait },
164	{ name: "true", tok: TokenKind::True },
165	{ name: "try", tok: TokenKind::Try },
166	{ name: "u16", tok: TokenKind::U16 },
167	{ name: "u32", tok: TokenKind::U32 },
168	{ name: "u64", tok: TokenKind::U64 },
169	{ name: "u8", tok: TokenKind::U8 },
170	{ name: "undefined", tok: TokenKind::Undefined },
171	{ name: "union", tok: TokenKind::Union },
172	{ name: "unsafe", tok: TokenKind::Unsafe },
173	{ name: "use", tok: TokenKind::Use },
174	{ name: "where", tok: TokenKind::Where },
175	{ name: "while", tok: TokenKind::While },
176	];
177
178	/// Describes where source code originated from.
179	export union SourceLoc: Copy {
180	/// Source loaded from a file at the given path.
181	File(*[u8]),
182	/// Source provided as an inline string (no file path).
183	String,
184	}
185
186	/// Lexical scanner state for tokenizing Radiance source code.
187	///
188	/// Maintains position information and source buffer reference.
189	export record Scanner: Copy {
190	/// Origin of the source being scanned.
191	sourceLoc: SourceLoc,
192	/// Source buffer.
193	source: *[u8],
194	/// Offset of current token into buffer.
195	token: u32,
196	/// Offset of current character being scanned.
197	cursor: u32,
198	/// Interned string pool. It must remain valid while scanning.
199	pool: *unsafe mut strings::Pool,
200	}
201
202	/// Individual token with kind, source text, and position.
203	///
204	/// Represents a single lexical element extracted from source,
205	/// including its original text and byte offset for error reporting.
206	export record Token: Copy {
207	/// Token kind.
208	kind: TokenKind,
209	/// Token source string.
210	source: *[u8],
211	/// Byte offset of `source` in input buffer.
212	offset: u32,
213	}
214
215	/// Source code location with line/column information.
216	///
217	/// Used for error reporting and debugging.
218	export record Location: Copy {
219	/// Origin of the source.
220	source: SourceLoc,
221	/// Line number.
222	line: u16,
223	/// Column number.
224	col: u16,
225	}
226
227	/// Create a new scanner object.
228	export unsafe fn scanner(sourceLoc: SourceLoc, source: [u8], pool: unsafe mut strings::Pool) -> Scanner {
229	// Intern built-in functions and attributes.
230	strings::intern(pool, "@sizeOf");
231	strings::intern(pool, "@alignOf");
232	strings::intern(pool, "@sliceOf");
233	strings::intern(pool, "@default");
234	strings::intern(pool, "@intrinsic");
235	strings::intern(pool, "@test");
236	// Intern built-in slice methods.
237	strings::intern(pool, "append");
238	strings::intern(pool, "delete");
239
240	return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
241	}
242
243	/// Check if we've reached the end of input.
244	export fn isEof(s: &Scanner) -> bool {
245	return s.cursor >= s.source.len;
246	}
247
248	/// Get the current character, if any.
249	export fn current(s: &Scanner) -> ?u8 {
250	if isEof(s) {
251	return nil;
252	}
253	return s.source[s.cursor];
254	}
255
256	/// Peek at the next character without advancing the scanner.
257	fn peek(s: &Scanner) -> ?u8 {
258	if s.cursor + 1 >= s.source.len {
259	return nil;
260	}
261	return s.source[s.cursor + 1];
262	}
263
264	/// Advance scanner and return the character that was consumed.
265	fn advance(s: &mut Scanner) -> u8 {
266	let ch = s.source[s.cursor];
267	set s.cursor += 1;
268	return ch;
269	}
270
271	/// Consume the expected character if it matches the current position.
272	fn consume(s: &mut Scanner, expected: u8) -> bool {
273	if let c = current(s); c == expected {
274	advance(s);
275	return true;
276	}
277	return false;
278	}
279
280	/// Create a token from the current scanner state.
281	fn tok(s: &Scanner, kind: TokenKind) -> Token {
282	return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
283	}
284
285	/// Create an invalid token with the given message.
286	export fn invalid(offset: u32, message: *[u8]) -> Token {
287	return Token { kind: TokenKind::Invalid, source: message, offset };
288	}
289
290	/// Skip whitespace characters and line comments.
291	fn skipWhitespace(s: &mut Scanner) {
292	while let ch = current(s) {
293	match ch {
294	case ' ', '\n', '\r', '\t' => advance(s),
295	case '/' => {
296	if let c = peek(s); c == '/' {
297	while let ch = current(s); ch <> '\n' {
298	advance(s);
299	}
300	} else {
301	return;
302	}
303	}
304	else => return,
305	}
306	}
307	}
308
309	/// Scan numeric literal (decimal, hex, or binary).
310	fn scanNumber(s: &mut Scanner) -> Token {
311	// Check for hex literal (`0x` or `0X` prefix).
312	if s.source[s.cursor - 1] == '0' {
313	if let ch = current(s); ch == 'x' or ch == 'X' {
314	advance(s);
315	// Must have at least one hex digit after `0x`.
316	if let ch = current(s); not char::isHexDigit(ch) {
317	return invalid(s.token, "invalid hex literal");
318	}
319	while let ch = current(s); char::isHexDigit(ch) {
320	advance(s);
321	}
322	return tok(s, TokenKind::Number);
323	}
324	// Check for binary literal (`0b` or `0B` prefix).
325	if let ch = current(s); ch == 'b' or ch == 'B' {
326	advance(s);
327	// Must have at least one binary digit after `0b`.
328	if let ch = current(s); not char::isBinDigit(ch) {
329	return invalid(s.token, "invalid binary literal");
330	}
331	while let ch = current(s); char::isBinDigit(ch) {
332	advance(s);
333	}
334	return tok(s, TokenKind::Number);
335	}
336	}
337	// Regular decimal number.
338	while let ch = current(s); char::isDigit(ch) {
339	advance(s);
340	}
341	// Look for decimal part.
342	if let ch = current(s); ch == '.' {
343	if let p = peek(s); char::isDigit(p) {
344	advance(s); // Consume the "."
345	while let ch = current(s); char::isDigit(ch) {
346	advance(s);
347	}
348	}
349	}
350	return tok(s, TokenKind::Number);
351	}
352
353	fn scanDelimited(s: &mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
354	while let ch = current(s); ch <> delim {
355	if not char::isPrint(ch) {
356	return invalid(s.token, "invalid character");
357	}
358	if consume(s, '\\') { // Consume escapes
359	if isEof(s) {
360	return nil;
361	}
362	}
363	advance(s);
364	}
365	if not consume(s, delim) {
366	return nil;
367	}
368	return tok(s, kind);
369	}
370
371	/// Scan string literal enclosed in double quotes.
372	fn scanString(s: &mut Scanner) -> Token {
373	if let tok = scanDelimited(s, '"', TokenKind::String) {
374	return tok;
375	}
376	return invalid(s.token, "unterminated string");
377	}
378
379	/// Scan an apostrophe-prefixed region name or a quoted character literal.
380	fn scanChar(s: &mut Scanner) -> Token {
381	if let ch = current(s); char::isAlpha(ch) or ch == '_' {
382	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
383	advance(s);
384	}
385	if consume(s, '\'') {
386	return tok(s, TokenKind::Char);
387	}
388	return tok(s, TokenKind::Region);
389	}
390	if let tok = scanDelimited(s, '\'', TokenKind::Char) {
391	return tok;
392	}
393	return invalid(s.token, "unterminated character");
394	}
395
396	/// Scan a keyword or an identifier.
397	fn keywordOrIdent(src: *[u8]) -> TokenKind {
398	let mut left: u32 = 0;
399	let mut right: u32 = KEYWORDS.len;
400
401	while left < right {
402	let mid = left + ((right - left) / 2);
403	let kw = &KEYWORDS[mid];
404	let cmp = mem::cmp(src, kw.name);
405
406	match cmp {
407	case -1 => set right = mid,
408	case 1 => set left = mid + 1,
409	else => return kw.tok,
410	}
411	}
412	return TokenKind::Ident;
413	}
414
415	/// Scan an identifier, keyword, or label.
416	unsafe fn scanIdentifier(s: &mut Scanner) -> Token {
417	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
418	advance(s);
419	}
420	let ident = &s.source[s.token..s.cursor];
421	let kind = keywordOrIdent(ident);
422
423	// Only intern actual identifiers, not keywords.
424	if kind == TokenKind::Ident {
425	return Token { kind, source: strings::intern(s.pool, ident), offset: s.token };
426	}
427	return tok(s, kind);
428	}
429
430	/// Scan the next token. The retained string pool must be valid and writable.
431	export unsafe fn next(s: &mut Scanner) -> Token {
432	skipWhitespace(s); // Skip any whitespace between tokens.
433	set s.token = s.cursor; // Token starts at current position.
434
435	if isEof(s) {
436	return tok(s, TokenKind::Eof);
437	}
438	let c: u8 = advance(s);
439
440	if char::isDigit(c) {
441	return scanNumber(s);
442	}
443	if char::isAlpha(c) {
444	return scanIdentifier(s);
445	}
446	match c {
447	case '\'' => return scanChar(s),
448	case '"' => return scanString(s),
449	case '(' => return tok(s, TokenKind::LParen),
450	case ')' => return tok(s, TokenKind::RParen),
451	case '{' => return tok(s, TokenKind::LBrace),
452	case '}' => return tok(s, TokenKind::RBrace),
453	case '[' => return tok(s, TokenKind::LBracket),
454	case ']' => return tok(s, TokenKind::RBracket),
455	case ';' => return tok(s, TokenKind::Semicolon),
456	case ',' => return tok(s, TokenKind::Comma),
457	case '.' => {
458	if consume(s, '.') {
459	return tok(s, TokenKind::DotDot);
460	}
461	return tok(s, TokenKind::Dot);
462	}
463	case ':' => {
464	if consume(s, ':') {
465	return tok(s, TokenKind::ColonColon);
466	}
467	return tok(s, TokenKind::Colon);
468	}
469	case '-' => {
470	if consume(s, '>') {
471	return tok(s, TokenKind::Arrow);
472	}
473	if consume(s, '=') {
474	return tok(s, TokenKind::MinusEqual);
475	}
476	return tok(s, TokenKind::Minus);
477	}
478	case '+' => {
479	if consume(s, '=') {
480	return tok(s, TokenKind::PlusEqual);
481	}
482	return tok(s, TokenKind::Plus);
483	}
484	case '/' => {
485	if consume(s, '=') {
486	return tok(s, TokenKind::SlashEqual);
487	}
488	return tok(s, TokenKind::Slash);
489	}
490	case '*' => {
491	if consume(s, '=') {
492	return tok(s, TokenKind::StarEqual);
493	}
494	return tok(s, TokenKind::Star);
495	}
496	case '%' => {
497	if consume(s, '=') {
498	return tok(s, TokenKind::PercentEqual);
499	}
500	return tok(s, TokenKind::Percent);
501	}
502	case '&' => {
503	if consume(s, '=') {
504	return tok(s, TokenKind::AmpEqual);
505	}
506	return tok(s, TokenKind::Amp);
507	}
508	case '?' => return tok(s, TokenKind::Question),
509	case '\|' => {
510	if consume(s, '=') {
511	return tok(s, TokenKind::PipeEqual);
512	}
513	return tok(s, TokenKind::Pipe);
514	}
515	case '^' => {
516	if consume(s, '=') {
517	return tok(s, TokenKind::CaretEqual);
518	}
519	return tok(s, TokenKind::Caret);
520	}
521	case '~' => return tok(s, TokenKind::Tilde),
522	case '!' => return tok(s, TokenKind::Bang),
523	case '=' => {
524	if consume(s, '>') {
525	return tok(s, TokenKind::FatArrow);
526	}
527	if consume(s, '=') {
528	return tok(s, TokenKind::EqualEqual);
529	}
530	return tok(s, TokenKind::Equal);
531	}
532	case '<' => {
533	if consume(s, '>') {
534	return tok(s, TokenKind::LtGt);
535	}
536	if consume(s, '<') {
537	if consume(s, '=') {
538	return tok(s, TokenKind::LtLtEqual);
539	}
540	return tok(s, TokenKind::LtLt);
541	}
542	if consume(s, '=') {
543	return tok(s, TokenKind::LtEqual);
544	}
545	return tok(s, TokenKind::Lt);
546	}
547	case '>' => {
548	if consume(s, '>') {
549	if consume(s, '=') {
550	return tok(s, TokenKind::GtGtEqual);
551	}
552	return tok(s, TokenKind::GtGt);
553	}
554	if consume(s, '=') {
555	return tok(s, TokenKind::GtEqual);
556	}
557	return tok(s, TokenKind::Gt);
558	}
559	case '@' => {
560	// Scan `@identifier` as a single token.
561	let ch = current(s) else {
562	return invalid(s.token, "expected identifier after `@`");
563	};
564	if not char::isAlpha(ch) and ch <> '_' {
565	return invalid(s.token, "expected identifier after `@`");
566	}
567	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
568	advance(s);
569	}
570	let name = &s.source[s.token..s.cursor];
571	return Token {
572	kind: TokenKind::AtIdent,
573	source: strings::intern(s.pool, name),
574	offset: s.token,
575	};
576	}
577	case '_' => {
578	if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
579	// This is part of an identifier like `_foo` or `__start`
580	return scanIdentifier(s);
581	}
582	return tok(s, TokenKind::Underscore);
583	}
584	else => return invalid(s.token, "unexpected character"),
585	}
586	}
587
588	/// Get the source code location from a byte offset.
589	export fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
590	let mut l: u16 = 1;
591	let mut c: u16 = 1;
592
593	if offset >= source.len {
594	return nil;
595	}
596	for ch in &source[..offset] {
597	if ch == '\n' {
598	set c = 1;
599	set l += 1;
600	} else {
601	set c += 1;
602	}
603	}
604	return Location { source: sourceLoc, line: l, col: c };
605	}