radiance · Radiance self-hosting compiler

compiler/ lib/ examples/ std/ arch/ char/ collections/ lang/ alloc/ ast/ gen/ il/ module/ parser/ resolver/ scanner/ alloc.rad 4.3 KiB ast.rad 23.9 KiB gen.rad 507 B il.rad 15.3 KiB lower.rad 287.8 KiB module.rad 13.5 KiB package.rad 1.2 KiB parser.rad 82.9 KiB resolver.rad 407.8 KiB scanner.rad 18.3 KiB sexpr.rad 6.3 KiB strings.rad 2.2 KiB types.rad 280 B sys/ arch.rad 68 B char.rad 855 B collections.rad 39 B fmt.rad 8.1 KiB intrinsics.rad 467 B io.rad 1.3 KiB lang.rad 276 B mem.rad 2.2 KiB sys.rad 173 B testing.rad 2.4 KiB tests.rad 12.9 KiB vec.rad 1.7 KiB std.rad 281 B scripts/ seed/ sublime/ test/ vim/ .gitignore 336 B .gitsigners 112 B LICENSE 1.1 KiB Makefile 3.7 KiB README 4.8 KiB STYLE 2.5 KiB std.lib 1.2 KiB std.lib.test 347 B

lib/std/lang/scanner.rad 18.3 KiB raw

//! Lexical scanner for the Radiance programming language.
//!
//! This module implements a hand-written scanner that tokenizes Radiance
//! source code into a stream of tokens for consumption by the parser.
@test mod tests;

use std::char;
use std::mem;
use std::lang::strings;

/// Token kinds representing all lexical elements in Radiance.
///
/// This enum covers operators, keywords, literals, and structural
/// elements used by the parser to build the AST.
export union TokenKind {
    /// Special end of file token generated when the input is exhausted.
    Eof,
    /// Special invalid token.
    Invalid,

    LParen,     // (
    RParen,     // )
    LBrace,     // {
    RBrace,     // }
    LBracket,   // [
    RBracket,   // ]
    LAngle,     // U+27E8
    RAngle,     // U+27E9
    Comma,      // ,
    Dot,        // .
    DotDot,     // ..
    Minus,      // -
    Plus,       // +
    Colon,      // :
    ColonColon, // ::
    Semicolon,  // ;
    Slash,      // /
    Star,       // *
    Percent,    // %
    Amp,        // &
    Pipe,       // |
    Caret,      // ^
    Tilde,      // ~
    Underscore, // _
    Question,   // ?
    Bang,       // !
    LtGt,       // <>
    Equal,      // =
    EqualEqual, // ==
    Gt,         // >
    GtEqual,    // >=
    Lt,         // <
    LtEqual,    // <=
    LtLt,       // <<
    GtGt,       // >>
    Arrow,      // ->
    FatArrow,   // =>

    // Compound assignment operators.
    PlusEqual,    // +=
    MinusEqual,   // -=
    StarEqual,    // *=
    SlashEqual,   // /=
    PercentEqual, // %=
    AmpEqual,     // &=
    PipeEqual,    // |=
    CaretEqual,   // ^=
    LtLtEqual,    // <<=
    GtGtEqual,    // >>=

    // Boolean operators.
    Not, And, Or,

    /// Eg. `fnord`
    Ident,
    /// Eg. `@default`
    AtIdent,
    /// The `log` keyword.
    Log,

    // Literals.
    String,     // "fnord"
    Char,       // 'f'
    Number,     // 42
    True,       // true
    False,      // false
    Nil,        // nil
    Undefined,  // undefined

    // Control flow tokens.
    If, Else, Return, Break,
    Continue, While, For, In,
    Loop, Match, Case, Try, Catch,
    Throw, Throws, Panic, Assert,

    // Variable binding tokens.
    Let, Mut, Set, Constant, Align,

    // Module-related tokens.
    Mod, Use, Super,

    // Type or function attributes.
    Export, Static,

    // Trait and generic-related tokens.
    Trait, Instance, Instantiate,

    // Type-related tokens.
    I8, I16, I32, I64, U8, U16, U32, U64,
    Opaque, Fn, Bool, Union, Record, As, Unsafe
}

/// A reserved keyword.
record Keyword {
    /// Keyword string.
    name: *[u8],
    /// Corresponding token.
    tok: TokenKind,
}

/// Sorted keyword table for binary search.
constant KEYWORDS: [Keyword; 53] = [
    { name: "align", tok: TokenKind::Align },
    { name: "and", tok: TokenKind::And },
    { name: "as", tok: TokenKind::As },
    { name: "assert", tok: TokenKind::Assert },
    { name: "bool", tok: TokenKind::Bool },
    { name: "break", tok: TokenKind::Break },
    { name: "case", tok: TokenKind::Case },
    { name: "catch", tok: TokenKind::Catch },
    { name: "constant", tok: TokenKind::Constant },
    { name: "continue", tok: TokenKind::Continue },
    { name: "else", tok: TokenKind::Else },
    { name: "export", tok: TokenKind::Export },
    { name: "false", tok: TokenKind::False },
    { name: "fn", tok: TokenKind::Fn },
    { name: "for", tok: TokenKind::For },
    { name: "i16", tok: TokenKind::I16 },
    { name: "i32", tok: TokenKind::I32 },
    { name: "i64", tok: TokenKind::I64 },
    { name: "i8", tok: TokenKind::I8 },
    { name: "if", tok: TokenKind::If },
    { name: "in", tok: TokenKind::In },
    { name: "instance", tok: TokenKind::Instance },
    { name: "instantiate", tok: TokenKind::Instantiate },
    { name: "let", tok: TokenKind::Let },
    { name: "log", tok: TokenKind::Log },
    { name: "loop", tok: TokenKind::Loop },
    { name: "match", tok: TokenKind::Match },
    { name: "mod", tok: TokenKind::Mod },
    { name: "mut", tok: TokenKind::Mut },
    { name: "nil", tok: TokenKind::Nil },
    { name: "not", tok: TokenKind::Not },
    { name: "opaque", tok: TokenKind::Opaque },
    { name: "or", tok: TokenKind::Or },
    { name: "panic", tok: TokenKind::Panic },
    { name: "record", tok: TokenKind::Record },
    { name: "return", tok: TokenKind::Return },
    { name: "set", tok: TokenKind::Set },
    { name: "static", tok: TokenKind::Static },
    { name: "super", tok: TokenKind::Super },
    { name: "throw", tok: TokenKind::Throw },
    { name: "throws", tok: TokenKind::Throws },
    { name: "trait", tok: TokenKind::Trait },
    { name: "true", tok: TokenKind::True },
    { name: "try", tok: TokenKind::Try },
    { name: "u16", tok: TokenKind::U16 },
    { name: "u32", tok: TokenKind::U32 },
    { name: "u64", tok: TokenKind::U64 },
    { name: "u8", tok: TokenKind::U8 },
    { name: "undefined", tok: TokenKind::Undefined },
    { name: "union", tok: TokenKind::Union },
    { name: "unsafe", tok: TokenKind::Unsafe },
    { name: "use", tok: TokenKind::Use },
    { name: "while", tok: TokenKind::While },
];

/// Describes where source code originated from.
export union SourceLoc {
    /// Source loaded from a file at the given path.
    File(*[u8]),
    /// Source provided as an inline string (no file path).
    String,
}

/// Lexical scanner state for tokenizing Radiance source code.
///
/// Maintains position information and source buffer reference.
export record Scanner {
    /// Origin of the source being scanned.
    sourceLoc: SourceLoc,
    /// Source buffer.
    source: *[u8],
    /// Offset of current token into buffer.
    token: u32,
    /// Offset of current character being scanned.
    cursor: u32,
    /// Interned string pool.
    pool: *mut strings::Pool,
}

/// Individual token with kind, source text, and position.
///
/// Represents a single lexical element extracted from source,
/// including its original text and byte offset for error reporting.
export record Token {
    /// Token kind.
    kind: TokenKind,
    /// Token source string.
    source: *[u8],
    /// Byte offset of `source` in input buffer.
    offset: u32,
}

/// Source code location with line/column information.
///
/// Used for error reporting and debugging.
export record Location {
    /// Origin of the source.
    source: SourceLoc,
    /// Line number.
    line: u16,
    /// Column number.
    col: u16,
}

/// Create a new scanner object.
export fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: *mut strings::Pool) -> Scanner {
    // Intern built-in functions and attributes.
    strings::intern(pool, "@sizeOf");
    strings::intern(pool, "@alignOf");
    strings::intern(pool, "@sliceOf");
    strings::intern(pool, "@default");
    strings::intern(pool, "@intrinsic");
    strings::intern(pool, "@test");
    // Intern built-in slice methods.
    strings::intern(pool, "append");
    strings::intern(pool, "delete");

    return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
}

/// Check if we've reached the end of input.
export fn isEof(s: *Scanner) -> bool {
    return s.cursor >= s.source.len;
}

/// Get the current character, if any.
export fn current(s: *Scanner) -> ?u8 {
    if isEof(s) {
        return nil;
    }
    return s.source[s.cursor];
}

/// Peek at the next character without advancing the scanner.
fn peek(s: *Scanner) -> ?u8 {
    if s.cursor + 1 >= s.source.len {
        return nil;
    }
    return s.source[s.cursor + 1];
}

/// Advance scanner and return the character that was consumed.
fn advance(s: *mut Scanner) -> u8 {
    let ch = s.source[s.cursor];
    set s.cursor += 1;
    return ch;
}

/// Consume the expected character if it matches the current position.
fn consume(s: *mut Scanner, expected: u8) -> bool {
    if let c = current(s); c == expected {
        advance(s);
        return true;
    }
    return false;
}

/// Create a token from the current scanner state.
fn tok(s: *Scanner, kind: TokenKind) -> Token {
    return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
}

/// Create an invalid token with the given message.
export fn invalid(offset: u32, message: *[u8]) -> Token {
    return Token { kind: TokenKind::Invalid, source: message, offset };
}

/// Skip whitespace characters and line comments.
fn skipWhitespace(s: *mut Scanner) {
    while let ch = current(s) {
        match ch {
            case ' ', '\n', '\r', '\t' => advance(s),
            case '/' => {
                if let c = peek(s); c == '/' {
                    while let ch = current(s); ch <> '\n' {
                        advance(s);
                    }
                } else {
                    return;
                }
            }
            else => return,
        }
    }
}

/// Scan numeric literal (decimal, hex, or binary).
fn scanNumber(s: *mut Scanner) -> Token {
    // Check for hex literal (`0x` or `0X` prefix).
    if s.source[s.cursor - 1] == '0' {
        if let ch = current(s); ch == 'x' or ch == 'X' {
            advance(s);
            // Must have at least one hex digit after `0x`.
            if let ch = current(s); not char::isHexDigit(ch) {
                return invalid(s.token, "invalid hex literal");
            }
            while let ch = current(s); char::isHexDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
        // Check for binary literal (`0b` or `0B` prefix).
        if let ch = current(s); ch == 'b' or ch == 'B' {
            advance(s);
            // Must have at least one binary digit after `0b`.
            if let ch = current(s); not char::isBinDigit(ch) {
                return invalid(s.token, "invalid binary literal");
            }
            while let ch = current(s); char::isBinDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
    }
    // Regular decimal number.
    while let ch = current(s); char::isDigit(ch) {
        advance(s);
    }
    // Look for decimal part.
    if let ch = current(s); ch == '.' {
        if let p = peek(s); char::isDigit(p) {
            advance(s); // Consume the "."
            while let ch = current(s); char::isDigit(ch) {
                advance(s);
            }
        }
    }
    return tok(s, TokenKind::Number);
}

/// Scan a token through its closing delimiter.
fn scanDelimited(s: *mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
    while let ch = current(s); ch <> delim {
        if ch == 0xe2 and s.cursor + 2 < s.source.len and
           s.source[s.cursor + 1] == 0x9f and
           (s.source[s.cursor + 2] == 0xa8 or s.source[s.cursor + 2] == 0xa9)
        {
            set s.cursor += 3;
            continue;
        }
        if not char::isPrint(ch) {
            return invalid(s.token, "invalid character");
        }
        if consume(s, '\\') { // Consume escapes
            if isEof(s) {
                return nil;
            }
        }
        advance(s);
    }
    if not consume(s, delim) {
        return nil;
    }
    return tok(s, kind);
}

/// Scan string literal enclosed in double quotes.
fn scanString(s: *mut Scanner) -> Token {
    if let tok = scanDelimited(s, '"', TokenKind::String) {
        return tok;
    }
    return invalid(s.token, "unterminated string");
}

/// Scan character literal enclosed in single quotes.
fn scanChar(s: *mut Scanner) -> Token {
    if let tok = scanDelimited(s, '\'', TokenKind::Char) {
        return tok;
    }
    return invalid(s.token, "unterminated character");
}

/// Scan a keyword or an identifier.
fn keywordOrIdent(src: *[u8]) -> TokenKind {
    let mut left: u32 = 0;
    let mut right: u32 = KEYWORDS.len;

    while left < right {
        let mid = left + ((right - left) / 2);
        let kw = KEYWORDS[mid];
        let cmp = mem::cmp(src, kw.name);

        match cmp {
            case -1 => set right = mid,
            case 1 => set left = mid + 1,
            else => return kw.tok,
        }
    }
    return TokenKind::Ident;
}

/// Scan an identifier, keyword, or label.
fn scanIdentifier(s: *mut Scanner) -> Token {
    while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
        advance(s);
    }
    let ident = &s.source[s.token..s.cursor];
    let kind = keywordOrIdent(ident);

    // Only intern actual identifiers, not keywords.
    if kind == TokenKind::Ident {
        return Token { kind, source: strings::intern(s.pool, ident), offset: s.token };
    }
    return tok(s, kind);
}

/// Scan the next token.
export fn next(s: *mut Scanner) -> Token {
    skipWhitespace(s);  // Skip any whitespace between tokens.
    set s.token = s.cursor; // Token starts at current position.

    if isEof(s) {
        return tok(s, TokenKind::Eof);
    }
    let c: u8 = advance(s);

    if char::isDigit(c) {
        return scanNumber(s);
    }
    if char::isAlpha(c) {
        return scanIdentifier(s);
    }
    match c {
        case '\'' => return scanChar(s),
        case '"'  => return scanString(s),
        case '('  => return tok(s, TokenKind::LParen),
        case ')'  => return tok(s, TokenKind::RParen),
        case '{'  => return tok(s, TokenKind::LBrace),
        case '}'  => return tok(s, TokenKind::RBrace),
        case '['  => return tok(s, TokenKind::LBracket),
        case ']'  => return tok(s, TokenKind::RBracket),
        case 0xe2 => {
            // Scan one of the dedicated UTF-8 generic delimiters.
            if not consume(s, 0x9f) {
                return invalid(s.token, "unexpected UTF-8 character");
            }
            if consume(s, 0xa8) {
                return tok(s, TokenKind::LAngle);
            }
            if consume(s, 0xa9) {
                return tok(s, TokenKind::RAngle);
            }
            return invalid(s.token, "unexpected UTF-8 character");
        }
        case ';'  => return tok(s, TokenKind::Semicolon),
        case ','  => return tok(s, TokenKind::Comma),
        case '.'  => {
            if consume(s, '.') {
                return tok(s, TokenKind::DotDot);
            }
            return tok(s, TokenKind::Dot);
        }
        case ':'  => {
            if consume(s, ':') {
                return tok(s, TokenKind::ColonColon);
            }
            return tok(s, TokenKind::Colon);
        }
        case '-'  => {
            if consume(s, '>') {
                return tok(s, TokenKind::Arrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::MinusEqual);
            }
            return tok(s, TokenKind::Minus);
        }
        case '+' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PlusEqual);
            }
            return tok(s, TokenKind::Plus);
        }
        case '/' => {
            if consume(s, '=') {
                return tok(s, TokenKind::SlashEqual);
            }
            return tok(s, TokenKind::Slash);
        }
        case '*' => {
            if consume(s, '=') {
                return tok(s, TokenKind::StarEqual);
            }
            return tok(s, TokenKind::Star);
        }
        case '%' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PercentEqual);
            }
            return tok(s, TokenKind::Percent);
        }
        case '&' => {
            if consume(s, '=') {
                return tok(s, TokenKind::AmpEqual);
            }
            return tok(s, TokenKind::Amp);
        }
        case '?' => return tok(s, TokenKind::Question),
        case '|' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PipeEqual);
            }
            return tok(s, TokenKind::Pipe);
        }
        case '^' => {
            if consume(s, '=') {
                return tok(s, TokenKind::CaretEqual);
            }
            return tok(s, TokenKind::Caret);
        }
        case '~' => return tok(s, TokenKind::Tilde),
        case '!' => return tok(s, TokenKind::Bang),
        case '=' => {
            if consume(s, '>') {
                return tok(s, TokenKind::FatArrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::EqualEqual);
            }
            return tok(s, TokenKind::Equal);
        }
        case '<' => {
            if consume(s, '>') {
                return tok(s, TokenKind::LtGt);
            }
            if consume(s, '<') {
                if consume(s, '=') {
                    return tok(s, TokenKind::LtLtEqual);
                }
                return tok(s, TokenKind::LtLt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::LtEqual);
            }
            return tok(s, TokenKind::Lt);
        }
        case '>' => {
            if consume(s, '>') {
                if consume(s, '=') {
                    return tok(s, TokenKind::GtGtEqual);
                }
                return tok(s, TokenKind::GtGt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::GtEqual);
            }
            return tok(s, TokenKind::Gt);
        }
        case '@' => {
            // Scan `@identifier` as a single token.
            let ch = current(s) else {
                return invalid(s.token, "expected identifier after `@`");
            };
            if not char::isAlpha(ch) and ch <> '_' {
                return invalid(s.token, "expected identifier after `@`");
            }
            while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
                advance(s);
            }
            let name = &s.source[s.token..s.cursor];
            return Token {
                kind: TokenKind::AtIdent,
                source: strings::intern(s.pool, name),
                offset: s.token,
            };
        }
        case '_' => {
            if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
                // This is part of an identifier like `_foo` or `__start`
                return scanIdentifier(s);
            }
            return tok(s, TokenKind::Underscore);
        }
        else => return invalid(s.token, "unexpected character"),
    }
}

/// Get the source code location from a byte offset.
export fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
    let mut l: u16 = 1;
    let mut c: u16 = 1;

    if offset >= source.len {
        return nil;
    }
    for ch in &source[..offset] {
        if ch == '\n' {
            set c = 1;
            set l += 1;
        } else {
            set c += 1;
        }
    }
    return Location { source: sourceLoc, line: l, col: c };
}

1	//! Lexical scanner for the Radiance programming language.
2	//!
3	//! This module implements a hand-written scanner that tokenizes Radiance
4	//! source code into a stream of tokens for consumption by the parser.
5	@test mod tests;
6
7	use std::char;
8	use std::mem;
9	use std::lang::strings;
10
11	/// Token kinds representing all lexical elements in Radiance.
12	///
13	/// This enum covers operators, keywords, literals, and structural
14	/// elements used by the parser to build the AST.
15	export union TokenKind {
16	/// Special end of file token generated when the input is exhausted.
17	Eof,
18	/// Special invalid token.
19	Invalid,
20
21	LParen, // (
22	RParen, // )
23	LBrace, // {
24	RBrace, // }
25	LBracket, // [
26	RBracket, // ]
27	LAngle, // U+27E8
28	RAngle, // U+27E9
29	Comma, // ,
30	Dot, // .
31	DotDot, // ..
32	Minus, // -
33	Plus, // +
34	Colon, // :
35	ColonColon, // ::
36	Semicolon, // ;
37	Slash, // /
38	Star, // *
39	Percent, // %
40	Amp, // &
41	Pipe, // \|
42	Caret, // ^
43	Tilde, // ~
44	Underscore, // _
45	Question, // ?
46	Bang, // !
47	LtGt, // <>
48	Equal, // =
49	EqualEqual, // ==
50	Gt, // >
51	GtEqual, // >=
52	Lt, // <
53	LtEqual, // <=
54	LtLt, // <<
55	GtGt, // >>
56	Arrow, // ->
57	FatArrow, // =>
58
59	// Compound assignment operators.
60	PlusEqual, // +=
61	MinusEqual, // -=
62	StarEqual, // *=
63	SlashEqual, // /=
64	PercentEqual, // %=
65	AmpEqual, // &=
66	PipeEqual, // \|=
67	CaretEqual, // ^=
68	LtLtEqual, // <<=
69	GtGtEqual, // >>=
70
71	// Boolean operators.
72	Not, And, Or,
73
74	/// Eg. `fnord`
75	Ident,
76	/// Eg. `@default`
77	AtIdent,
78	/// The `log` keyword.
79	Log,
80
81	// Literals.
82	String, // "fnord"
83	Char, // 'f'
84	Number, // 42
85	True, // true
86	False, // false
87	Nil, // nil
88	Undefined, // undefined
89
90	// Control flow tokens.
91	If, Else, Return, Break,
92	Continue, While, For, In,
93	Loop, Match, Case, Try, Catch,
94	Throw, Throws, Panic, Assert,
95
96	// Variable binding tokens.
97	Let, Mut, Set, Constant, Align,
98
99	// Module-related tokens.
100	Mod, Use, Super,
101
102	// Type or function attributes.
103	Export, Static,
104
105	// Trait and generic-related tokens.
106	Trait, Instance, Instantiate,
107
108	// Type-related tokens.
109	I8, I16, I32, I64, U8, U16, U32, U64,
110	Opaque, Fn, Bool, Union, Record, As, Unsafe
111	}
112
113	/// A reserved keyword.
114	record Keyword {
115	/// Keyword string.
116	name: *[u8],
117	/// Corresponding token.
118	tok: TokenKind,
119	}
120
121	/// Sorted keyword table for binary search.
122	constant KEYWORDS: [Keyword; 53] = [
123	{ name: "align", tok: TokenKind::Align },
124	{ name: "and", tok: TokenKind::And },
125	{ name: "as", tok: TokenKind::As },
126	{ name: "assert", tok: TokenKind::Assert },
127	{ name: "bool", tok: TokenKind::Bool },
128	{ name: "break", tok: TokenKind::Break },
129	{ name: "case", tok: TokenKind::Case },
130	{ name: "catch", tok: TokenKind::Catch },
131	{ name: "constant", tok: TokenKind::Constant },
132	{ name: "continue", tok: TokenKind::Continue },
133	{ name: "else", tok: TokenKind::Else },
134	{ name: "export", tok: TokenKind::Export },
135	{ name: "false", tok: TokenKind::False },
136	{ name: "fn", tok: TokenKind::Fn },
137	{ name: "for", tok: TokenKind::For },
138	{ name: "i16", tok: TokenKind::I16 },
139	{ name: "i32", tok: TokenKind::I32 },
140	{ name: "i64", tok: TokenKind::I64 },
141	{ name: "i8", tok: TokenKind::I8 },
142	{ name: "if", tok: TokenKind::If },
143	{ name: "in", tok: TokenKind::In },
144	{ name: "instance", tok: TokenKind::Instance },
145	{ name: "instantiate", tok: TokenKind::Instantiate },
146	{ name: "let", tok: TokenKind::Let },
147	{ name: "log", tok: TokenKind::Log },
148	{ name: "loop", tok: TokenKind::Loop },
149	{ name: "match", tok: TokenKind::Match },
150	{ name: "mod", tok: TokenKind::Mod },
151	{ name: "mut", tok: TokenKind::Mut },
152	{ name: "nil", tok: TokenKind::Nil },
153	{ name: "not", tok: TokenKind::Not },
154	{ name: "opaque", tok: TokenKind::Opaque },
155	{ name: "or", tok: TokenKind::Or },
156	{ name: "panic", tok: TokenKind::Panic },
157	{ name: "record", tok: TokenKind::Record },
158	{ name: "return", tok: TokenKind::Return },
159	{ name: "set", tok: TokenKind::Set },
160	{ name: "static", tok: TokenKind::Static },
161	{ name: "super", tok: TokenKind::Super },
162	{ name: "throw", tok: TokenKind::Throw },
163	{ name: "throws", tok: TokenKind::Throws },
164	{ name: "trait", tok: TokenKind::Trait },
165	{ name: "true", tok: TokenKind::True },
166	{ name: "try", tok: TokenKind::Try },
167	{ name: "u16", tok: TokenKind::U16 },
168	{ name: "u32", tok: TokenKind::U32 },
169	{ name: "u64", tok: TokenKind::U64 },
170	{ name: "u8", tok: TokenKind::U8 },
171	{ name: "undefined", tok: TokenKind::Undefined },
172	{ name: "union", tok: TokenKind::Union },
173	{ name: "unsafe", tok: TokenKind::Unsafe },
174	{ name: "use", tok: TokenKind::Use },
175	{ name: "while", tok: TokenKind::While },
176	];
177
178	/// Describes where source code originated from.
179	export union SourceLoc {
180	/// Source loaded from a file at the given path.
181	File(*[u8]),
182	/// Source provided as an inline string (no file path).
183	String,
184	}
185
186	/// Lexical scanner state for tokenizing Radiance source code.
187	///
188	/// Maintains position information and source buffer reference.
189	export record Scanner {
190	/// Origin of the source being scanned.
191	sourceLoc: SourceLoc,
192	/// Source buffer.
193	source: *[u8],
194	/// Offset of current token into buffer.
195	token: u32,
196	/// Offset of current character being scanned.
197	cursor: u32,
198	/// Interned string pool.
199	pool: *mut strings::Pool,
200	}
201
202	/// Individual token with kind, source text, and position.
203	///
204	/// Represents a single lexical element extracted from source,
205	/// including its original text and byte offset for error reporting.
206	export record Token {
207	/// Token kind.
208	kind: TokenKind,
209	/// Token source string.
210	source: *[u8],
211	/// Byte offset of `source` in input buffer.
212	offset: u32,
213	}
214
215	/// Source code location with line/column information.
216	///
217	/// Used for error reporting and debugging.
218	export record Location {
219	/// Origin of the source.
220	source: SourceLoc,
221	/// Line number.
222	line: u16,
223	/// Column number.
224	col: u16,
225	}
226
227	/// Create a new scanner object.
228	export fn scanner(sourceLoc: SourceLoc, source: [u8], pool: mut strings::Pool) -> Scanner {
229	// Intern built-in functions and attributes.
230	strings::intern(pool, "@sizeOf");
231	strings::intern(pool, "@alignOf");
232	strings::intern(pool, "@sliceOf");
233	strings::intern(pool, "@default");
234	strings::intern(pool, "@intrinsic");
235	strings::intern(pool, "@test");
236	// Intern built-in slice methods.
237	strings::intern(pool, "append");
238	strings::intern(pool, "delete");
239
240	return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
241	}
242
243	/// Check if we've reached the end of input.
244	export fn isEof(s: *Scanner) -> bool {
245	return s.cursor >= s.source.len;
246	}
247
248	/// Get the current character, if any.
249	export fn current(s: *Scanner) -> ?u8 {
250	if isEof(s) {
251	return nil;
252	}
253	return s.source[s.cursor];
254	}
255
256	/// Peek at the next character without advancing the scanner.
257	fn peek(s: *Scanner) -> ?u8 {
258	if s.cursor + 1 >= s.source.len {
259	return nil;
260	}
261	return s.source[s.cursor + 1];
262	}
263
264	/// Advance scanner and return the character that was consumed.
265	fn advance(s: *mut Scanner) -> u8 {
266	let ch = s.source[s.cursor];
267	set s.cursor += 1;
268	return ch;
269	}
270
271	/// Consume the expected character if it matches the current position.
272	fn consume(s: *mut Scanner, expected: u8) -> bool {
273	if let c = current(s); c == expected {
274	advance(s);
275	return true;
276	}
277	return false;
278	}
279
280	/// Create a token from the current scanner state.
281	fn tok(s: *Scanner, kind: TokenKind) -> Token {
282	return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
283	}
284
285	/// Create an invalid token with the given message.
286	export fn invalid(offset: u32, message: *[u8]) -> Token {
287	return Token { kind: TokenKind::Invalid, source: message, offset };
288	}
289
290	/// Skip whitespace characters and line comments.
291	fn skipWhitespace(s: *mut Scanner) {
292	while let ch = current(s) {
293	match ch {
294	case ' ', '\n', '\r', '\t' => advance(s),
295	case '/' => {
296	if let c = peek(s); c == '/' {
297	while let ch = current(s); ch <> '\n' {
298	advance(s);
299	}
300	} else {
301	return;
302	}
303	}
304	else => return,
305	}
306	}
307	}
308
309	/// Scan numeric literal (decimal, hex, or binary).
310	fn scanNumber(s: *mut Scanner) -> Token {
311	// Check for hex literal (`0x` or `0X` prefix).
312	if s.source[s.cursor - 1] == '0' {
313	if let ch = current(s); ch == 'x' or ch == 'X' {
314	advance(s);
315	// Must have at least one hex digit after `0x`.
316	if let ch = current(s); not char::isHexDigit(ch) {
317	return invalid(s.token, "invalid hex literal");
318	}
319	while let ch = current(s); char::isHexDigit(ch) {
320	advance(s);
321	}
322	return tok(s, TokenKind::Number);
323	}
324	// Check for binary literal (`0b` or `0B` prefix).
325	if let ch = current(s); ch == 'b' or ch == 'B' {
326	advance(s);
327	// Must have at least one binary digit after `0b`.
328	if let ch = current(s); not char::isBinDigit(ch) {
329	return invalid(s.token, "invalid binary literal");
330	}
331	while let ch = current(s); char::isBinDigit(ch) {
332	advance(s);
333	}
334	return tok(s, TokenKind::Number);
335	}
336	}
337	// Regular decimal number.
338	while let ch = current(s); char::isDigit(ch) {
339	advance(s);
340	}
341	// Look for decimal part.
342	if let ch = current(s); ch == '.' {
343	if let p = peek(s); char::isDigit(p) {
344	advance(s); // Consume the "."
345	while let ch = current(s); char::isDigit(ch) {
346	advance(s);
347	}
348	}
349	}
350	return tok(s, TokenKind::Number);
351	}
352
353	/// Scan a token through its closing delimiter.
354	fn scanDelimited(s: *mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
355	while let ch = current(s); ch <> delim {
356	if ch == 0xe2 and s.cursor + 2 < s.source.len and
357	s.source[s.cursor + 1] == 0x9f and
358	(s.source[s.cursor + 2] == 0xa8 or s.source[s.cursor + 2] == 0xa9)
359	{
360	set s.cursor += 3;
361	continue;
362	}
363	if not char::isPrint(ch) {
364	return invalid(s.token, "invalid character");
365	}
366	if consume(s, '\\') { // Consume escapes
367	if isEof(s) {
368	return nil;
369	}
370	}
371	advance(s);
372	}
373	if not consume(s, delim) {
374	return nil;
375	}
376	return tok(s, kind);
377	}
378
379	/// Scan string literal enclosed in double quotes.
380	fn scanString(s: *mut Scanner) -> Token {
381	if let tok = scanDelimited(s, '"', TokenKind::String) {
382	return tok;
383	}
384	return invalid(s.token, "unterminated string");
385	}
386
387	/// Scan character literal enclosed in single quotes.
388	fn scanChar(s: *mut Scanner) -> Token {
389	if let tok = scanDelimited(s, '\'', TokenKind::Char) {
390	return tok;
391	}
392	return invalid(s.token, "unterminated character");
393	}
394
395	/// Scan a keyword or an identifier.
396	fn keywordOrIdent(src: *[u8]) -> TokenKind {
397	let mut left: u32 = 0;
398	let mut right: u32 = KEYWORDS.len;
399
400	while left < right {
401	let mid = left + ((right - left) / 2);
402	let kw = KEYWORDS[mid];
403	let cmp = mem::cmp(src, kw.name);
404
405	match cmp {
406	case -1 => set right = mid,
407	case 1 => set left = mid + 1,
408	else => return kw.tok,
409	}
410	}
411	return TokenKind::Ident;
412	}
413
414	/// Scan an identifier, keyword, or label.
415	fn scanIdentifier(s: *mut Scanner) -> Token {
416	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
417	advance(s);
418	}
419	let ident = &s.source[s.token..s.cursor];
420	let kind = keywordOrIdent(ident);
421
422	// Only intern actual identifiers, not keywords.
423	if kind == TokenKind::Ident {
424	return Token { kind, source: strings::intern(s.pool, ident), offset: s.token };
425	}
426	return tok(s, kind);
427	}
428
429	/// Scan the next token.
430	export fn next(s: *mut Scanner) -> Token {
431	skipWhitespace(s); // Skip any whitespace between tokens.
432	set s.token = s.cursor; // Token starts at current position.
433
434	if isEof(s) {
435	return tok(s, TokenKind::Eof);
436	}
437	let c: u8 = advance(s);
438
439	if char::isDigit(c) {
440	return scanNumber(s);
441	}
442	if char::isAlpha(c) {
443	return scanIdentifier(s);
444	}
445	match c {
446	case '\'' => return scanChar(s),
447	case '"' => return scanString(s),
448	case '(' => return tok(s, TokenKind::LParen),
449	case ')' => return tok(s, TokenKind::RParen),
450	case '{' => return tok(s, TokenKind::LBrace),
451	case '}' => return tok(s, TokenKind::RBrace),
452	case '[' => return tok(s, TokenKind::LBracket),
453	case ']' => return tok(s, TokenKind::RBracket),
454	case 0xe2 => {
455	// Scan one of the dedicated UTF-8 generic delimiters.
456	if not consume(s, 0x9f) {
457	return invalid(s.token, "unexpected UTF-8 character");
458	}
459	if consume(s, 0xa8) {
460	return tok(s, TokenKind::LAngle);
461	}
462	if consume(s, 0xa9) {
463	return tok(s, TokenKind::RAngle);
464	}
465	return invalid(s.token, "unexpected UTF-8 character");
466	}
467	case ';' => return tok(s, TokenKind::Semicolon),
468	case ',' => return tok(s, TokenKind::Comma),
469	case '.' => {
470	if consume(s, '.') {
471	return tok(s, TokenKind::DotDot);
472	}
473	return tok(s, TokenKind::Dot);
474	}
475	case ':' => {
476	if consume(s, ':') {
477	return tok(s, TokenKind::ColonColon);
478	}
479	return tok(s, TokenKind::Colon);
480	}
481	case '-' => {
482	if consume(s, '>') {
483	return tok(s, TokenKind::Arrow);
484	}
485	if consume(s, '=') {
486	return tok(s, TokenKind::MinusEqual);
487	}
488	return tok(s, TokenKind::Minus);
489	}
490	case '+' => {
491	if consume(s, '=') {
492	return tok(s, TokenKind::PlusEqual);
493	}
494	return tok(s, TokenKind::Plus);
495	}
496	case '/' => {
497	if consume(s, '=') {
498	return tok(s, TokenKind::SlashEqual);
499	}
500	return tok(s, TokenKind::Slash);
501	}
502	case '*' => {
503	if consume(s, '=') {
504	return tok(s, TokenKind::StarEqual);
505	}
506	return tok(s, TokenKind::Star);
507	}
508	case '%' => {
509	if consume(s, '=') {
510	return tok(s, TokenKind::PercentEqual);
511	}
512	return tok(s, TokenKind::Percent);
513	}
514	case '&' => {
515	if consume(s, '=') {
516	return tok(s, TokenKind::AmpEqual);
517	}
518	return tok(s, TokenKind::Amp);
519	}
520	case '?' => return tok(s, TokenKind::Question),
521	case '\|' => {
522	if consume(s, '=') {
523	return tok(s, TokenKind::PipeEqual);
524	}
525	return tok(s, TokenKind::Pipe);
526	}
527	case '^' => {
528	if consume(s, '=') {
529	return tok(s, TokenKind::CaretEqual);
530	}
531	return tok(s, TokenKind::Caret);
532	}
533	case '~' => return tok(s, TokenKind::Tilde),
534	case '!' => return tok(s, TokenKind::Bang),
535	case '=' => {
536	if consume(s, '>') {
537	return tok(s, TokenKind::FatArrow);
538	}
539	if consume(s, '=') {
540	return tok(s, TokenKind::EqualEqual);
541	}
542	return tok(s, TokenKind::Equal);
543	}
544	case '<' => {
545	if consume(s, '>') {
546	return tok(s, TokenKind::LtGt);
547	}
548	if consume(s, '<') {
549	if consume(s, '=') {
550	return tok(s, TokenKind::LtLtEqual);
551	}
552	return tok(s, TokenKind::LtLt);
553	}
554	if consume(s, '=') {
555	return tok(s, TokenKind::LtEqual);
556	}
557	return tok(s, TokenKind::Lt);
558	}
559	case '>' => {
560	if consume(s, '>') {
561	if consume(s, '=') {
562	return tok(s, TokenKind::GtGtEqual);
563	}
564	return tok(s, TokenKind::GtGt);
565	}
566	if consume(s, '=') {
567	return tok(s, TokenKind::GtEqual);
568	}
569	return tok(s, TokenKind::Gt);
570	}
571	case '@' => {
572	// Scan `@identifier` as a single token.
573	let ch = current(s) else {
574	return invalid(s.token, "expected identifier after `@`");
575	};
576	if not char::isAlpha(ch) and ch <> '_' {
577	return invalid(s.token, "expected identifier after `@`");
578	}
579	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
580	advance(s);
581	}
582	let name = &s.source[s.token..s.cursor];
583	return Token {
584	kind: TokenKind::AtIdent,
585	source: strings::intern(s.pool, name),
586	offset: s.token,
587	};
588	}
589	case '_' => {
590	if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
591	// This is part of an identifier like `_foo` or `__start`
592	return scanIdentifier(s);
593	}
594	return tok(s, TokenKind::Underscore);
595	}
596	else => return invalid(s.token, "unexpected character"),
597	}
598	}
599
600	/// Get the source code location from a byte offset.
601	export fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
602	let mut l: u16 = 1;
603	let mut c: u16 = 1;
604
605	if offset >= source.len {
606	return nil;
607	}
608	for ch in &source[..offset] {
609	if ch == '\n' {
610	set c = 1;
611	set l += 1;
612	} else {
613	set c += 1;
614	}
615	}
616	return Location { source: sourceLoc, line: l, col: c };
617	}