radiance · Radiance self-hosting compiler

compiler/ lib/ examples/ std/ arch/ char/ collections/ graph/ lang/ alloc/ ast/ gen/ il/ module/ parser/ resolver/ scanner/ alloc.rad 7.1 KiB ast.rad 26.9 KiB gen.rad 513 B il.rad 20.4 KiB lower.rad 321.7 KiB module.rad 17.3 KiB package.rad 1.3 KiB parser.rad 92.2 KiB resolver.rad 511.1 KiB scanner.rad 17.9 KiB sexpr.rad 6.7 KiB strings.rad 2.2 KiB types.rad 1.6 KiB sys/ arch.rad 68 B char.rad 855 B collections.rad 39 B fmt.rad 8.3 KiB graph.rad 4.3 KiB intrinsics.rad 467 B io.rad 1.7 KiB lang.rad 276 B mem.rad 2.3 KiB sys.rad 179 B testing.rad 2.4 KiB tests.rad 15.7 KiB vec.rad 3.2 KiB std.rad 299 B scripts/ seed/ sublime/ test/ vim/ .gitignore 336 B .gitsigners 112 B CELL_PERMISSIONS 6.8 KiB CONTRIBUTING 2.1 KiB LICENSE 1.1 KiB Makefile 5.4 KiB README 2.5 KiB STYLE 2.5 KiB std.lib 1.5 KiB std.lib.test 808 B

lib/std/lang/scanner.rad 17.9 KiB raw

//! Lexical scanner for the Radiance programming language.
//!
//! This module implements a hand-written scanner that tokenizes Radiance
//! source code into a stream of tokens for consumption by the parser.
@test mod tests;

use std::char;
use std::mem;
use std::lang::strings;

/// Token kinds representing all lexical elements in Radiance.
///
/// This enum covers operators, keywords, literals, and structural
/// elements used by the parser to build the AST.
export union TokenKind: Copy {
    /// Special end of file token generated when the input is exhausted.
    Eof,
    /// Special invalid token.
    Invalid,

    LParen,     // (
    RParen,     // )
    LBrace,     // {
    RBrace,     // }
    LBracket,   // [
    RBracket,   // ]
    Comma,      // ,
    Dot,        // .
    DotDot,     // ..
    Minus,      // -
    Plus,       // +
    Colon,      // :
    ColonColon, // ::
    Semicolon,  // ;
    Slash,      // /
    Star,       // *
    Percent,    // %
    Amp,        // &
    Pipe,       // |
    Caret,      // ^
    Tilde,      // ~
    Underscore, // _
    Question,   // ?
    Bang,       // !
    LtGt,       // <>
    Equal,      // =
    EqualEqual, // ==
    Gt,         // >
    GtEqual,    // >=
    Lt,         // <
    LtEqual,    // <=
    LtLt,       // <<
    GtGt,       // >>
    Arrow,      // ->
    FatArrow,   // =>

    // Compound assignment operators.
    PlusEqual,    // +=
    MinusEqual,   // -=
    StarEqual,    // *=
    SlashEqual,   // /=
    PercentEqual, // %=
    AmpEqual,     // &=
    PipeEqual,    // |=
    CaretEqual,   // ^=
    LtLtEqual,    // <<=
    GtGtEqual,    // >>=

    // Boolean operators.
    Not, And, Or,

    /// Eg. `fnord`
    Ident,
    /// Eg. `@default`
    AtIdent,
    /// Region name with an apostrophe prefix.
    Region,
    /// The `log` keyword.
    Log,

    // Literals.
    String,     // "fnord"
    Char,       // 'f'
    Number,     // 42
    True,       // true
    False,      // false
    Nil,        // nil
    Undefined,  // undefined

    // Control flow tokens.
    If, Else, Return, Break,
    Continue, While, For, In,
    Loop, Match, Case, Try, Catch,
    Throw, Throws, Panic, Assert,

    // Variable binding tokens.
    Let, Mut, Set, Constant, Align,

    // Module-related tokens.
    Mod, Use, Super,

    // Type or function attributes.
    Export, Static,

    // Trait-related tokens.
    Trait, Instance,

    // Type-related tokens.
    I8, I16, I32, I64, U8, U16, U32, U64,
    Opaque, Fn, Bool, Union, Record, As, Unsafe, Where
}

/// A reserved keyword.
record Keyword: Copy {
    /// Keyword string.
    name: *[u8],
    /// Corresponding token.
    tok: TokenKind,
}

/// Sorted keyword table for binary search.
constant KEYWORDS: [Keyword; 53] = [
    { name: "align", tok: TokenKind::Align },
    { name: "and", tok: TokenKind::And },
    { name: "as", tok: TokenKind::As },
    { name: "assert", tok: TokenKind::Assert },
    { name: "bool", tok: TokenKind::Bool },
    { name: "break", tok: TokenKind::Break },
    { name: "case", tok: TokenKind::Case },
    { name: "catch", tok: TokenKind::Catch },
    { name: "constant", tok: TokenKind::Constant },
    { name: "continue", tok: TokenKind::Continue },
    { name: "else", tok: TokenKind::Else },
    { name: "export", tok: TokenKind::Export },
    { name: "false", tok: TokenKind::False },
    { name: "fn", tok: TokenKind::Fn },
    { name: "for", tok: TokenKind::For },
    { name: "i16", tok: TokenKind::I16 },
    { name: "i32", tok: TokenKind::I32 },
    { name: "i64", tok: TokenKind::I64 },
    { name: "i8", tok: TokenKind::I8 },
    { name: "if", tok: TokenKind::If },
    { name: "in", tok: TokenKind::In },
    { name: "instance", tok: TokenKind::Instance },
    { name: "let", tok: TokenKind::Let },
    { name: "log", tok: TokenKind::Log },
    { name: "loop", tok: TokenKind::Loop },
    { name: "match", tok: TokenKind::Match },
    { name: "mod", tok: TokenKind::Mod },
    { name: "mut", tok: TokenKind::Mut },
    { name: "nil", tok: TokenKind::Nil },
    { name: "not", tok: TokenKind::Not },
    { name: "opaque", tok: TokenKind::Opaque },
    { name: "or", tok: TokenKind::Or },
    { name: "panic", tok: TokenKind::Panic },
    { name: "record", tok: TokenKind::Record },
    { name: "return", tok: TokenKind::Return },
    { name: "set", tok: TokenKind::Set },
    { name: "static", tok: TokenKind::Static },
    { name: "super", tok: TokenKind::Super },
    { name: "throw", tok: TokenKind::Throw },
    { name: "throws", tok: TokenKind::Throws },
    { name: "trait", tok: TokenKind::Trait },
    { name: "true", tok: TokenKind::True },
    { name: "try", tok: TokenKind::Try },
    { name: "u16", tok: TokenKind::U16 },
    { name: "u32", tok: TokenKind::U32 },
    { name: "u64", tok: TokenKind::U64 },
    { name: "u8", tok: TokenKind::U8 },
    { name: "undefined", tok: TokenKind::Undefined },
    { name: "union", tok: TokenKind::Union },
    { name: "unsafe", tok: TokenKind::Unsafe },
    { name: "use", tok: TokenKind::Use },
    { name: "where", tok: TokenKind::Where },
    { name: "while", tok: TokenKind::While },
];

/// Describes where source code originated from.
export union SourceLoc: Copy {
    /// Source loaded from a file at the given path.
    File(*[u8]),
    /// Source provided as an inline string (no file path).
    String,
}

/// Lexical scanner state for tokenizing Radiance source code.
///
/// Maintains position information and source buffer reference.
export record Scanner: Copy {
    /// Origin of the source being scanned.
    sourceLoc: SourceLoc,
    /// Source buffer.
    source: *[u8],
    /// Offset of current token into buffer.
    token: u32,
    /// Offset of current character being scanned.
    cursor: u32,
}

/// Individual token with kind, source text, and position.
///
/// Represents a single lexical element extracted from source,
/// including its original text and byte offset for error reporting.
export record Token: Copy {
    /// Token kind.
    kind: TokenKind,
    /// Token source string.
    source: *[u8],
    /// Byte offset of `source` in input buffer.
    offset: u32,
}

/// Source code location with line/column information.
///
/// Used for error reporting and debugging.
export record Location: Copy {
    /// Origin of the source.
    source: SourceLoc,
    /// Line number.
    line: u16,
    /// Column number.
    col: u16,
}

/// Create a new scanner object.
export fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: &mut strings::Pool) -> Scanner {
    // Intern built-in functions and attributes.
    strings::intern(pool, "@sizeOf");
    strings::intern(pool, "@alignOf");
    strings::intern(pool, "@sliceOf");
    strings::intern(pool, "@default");
    strings::intern(pool, "@intrinsic");
    strings::intern(pool, "@test");
    // Intern built-in slice methods.
    strings::intern(pool, "append");
    strings::intern(pool, "delete");

    return Scanner { sourceLoc, source, token: 0, cursor: 0 };
}

/// Check if we've reached the end of input.
export fn isEof(s: &Scanner) -> bool {
    return s.cursor >= s.source.len;
}

/// Get the current character, if any.
export fn current(s: &Scanner) -> ?u8 {
    if isEof(s) {
        return nil;
    }
    return s.source[s.cursor];
}

/// Peek at the next character without advancing the scanner.
fn peek(s: &Scanner) -> ?u8 {
    if s.cursor + 1 >= s.source.len {
        return nil;
    }
    return s.source[s.cursor + 1];
}

/// Advance scanner and return the character that was consumed.
fn advance(s: &mut Scanner) -> u8 {
    let ch = s.source[s.cursor];
    set s.cursor += 1;
    return ch;
}

/// Consume the expected character if it matches the current position.
fn consume(s: &mut Scanner, expected: u8) -> bool {
    if let c = current(s); c == expected {
        advance(s);
        return true;
    }
    return false;
}

/// Create a token from the current scanner state.
fn tok(s: &Scanner, kind: TokenKind) -> Token {
    return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
}

/// Create an invalid token with the given message.
export fn invalid(offset: u32, message: *[u8]) -> Token {
    return Token { kind: TokenKind::Invalid, source: message, offset };
}

/// Skip whitespace characters and line comments.
fn skipWhitespace(s: &mut Scanner) {
    while let ch = current(s) {
        match ch {
            case ' ', '\n', '\r', '\t' => advance(s),
            case '/' => {
                if let c = peek(s); c == '/' {
                    while let ch = current(s); ch <> '\n' {
                        advance(s);
                    }
                } else {
                    return;
                }
            }
            else => return,
        }
    }
}

/// Scan numeric literal (decimal, hex, or binary).
fn scanNumber(s: &mut Scanner) -> Token {
    // Check for hex literal (`0x` or `0X` prefix).
    if s.source[s.cursor - 1] == '0' {
        if let ch = current(s); ch == 'x' or ch == 'X' {
            advance(s);
            // Must have at least one hex digit after `0x`.
            if let ch = current(s); not char::isHexDigit(ch) {
                return invalid(s.token, "invalid hex literal");
            }
            while let ch = current(s); char::isHexDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
        // Check for binary literal (`0b` or `0B` prefix).
        if let ch = current(s); ch == 'b' or ch == 'B' {
            advance(s);
            // Must have at least one binary digit after `0b`.
            if let ch = current(s); not char::isBinDigit(ch) {
                return invalid(s.token, "invalid binary literal");
            }
            while let ch = current(s); char::isBinDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
    }
    // Regular decimal number.
    while let ch = current(s); char::isDigit(ch) {
        advance(s);
    }
    // Look for decimal part.
    if let ch = current(s); ch == '.' {
        if let p = peek(s); char::isDigit(p) {
            advance(s); // Consume the "."
            while let ch = current(s); char::isDigit(ch) {
                advance(s);
            }
        }
    }
    return tok(s, TokenKind::Number);
}

fn scanDelimited(s: &mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
    while let ch = current(s); ch <> delim {
        if not char::isPrint(ch) {
            return invalid(s.token, "invalid character");
        }
        if consume(s, '\\') { // Consume escapes
            if isEof(s) {
                return nil;
            }
        }
        advance(s);
    }
    if not consume(s, delim) {
        return nil;
    }
    return tok(s, kind);
}

/// Scan string literal enclosed in double quotes.
fn scanString(s: &mut Scanner) -> Token {
    if let tok = scanDelimited(s, '"', TokenKind::String) {
        return tok;
    }
    return invalid(s.token, "unterminated string");
}

/// Scan an apostrophe-prefixed region name or a quoted character literal.
fn scanChar(s: &mut Scanner) -> Token {
    if let ch = current(s); char::isAlpha(ch) or ch == '_' {
        while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
            advance(s);
        }
        if consume(s, '\'') {
            return tok(s, TokenKind::Char);
        }
        return tok(s, TokenKind::Region);
    }
    if let tok = scanDelimited(s, '\'', TokenKind::Char) {
        return tok;
    }
    return invalid(s.token, "unterminated character");
}

/// Scan a keyword or an identifier.
fn keywordOrIdent(src: *[u8]) -> TokenKind {
    let mut left: u32 = 0;
    let mut right: u32 = KEYWORDS.len;

    while left < right {
        let mid = left + ((right - left) / 2);
        let kw = &KEYWORDS[mid];
        let cmp = mem::cmp(src, kw.name);

        match cmp {
            case -1 => set right = mid,
            case 1 => set left = mid + 1,
            else => return kw.tok,
        }
    }
    return TokenKind::Ident;
}

/// Scan an identifier, keyword, or label.
fn scanIdentifier(s: &mut Scanner, pool: &mut strings::Pool) -> Token {
    while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
        advance(s);
    }
    let ident = &s.source[s.token..s.cursor];
    let kind = keywordOrIdent(ident);

    // Only intern actual identifiers, not keywords.
    if kind == TokenKind::Ident {
        return Token { kind, source: strings::intern(pool, ident), offset: s.token };
    }
    return tok(s, kind);
}

/// Scan the next token and intern identifiers in the supplied pool.
export fn next(s: &mut Scanner, pool: &mut strings::Pool) -> Token {
    skipWhitespace(s);  // Skip any whitespace between tokens.
    set s.token = s.cursor; // Token starts at current position.

    if isEof(s) {
        return tok(s, TokenKind::Eof);
    }
    let c: u8 = advance(s);

    if char::isDigit(c) {
        return scanNumber(s);
    }
    if char::isAlpha(c) {
        return scanIdentifier(s, pool);
    }
    match c {
        case '\'' => return scanChar(s),
        case '"'  => return scanString(s),
        case '('  => return tok(s, TokenKind::LParen),
        case ')'  => return tok(s, TokenKind::RParen),
        case '{'  => return tok(s, TokenKind::LBrace),
        case '}'  => return tok(s, TokenKind::RBrace),
        case '['  => return tok(s, TokenKind::LBracket),
        case ']'  => return tok(s, TokenKind::RBracket),
        case ';'  => return tok(s, TokenKind::Semicolon),
        case ','  => return tok(s, TokenKind::Comma),
        case '.'  => {
            if consume(s, '.') {
                return tok(s, TokenKind::DotDot);
            }
            return tok(s, TokenKind::Dot);
        }
        case ':'  => {
            if consume(s, ':') {
                return tok(s, TokenKind::ColonColon);
            }
            return tok(s, TokenKind::Colon);
        }
        case '-'  => {
            if consume(s, '>') {
                return tok(s, TokenKind::Arrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::MinusEqual);
            }
            return tok(s, TokenKind::Minus);
        }
        case '+' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PlusEqual);
            }
            return tok(s, TokenKind::Plus);
        }
        case '/' => {
            if consume(s, '=') {
                return tok(s, TokenKind::SlashEqual);
            }
            return tok(s, TokenKind::Slash);
        }
        case '*' => {
            if consume(s, '=') {
                return tok(s, TokenKind::StarEqual);
            }
            return tok(s, TokenKind::Star);
        }
        case '%' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PercentEqual);
            }
            return tok(s, TokenKind::Percent);
        }
        case '&' => {
            if consume(s, '=') {
                return tok(s, TokenKind::AmpEqual);
            }
            return tok(s, TokenKind::Amp);
        }
        case '?' => return tok(s, TokenKind::Question),
        case '|' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PipeEqual);
            }
            return tok(s, TokenKind::Pipe);
        }
        case '^' => {
            if consume(s, '=') {
                return tok(s, TokenKind::CaretEqual);
            }
            return tok(s, TokenKind::Caret);
        }
        case '~' => return tok(s, TokenKind::Tilde),
        case '!' => return tok(s, TokenKind::Bang),
        case '=' => {
            if consume(s, '>') {
                return tok(s, TokenKind::FatArrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::EqualEqual);
            }
            return tok(s, TokenKind::Equal);
        }
        case '<' => {
            if consume(s, '>') {
                return tok(s, TokenKind::LtGt);
            }
            if consume(s, '<') {
                if consume(s, '=') {
                    return tok(s, TokenKind::LtLtEqual);
                }
                return tok(s, TokenKind::LtLt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::LtEqual);
            }
            return tok(s, TokenKind::Lt);
        }
        case '>' => {
            if consume(s, '>') {
                if consume(s, '=') {
                    return tok(s, TokenKind::GtGtEqual);
                }
                return tok(s, TokenKind::GtGt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::GtEqual);
            }
            return tok(s, TokenKind::Gt);
        }
        case '@' => {
            // Scan `@identifier` as a single token.
            let ch = current(s) else {
                return invalid(s.token, "expected identifier after `@`");
            };
            if not char::isAlpha(ch) and ch <> '_' {
                return invalid(s.token, "expected identifier after `@`");
            }
            while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
                advance(s);
            }
            let name = &s.source[s.token..s.cursor];
            return Token {
                kind: TokenKind::AtIdent,
                source: strings::intern(pool, name),
                offset: s.token,
            };
        }
        case '_' => {
            if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
                // This is part of an identifier like `_foo` or `__start`
                return scanIdentifier(s, pool);
            }
            return tok(s, TokenKind::Underscore);
        }
        else => return invalid(s.token, "unexpected character"),
    }
}

/// Get the source code location from a byte offset.
export fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
    let mut l: u16 = 1;
    let mut c: u16 = 1;

    if offset >= source.len {
        return nil;
    }
    for ch in &source[..offset] {
        if ch == '\n' {
            set c = 1;
            set l += 1;
        } else {
            set c += 1;
        }
    }
    return Location { source: sourceLoc, line: l, col: c };
}

1	//! Lexical scanner for the Radiance programming language.
2	//!
3	//! This module implements a hand-written scanner that tokenizes Radiance
4	//! source code into a stream of tokens for consumption by the parser.
5	@test mod tests;
6
7	use std::char;
8	use std::mem;
9	use std::lang::strings;
10
11	/// Token kinds representing all lexical elements in Radiance.
12	///
13	/// This enum covers operators, keywords, literals, and structural
14	/// elements used by the parser to build the AST.
15	export union TokenKind: Copy {
16	/// Special end of file token generated when the input is exhausted.
17	Eof,
18	/// Special invalid token.
19	Invalid,
20
21	LParen, // (
22	RParen, // )
23	LBrace, // {
24	RBrace, // }
25	LBracket, // [
26	RBracket, // ]
27	Comma, // ,
28	Dot, // .
29	DotDot, // ..
30	Minus, // -
31	Plus, // +
32	Colon, // :
33	ColonColon, // ::
34	Semicolon, // ;
35	Slash, // /
36	Star, // *
37	Percent, // %
38	Amp, // &
39	Pipe, // \|
40	Caret, // ^
41	Tilde, // ~
42	Underscore, // _
43	Question, // ?
44	Bang, // !
45	LtGt, // <>
46	Equal, // =
47	EqualEqual, // ==
48	Gt, // >
49	GtEqual, // >=
50	Lt, // <
51	LtEqual, // <=
52	LtLt, // <<
53	GtGt, // >>
54	Arrow, // ->
55	FatArrow, // =>
56
57	// Compound assignment operators.
58	PlusEqual, // +=
59	MinusEqual, // -=
60	StarEqual, // *=
61	SlashEqual, // /=
62	PercentEqual, // %=
63	AmpEqual, // &=
64	PipeEqual, // \|=
65	CaretEqual, // ^=
66	LtLtEqual, // <<=
67	GtGtEqual, // >>=
68
69	// Boolean operators.
70	Not, And, Or,
71
72	/// Eg. `fnord`
73	Ident,
74	/// Eg. `@default`
75	AtIdent,
76	/// Region name with an apostrophe prefix.
77	Region,
78	/// The `log` keyword.
79	Log,
80
81	// Literals.
82	String, // "fnord"
83	Char, // 'f'
84	Number, // 42
85	True, // true
86	False, // false
87	Nil, // nil
88	Undefined, // undefined
89
90	// Control flow tokens.
91	If, Else, Return, Break,
92	Continue, While, For, In,
93	Loop, Match, Case, Try, Catch,
94	Throw, Throws, Panic, Assert,
95
96	// Variable binding tokens.
97	Let, Mut, Set, Constant, Align,
98
99	// Module-related tokens.
100	Mod, Use, Super,
101
102	// Type or function attributes.
103	Export, Static,
104
105	// Trait-related tokens.
106	Trait, Instance,
107
108	// Type-related tokens.
109	I8, I16, I32, I64, U8, U16, U32, U64,
110	Opaque, Fn, Bool, Union, Record, As, Unsafe, Where
111	}
112
113	/// A reserved keyword.
114	record Keyword: Copy {
115	/// Keyword string.
116	name: *[u8],
117	/// Corresponding token.
118	tok: TokenKind,
119	}
120
121	/// Sorted keyword table for binary search.
122	constant KEYWORDS: [Keyword; 53] = [
123	{ name: "align", tok: TokenKind::Align },
124	{ name: "and", tok: TokenKind::And },
125	{ name: "as", tok: TokenKind::As },
126	{ name: "assert", tok: TokenKind::Assert },
127	{ name: "bool", tok: TokenKind::Bool },
128	{ name: "break", tok: TokenKind::Break },
129	{ name: "case", tok: TokenKind::Case },
130	{ name: "catch", tok: TokenKind::Catch },
131	{ name: "constant", tok: TokenKind::Constant },
132	{ name: "continue", tok: TokenKind::Continue },
133	{ name: "else", tok: TokenKind::Else },
134	{ name: "export", tok: TokenKind::Export },
135	{ name: "false", tok: TokenKind::False },
136	{ name: "fn", tok: TokenKind::Fn },
137	{ name: "for", tok: TokenKind::For },
138	{ name: "i16", tok: TokenKind::I16 },
139	{ name: "i32", tok: TokenKind::I32 },
140	{ name: "i64", tok: TokenKind::I64 },
141	{ name: "i8", tok: TokenKind::I8 },
142	{ name: "if", tok: TokenKind::If },
143	{ name: "in", tok: TokenKind::In },
144	{ name: "instance", tok: TokenKind::Instance },
145	{ name: "let", tok: TokenKind::Let },
146	{ name: "log", tok: TokenKind::Log },
147	{ name: "loop", tok: TokenKind::Loop },
148	{ name: "match", tok: TokenKind::Match },
149	{ name: "mod", tok: TokenKind::Mod },
150	{ name: "mut", tok: TokenKind::Mut },
151	{ name: "nil", tok: TokenKind::Nil },
152	{ name: "not", tok: TokenKind::Not },
153	{ name: "opaque", tok: TokenKind::Opaque },
154	{ name: "or", tok: TokenKind::Or },
155	{ name: "panic", tok: TokenKind::Panic },
156	{ name: "record", tok: TokenKind::Record },
157	{ name: "return", tok: TokenKind::Return },
158	{ name: "set", tok: TokenKind::Set },
159	{ name: "static", tok: TokenKind::Static },
160	{ name: "super", tok: TokenKind::Super },
161	{ name: "throw", tok: TokenKind::Throw },
162	{ name: "throws", tok: TokenKind::Throws },
163	{ name: "trait", tok: TokenKind::Trait },
164	{ name: "true", tok: TokenKind::True },
165	{ name: "try", tok: TokenKind::Try },
166	{ name: "u16", tok: TokenKind::U16 },
167	{ name: "u32", tok: TokenKind::U32 },
168	{ name: "u64", tok: TokenKind::U64 },
169	{ name: "u8", tok: TokenKind::U8 },
170	{ name: "undefined", tok: TokenKind::Undefined },
171	{ name: "union", tok: TokenKind::Union },
172	{ name: "unsafe", tok: TokenKind::Unsafe },
173	{ name: "use", tok: TokenKind::Use },
174	{ name: "where", tok: TokenKind::Where },
175	{ name: "while", tok: TokenKind::While },
176	];
177
178	/// Describes where source code originated from.
179	export union SourceLoc: Copy {
180	/// Source loaded from a file at the given path.
181	File(*[u8]),
182	/// Source provided as an inline string (no file path).
183	String,
184	}
185
186	/// Lexical scanner state for tokenizing Radiance source code.
187	///
188	/// Maintains position information and source buffer reference.
189	export record Scanner: Copy {
190	/// Origin of the source being scanned.
191	sourceLoc: SourceLoc,
192	/// Source buffer.
193	source: *[u8],
194	/// Offset of current token into buffer.
195	token: u32,
196	/// Offset of current character being scanned.
197	cursor: u32,
198	}
199
200	/// Individual token with kind, source text, and position.
201	///
202	/// Represents a single lexical element extracted from source,
203	/// including its original text and byte offset for error reporting.
204	export record Token: Copy {
205	/// Token kind.
206	kind: TokenKind,
207	/// Token source string.
208	source: *[u8],
209	/// Byte offset of `source` in input buffer.
210	offset: u32,
211	}
212
213	/// Source code location with line/column information.
214	///
215	/// Used for error reporting and debugging.
216	export record Location: Copy {
217	/// Origin of the source.
218	source: SourceLoc,
219	/// Line number.
220	line: u16,
221	/// Column number.
222	col: u16,
223	}
224
225	/// Create a new scanner object.
226	export fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: &mut strings::Pool) -> Scanner {
227	// Intern built-in functions and attributes.
228	strings::intern(pool, "@sizeOf");
229	strings::intern(pool, "@alignOf");
230	strings::intern(pool, "@sliceOf");
231	strings::intern(pool, "@default");
232	strings::intern(pool, "@intrinsic");
233	strings::intern(pool, "@test");
234	// Intern built-in slice methods.
235	strings::intern(pool, "append");
236	strings::intern(pool, "delete");
237
238	return Scanner { sourceLoc, source, token: 0, cursor: 0 };
239	}
240
241	/// Check if we've reached the end of input.
242	export fn isEof(s: &Scanner) -> bool {
243	return s.cursor >= s.source.len;
244	}
245
246	/// Get the current character, if any.
247	export fn current(s: &Scanner) -> ?u8 {
248	if isEof(s) {
249	return nil;
250	}
251	return s.source[s.cursor];
252	}
253
254	/// Peek at the next character without advancing the scanner.
255	fn peek(s: &Scanner) -> ?u8 {
256	if s.cursor + 1 >= s.source.len {
257	return nil;
258	}
259	return s.source[s.cursor + 1];
260	}
261
262	/// Advance scanner and return the character that was consumed.
263	fn advance(s: &mut Scanner) -> u8 {
264	let ch = s.source[s.cursor];
265	set s.cursor += 1;
266	return ch;
267	}
268
269	/// Consume the expected character if it matches the current position.
270	fn consume(s: &mut Scanner, expected: u8) -> bool {
271	if let c = current(s); c == expected {
272	advance(s);
273	return true;
274	}
275	return false;
276	}
277
278	/// Create a token from the current scanner state.
279	fn tok(s: &Scanner, kind: TokenKind) -> Token {
280	return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
281	}
282
283	/// Create an invalid token with the given message.
284	export fn invalid(offset: u32, message: *[u8]) -> Token {
285	return Token { kind: TokenKind::Invalid, source: message, offset };
286	}
287
288	/// Skip whitespace characters and line comments.
289	fn skipWhitespace(s: &mut Scanner) {
290	while let ch = current(s) {
291	match ch {
292	case ' ', '\n', '\r', '\t' => advance(s),
293	case '/' => {
294	if let c = peek(s); c == '/' {
295	while let ch = current(s); ch <> '\n' {
296	advance(s);
297	}
298	} else {
299	return;
300	}
301	}
302	else => return,
303	}
304	}
305	}
306
307	/// Scan numeric literal (decimal, hex, or binary).
308	fn scanNumber(s: &mut Scanner) -> Token {
309	// Check for hex literal (`0x` or `0X` prefix).
310	if s.source[s.cursor - 1] == '0' {
311	if let ch = current(s); ch == 'x' or ch == 'X' {
312	advance(s);
313	// Must have at least one hex digit after `0x`.
314	if let ch = current(s); not char::isHexDigit(ch) {
315	return invalid(s.token, "invalid hex literal");
316	}
317	while let ch = current(s); char::isHexDigit(ch) {
318	advance(s);
319	}
320	return tok(s, TokenKind::Number);
321	}
322	// Check for binary literal (`0b` or `0B` prefix).
323	if let ch = current(s); ch == 'b' or ch == 'B' {
324	advance(s);
325	// Must have at least one binary digit after `0b`.
326	if let ch = current(s); not char::isBinDigit(ch) {
327	return invalid(s.token, "invalid binary literal");
328	}
329	while let ch = current(s); char::isBinDigit(ch) {
330	advance(s);
331	}
332	return tok(s, TokenKind::Number);
333	}
334	}
335	// Regular decimal number.
336	while let ch = current(s); char::isDigit(ch) {
337	advance(s);
338	}
339	// Look for decimal part.
340	if let ch = current(s); ch == '.' {
341	if let p = peek(s); char::isDigit(p) {
342	advance(s); // Consume the "."
343	while let ch = current(s); char::isDigit(ch) {
344	advance(s);
345	}
346	}
347	}
348	return tok(s, TokenKind::Number);
349	}
350
351	fn scanDelimited(s: &mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
352	while let ch = current(s); ch <> delim {
353	if not char::isPrint(ch) {
354	return invalid(s.token, "invalid character");
355	}
356	if consume(s, '\\') { // Consume escapes
357	if isEof(s) {
358	return nil;
359	}
360	}
361	advance(s);
362	}
363	if not consume(s, delim) {
364	return nil;
365	}
366	return tok(s, kind);
367	}
368
369	/// Scan string literal enclosed in double quotes.
370	fn scanString(s: &mut Scanner) -> Token {
371	if let tok = scanDelimited(s, '"', TokenKind::String) {
372	return tok;
373	}
374	return invalid(s.token, "unterminated string");
375	}
376
377	/// Scan an apostrophe-prefixed region name or a quoted character literal.
378	fn scanChar(s: &mut Scanner) -> Token {
379	if let ch = current(s); char::isAlpha(ch) or ch == '_' {
380	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
381	advance(s);
382	}
383	if consume(s, '\'') {
384	return tok(s, TokenKind::Char);
385	}
386	return tok(s, TokenKind::Region);
387	}
388	if let tok = scanDelimited(s, '\'', TokenKind::Char) {
389	return tok;
390	}
391	return invalid(s.token, "unterminated character");
392	}
393
394	/// Scan a keyword or an identifier.
395	fn keywordOrIdent(src: *[u8]) -> TokenKind {
396	let mut left: u32 = 0;
397	let mut right: u32 = KEYWORDS.len;
398
399	while left < right {
400	let mid = left + ((right - left) / 2);
401	let kw = &KEYWORDS[mid];
402	let cmp = mem::cmp(src, kw.name);
403
404	match cmp {
405	case -1 => set right = mid,
406	case 1 => set left = mid + 1,
407	else => return kw.tok,
408	}
409	}
410	return TokenKind::Ident;
411	}
412
413	/// Scan an identifier, keyword, or label.
414	fn scanIdentifier(s: &mut Scanner, pool: &mut strings::Pool) -> Token {
415	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
416	advance(s);
417	}
418	let ident = &s.source[s.token..s.cursor];
419	let kind = keywordOrIdent(ident);
420
421	// Only intern actual identifiers, not keywords.
422	if kind == TokenKind::Ident {
423	return Token { kind, source: strings::intern(pool, ident), offset: s.token };
424	}
425	return tok(s, kind);
426	}
427
428	/// Scan the next token and intern identifiers in the supplied pool.
429	export fn next(s: &mut Scanner, pool: &mut strings::Pool) -> Token {
430	skipWhitespace(s); // Skip any whitespace between tokens.
431	set s.token = s.cursor; // Token starts at current position.
432
433	if isEof(s) {
434	return tok(s, TokenKind::Eof);
435	}
436	let c: u8 = advance(s);
437
438	if char::isDigit(c) {
439	return scanNumber(s);
440	}
441	if char::isAlpha(c) {
442	return scanIdentifier(s, pool);
443	}
444	match c {
445	case '\'' => return scanChar(s),
446	case '"' => return scanString(s),
447	case '(' => return tok(s, TokenKind::LParen),
448	case ')' => return tok(s, TokenKind::RParen),
449	case '{' => return tok(s, TokenKind::LBrace),
450	case '}' => return tok(s, TokenKind::RBrace),
451	case '[' => return tok(s, TokenKind::LBracket),
452	case ']' => return tok(s, TokenKind::RBracket),
453	case ';' => return tok(s, TokenKind::Semicolon),
454	case ',' => return tok(s, TokenKind::Comma),
455	case '.' => {
456	if consume(s, '.') {
457	return tok(s, TokenKind::DotDot);
458	}
459	return tok(s, TokenKind::Dot);
460	}
461	case ':' => {
462	if consume(s, ':') {
463	return tok(s, TokenKind::ColonColon);
464	}
465	return tok(s, TokenKind::Colon);
466	}
467	case '-' => {
468	if consume(s, '>') {
469	return tok(s, TokenKind::Arrow);
470	}
471	if consume(s, '=') {
472	return tok(s, TokenKind::MinusEqual);
473	}
474	return tok(s, TokenKind::Minus);
475	}
476	case '+' => {
477	if consume(s, '=') {
478	return tok(s, TokenKind::PlusEqual);
479	}
480	return tok(s, TokenKind::Plus);
481	}
482	case '/' => {
483	if consume(s, '=') {
484	return tok(s, TokenKind::SlashEqual);
485	}
486	return tok(s, TokenKind::Slash);
487	}
488	case '*' => {
489	if consume(s, '=') {
490	return tok(s, TokenKind::StarEqual);
491	}
492	return tok(s, TokenKind::Star);
493	}
494	case '%' => {
495	if consume(s, '=') {
496	return tok(s, TokenKind::PercentEqual);
497	}
498	return tok(s, TokenKind::Percent);
499	}
500	case '&' => {
501	if consume(s, '=') {
502	return tok(s, TokenKind::AmpEqual);
503	}
504	return tok(s, TokenKind::Amp);
505	}
506	case '?' => return tok(s, TokenKind::Question),
507	case '\|' => {
508	if consume(s, '=') {
509	return tok(s, TokenKind::PipeEqual);
510	}
511	return tok(s, TokenKind::Pipe);
512	}
513	case '^' => {
514	if consume(s, '=') {
515	return tok(s, TokenKind::CaretEqual);
516	}
517	return tok(s, TokenKind::Caret);
518	}
519	case '~' => return tok(s, TokenKind::Tilde),
520	case '!' => return tok(s, TokenKind::Bang),
521	case '=' => {
522	if consume(s, '>') {
523	return tok(s, TokenKind::FatArrow);
524	}
525	if consume(s, '=') {
526	return tok(s, TokenKind::EqualEqual);
527	}
528	return tok(s, TokenKind::Equal);
529	}
530	case '<' => {
531	if consume(s, '>') {
532	return tok(s, TokenKind::LtGt);
533	}
534	if consume(s, '<') {
535	if consume(s, '=') {
536	return tok(s, TokenKind::LtLtEqual);
537	}
538	return tok(s, TokenKind::LtLt);
539	}
540	if consume(s, '=') {
541	return tok(s, TokenKind::LtEqual);
542	}
543	return tok(s, TokenKind::Lt);
544	}
545	case '>' => {
546	if consume(s, '>') {
547	if consume(s, '=') {
548	return tok(s, TokenKind::GtGtEqual);
549	}
550	return tok(s, TokenKind::GtGt);
551	}
552	if consume(s, '=') {
553	return tok(s, TokenKind::GtEqual);
554	}
555	return tok(s, TokenKind::Gt);
556	}
557	case '@' => {
558	// Scan `@identifier` as a single token.
559	let ch = current(s) else {
560	return invalid(s.token, "expected identifier after `@`");
561	};
562	if not char::isAlpha(ch) and ch <> '_' {
563	return invalid(s.token, "expected identifier after `@`");
564	}
565	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
566	advance(s);
567	}
568	let name = &s.source[s.token..s.cursor];
569	return Token {
570	kind: TokenKind::AtIdent,
571	source: strings::intern(pool, name),
572	offset: s.token,
573	};
574	}
575	case '_' => {
576	if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
577	// This is part of an identifier like `_foo` or `__start`
578	return scanIdentifier(s, pool);
579	}
580	return tok(s, TokenKind::Underscore);
581	}
582	else => return invalid(s.token, "unexpected character"),
583	}
584	}
585
586	/// Get the source code location from a byte offset.
587	export fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
588	let mut l: u16 = 1;
589	let mut c: u16 = 1;
590
591	if offset >= source.len {
592	return nil;
593	}
594	for ch in &source[..offset] {
595	if ch == '\n' {
596	set c = 1;
597	set l += 1;
598	} else {
599	set c += 1;
600	}
601	}
602	return Location { source: sourceLoc, line: l, col: c };
603	}