radiance · Radiance self-hosting compiler

compiler/ kernel/ lib/ examples/ std/ arch/ char/ collections/ lang/ alloc/ ast/ gen/ il/ module/ parser/ resolver/ scanner/ alloc.rad 4.3 KiB ast.rad 23.3 KiB gen.rad 513 B il.rad 16.1 KiB lower.rad 272.5 KiB module.rad 13.2 KiB package.rad 1.3 KiB parser.rad 77.8 KiB resolver.rad 308.6 KiB scanner.rad 17.5 KiB sexpr.rad 6.3 KiB strings.rad 2.2 KiB types.rad 286 B sys/ arch.rad 68 B char.rad 855 B collections.rad 39 B fmt.rad 8.1 KiB intrinsics.rad 467 B io.rad 1.3 KiB lang.rad 276 B mem.rad 2.2 KiB sys.rad 179 B testing.rad 2.4 KiB tests.rad 15.2 KiB vec.rad 3.2 KiB std.rad 281 B scripts/ seed/ sublime/ test/ vim/ .gitignore 351 B .gitsigners 112 B CONTRIBUTING 2.1 KiB LICENSE 1.1 KiB Makefile 3.8 KiB README 2.5 KiB STYLE 2.5 KiB std.lib 1.4 KiB std.lib.test 380 B

lib/std/lang/scanner.rad 17.5 KiB raw

//! Lexical scanner for the Radiance programming language.
//!
//! This module implements a hand-written scanner that tokenizes Radiance
//! source code into a stream of tokens for consumption by the parser.
@test mod tests;

use std::char;
use std::mem;
use std::lang::strings;

/// Token kinds representing all lexical elements in Radiance.
///
/// This enum covers operators, keywords, literals, and structural
/// elements used by the parser to build the AST.
export union TokenKind: Copy {
    /// Special end of file token generated when the input is exhausted.
    Eof,
    /// Special invalid token.
    Invalid,

    LParen,     // (
    RParen,     // )
    LBrace,     // {
    RBrace,     // }
    LBracket,   // [
    RBracket,   // ]
    Comma,      // ,
    Dot,        // .
    DotDot,     // ..
    Minus,      // -
    Plus,       // +
    Colon,      // :
    ColonColon, // ::
    Semicolon,  // ;
    Slash,      // /
    Star,       // *
    Percent,    // %
    Amp,        // &
    Pipe,       // |
    Caret,      // ^
    Tilde,      // ~
    Underscore, // _
    Question,   // ?
    Bang,       // !
    LtGt,       // <>
    Equal,      // =
    EqualEqual, // ==
    Gt,         // >
    GtEqual,    // >=
    Lt,         // <
    LtEqual,    // <=
    LtLt,       // <<
    GtGt,       // >>
    Arrow,      // ->
    FatArrow,   // =>

    // Compound assignment operators.
    PlusEqual,    // +=
    MinusEqual,   // -=
    StarEqual,    // *=
    SlashEqual,   // /=
    PercentEqual, // %=
    AmpEqual,     // &=
    PipeEqual,    // |=
    CaretEqual,   // ^=
    LtLtEqual,    // <<=
    GtGtEqual,    // >>=

    // Boolean operators.
    Not, And, Or,

    /// Eg. `fnord`
    Ident,
    /// Eg. `@default`
    AtIdent,
    /// The `log` keyword.
    Log,

    // Literals.
    String,     // "fnord"
    Char,       // 'f'
    Number,     // 42
    True,       // true
    False,      // false
    Nil,        // nil
    Undefined,  // undefined

    // Control flow tokens.
    If, Else, Return, Break,
    Continue, While, For, In,
    Loop, Match, Case, Try, Catch,
    Throw, Throws, Panic, Assert,

    // Variable binding tokens.
    Let, Mut, Set, Constant, Align,

    // Module-related tokens.
    Mod, Use, Super,

    // Type or function attributes.
    Export, Static,

    // Trait-related tokens.
    Trait, Instance,

    // Type-related tokens.
    I8, I16, I32, I64, U8, U16, U32, U64,
    Opaque, Fn, Bool, Union, Record, As, Unsafe
}

/// A reserved keyword.
record Keyword: Copy {
    /// Keyword string.
    name: *[u8],
    /// Corresponding token.
    tok: TokenKind,
}

/// Sorted keyword table for binary search.
constant KEYWORDS: [Keyword; 52] = [
    { name: "align", tok: TokenKind::Align },
    { name: "and", tok: TokenKind::And },
    { name: "as", tok: TokenKind::As },
    { name: "assert", tok: TokenKind::Assert },
    { name: "bool", tok: TokenKind::Bool },
    { name: "break", tok: TokenKind::Break },
    { name: "case", tok: TokenKind::Case },
    { name: "catch", tok: TokenKind::Catch },
    { name: "constant", tok: TokenKind::Constant },
    { name: "continue", tok: TokenKind::Continue },
    { name: "else", tok: TokenKind::Else },
    { name: "export", tok: TokenKind::Export },
    { name: "false", tok: TokenKind::False },
    { name: "fn", tok: TokenKind::Fn },
    { name: "for", tok: TokenKind::For },
    { name: "i16", tok: TokenKind::I16 },
    { name: "i32", tok: TokenKind::I32 },
    { name: "i64", tok: TokenKind::I64 },
    { name: "i8", tok: TokenKind::I8 },
    { name: "if", tok: TokenKind::If },
    { name: "in", tok: TokenKind::In },
    { name: "instance", tok: TokenKind::Instance },
    { name: "let", tok: TokenKind::Let },
    { name: "log", tok: TokenKind::Log },
    { name: "loop", tok: TokenKind::Loop },
    { name: "match", tok: TokenKind::Match },
    { name: "mod", tok: TokenKind::Mod },
    { name: "mut", tok: TokenKind::Mut },
    { name: "nil", tok: TokenKind::Nil },
    { name: "not", tok: TokenKind::Not },
    { name: "opaque", tok: TokenKind::Opaque },
    { name: "or", tok: TokenKind::Or },
    { name: "panic", tok: TokenKind::Panic },
    { name: "record", tok: TokenKind::Record },
    { name: "return", tok: TokenKind::Return },
    { name: "set", tok: TokenKind::Set },
    { name: "static", tok: TokenKind::Static },
    { name: "super", tok: TokenKind::Super },
    { name: "throw", tok: TokenKind::Throw },
    { name: "throws", tok: TokenKind::Throws },
    { name: "trait", tok: TokenKind::Trait },
    { name: "true", tok: TokenKind::True },
    { name: "try", tok: TokenKind::Try },
    { name: "u16", tok: TokenKind::U16 },
    { name: "u32", tok: TokenKind::U32 },
    { name: "u64", tok: TokenKind::U64 },
    { name: "u8", tok: TokenKind::U8 },
    { name: "undefined", tok: TokenKind::Undefined },
    { name: "union", tok: TokenKind::Union },
    { name: "unsafe", tok: TokenKind::Unsafe },
    { name: "use", tok: TokenKind::Use },
    { name: "while", tok: TokenKind::While },
];

/// Describes where source code originated from.
export union SourceLoc: Copy {
    /// Source loaded from a file at the given path.
    File(*[u8]),
    /// Source provided as an inline string (no file path).
    String,
}

/// Lexical scanner state for tokenizing Radiance source code.
///
/// Maintains position information and source buffer reference.
export record Scanner: Copy {
    /// Origin of the source being scanned.
    sourceLoc: SourceLoc,
    /// Source buffer.
    source: *[u8],
    /// Offset of current token into buffer.
    token: u32,
    /// Offset of current character being scanned.
    cursor: u32,
    /// Interned string pool.
    pool: *mut strings::Pool,
}

/// Individual token with kind, source text, and position.
///
/// Represents a single lexical element extracted from source,
/// including its original text and byte offset for error reporting.
export record Token: Copy {
    /// Token kind.
    kind: TokenKind,
    /// Token source string.
    source: *[u8],
    /// Byte offset of `source` in input buffer.
    offset: u32,
}

/// Source code location with line/column information.
///
/// Used for error reporting and debugging.
export record Location: Copy {
    /// Origin of the source.
    source: SourceLoc,
    /// Line number.
    line: u16,
    /// Column number.
    col: u16,
}

/// Create a new scanner object.
export fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: *mut strings::Pool) -> Scanner {
    // Intern built-in functions and attributes.
    strings::intern(pool, "@sizeOf");
    strings::intern(pool, "@alignOf");
    strings::intern(pool, "@sliceOf");
    strings::intern(pool, "@default");
    strings::intern(pool, "@intrinsic");
    strings::intern(pool, "@test");
    // Intern built-in slice methods.
    strings::intern(pool, "append");
    strings::intern(pool, "delete");

    return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
}

/// Check if we've reached the end of input.
export fn isEof(s: *Scanner) -> bool {
    return s.cursor >= s.source.len;
}

/// Get the current character, if any.
export fn current(s: *Scanner) -> ?u8 {
    if isEof(s) {
        return nil;
    }
    return s.source[s.cursor];
}

/// Peek at the next character without advancing the scanner.
fn peek(s: *Scanner) -> ?u8 {
    if s.cursor + 1 >= s.source.len {
        return nil;
    }
    return s.source[s.cursor + 1];
}

/// Advance scanner and return the character that was consumed.
fn advance(s: *mut Scanner) -> u8 {
    let ch = s.source[s.cursor];
    set s.cursor += 1;
    return ch;
}

/// Consume the expected character if it matches the current position.
fn consume(s: *mut Scanner, expected: u8) -> bool {
    if let c = current(s); c == expected {
        advance(s);
        return true;
    }
    return false;
}

/// Create a token from the current scanner state.
fn tok(s: *Scanner, kind: TokenKind) -> Token {
    return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
}

/// Create an invalid token with the given message.
export fn invalid(offset: u32, message: *[u8]) -> Token {
    return Token { kind: TokenKind::Invalid, source: message, offset };
}

/// Skip whitespace characters and line comments.
fn skipWhitespace(s: *mut Scanner) {
    while let ch = current(s) {
        match ch {
            case ' ', '\n', '\r', '\t' => advance(s),
            case '/' => {
                if let c = peek(s); c == '/' {
                    while let ch = current(s); ch <> '\n' {
                        advance(s);
                    }
                } else {
                    return;
                }
            }
            else => return,
        }
    }
}

/// Scan numeric literal (decimal, hex, or binary).
fn scanNumber(s: *mut Scanner) -> Token {
    // Check for hex literal (`0x` or `0X` prefix).
    if s.source[s.cursor - 1] == '0' {
        if let ch = current(s); ch == 'x' or ch == 'X' {
            advance(s);
            // Must have at least one hex digit after `0x`.
            if let ch = current(s); not char::isHexDigit(ch) {
                return invalid(s.token, "invalid hex literal");
            }
            while let ch = current(s); char::isHexDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
        // Check for binary literal (`0b` or `0B` prefix).
        if let ch = current(s); ch == 'b' or ch == 'B' {
            advance(s);
            // Must have at least one binary digit after `0b`.
            if let ch = current(s); not char::isBinDigit(ch) {
                return invalid(s.token, "invalid binary literal");
            }
            while let ch = current(s); char::isBinDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
    }
    // Regular decimal number.
    while let ch = current(s); char::isDigit(ch) {
        advance(s);
    }
    // Look for decimal part.
    if let ch = current(s); ch == '.' {
        if let p = peek(s); char::isDigit(p) {
            advance(s); // Consume the "."
            while let ch = current(s); char::isDigit(ch) {
                advance(s);
            }
        }
    }
    return tok(s, TokenKind::Number);
}

fn scanDelimited(s: *mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
    while let ch = current(s); ch <> delim {
        if not char::isPrint(ch) {
            return invalid(s.token, "invalid character");
        }
        if consume(s, '\\') { // Consume escapes
            if isEof(s) {
                return nil;
            }
        }
        advance(s);
    }
    if not consume(s, delim) {
        return nil;
    }
    return tok(s, kind);
}

/// Scan string literal enclosed in double quotes.
fn scanString(s: *mut Scanner) -> Token {
    if let tok = scanDelimited(s, '"', TokenKind::String) {
        return tok;
    }
    return invalid(s.token, "unterminated string");
}

/// Scan character literal enclosed in single quotes.
fn scanChar(s: *mut Scanner) -> Token {
    if let tok = scanDelimited(s, '\'', TokenKind::Char) {
        return tok;
    }
    return invalid(s.token, "unterminated character");
}

/// Scan a keyword or an identifier.
fn keywordOrIdent(src: *[u8]) -> TokenKind {
    let mut left: u32 = 0;
    let mut right: u32 = KEYWORDS.len;

    while left < right {
        let mid = left + ((right - left) / 2);
        let kw = &KEYWORDS[mid];
        let cmp = mem::cmp(src, kw.name);

        match cmp {
            case -1 => set right = mid,
            case 1 => set left = mid + 1,
            else => return kw.tok,
        }
    }
    return TokenKind::Ident;
}

/// Scan an identifier, keyword, or label.
fn scanIdentifier(s: *mut Scanner) -> Token {
    while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
        advance(s);
    }
    let ident = &s.source[s.token..s.cursor];
    let kind = keywordOrIdent(ident);

    // Only intern actual identifiers, not keywords.
    if kind == TokenKind::Ident {
        return Token { kind, source: strings::intern(s.pool, ident), offset: s.token };
    }
    return tok(s, kind);
}

/// Scan the next token.
export fn next(s: *mut Scanner) -> Token {
    skipWhitespace(s);  // Skip any whitespace between tokens.
    set s.token = s.cursor; // Token starts at current position.

    if isEof(s) {
        return tok(s, TokenKind::Eof);
    }
    let c: u8 = advance(s);

    if char::isDigit(c) {
        return scanNumber(s);
    }
    if char::isAlpha(c) {
        return scanIdentifier(s);
    }
    match c {
        case '\'' => return scanChar(s),
        case '"'  => return scanString(s),
        case '('  => return tok(s, TokenKind::LParen),
        case ')'  => return tok(s, TokenKind::RParen),
        case '{'  => return tok(s, TokenKind::LBrace),
        case '}'  => return tok(s, TokenKind::RBrace),
        case '['  => return tok(s, TokenKind::LBracket),
        case ']'  => return tok(s, TokenKind::RBracket),
        case ';'  => return tok(s, TokenKind::Semicolon),
        case ','  => return tok(s, TokenKind::Comma),
        case '.'  => {
            if consume(s, '.') {
                return tok(s, TokenKind::DotDot);
            }
            return tok(s, TokenKind::Dot);
        }
        case ':'  => {
            if consume(s, ':') {
                return tok(s, TokenKind::ColonColon);
            }
            return tok(s, TokenKind::Colon);
        }
        case '-'  => {
            if consume(s, '>') {
                return tok(s, TokenKind::Arrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::MinusEqual);
            }
            return tok(s, TokenKind::Minus);
        }
        case '+' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PlusEqual);
            }
            return tok(s, TokenKind::Plus);
        }
        case '/' => {
            if consume(s, '=') {
                return tok(s, TokenKind::SlashEqual);
            }
            return tok(s, TokenKind::Slash);
        }
        case '*' => {
            if consume(s, '=') {
                return tok(s, TokenKind::StarEqual);
            }
            return tok(s, TokenKind::Star);
        }
        case '%' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PercentEqual);
            }
            return tok(s, TokenKind::Percent);
        }
        case '&' => {
            if consume(s, '=') {
                return tok(s, TokenKind::AmpEqual);
            }
            return tok(s, TokenKind::Amp);
        }
        case '?' => return tok(s, TokenKind::Question),
        case '|' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PipeEqual);
            }
            return tok(s, TokenKind::Pipe);
        }
        case '^' => {
            if consume(s, '=') {
                return tok(s, TokenKind::CaretEqual);
            }
            return tok(s, TokenKind::Caret);
        }
        case '~' => return tok(s, TokenKind::Tilde),
        case '!' => return tok(s, TokenKind::Bang),
        case '=' => {
            if consume(s, '>') {
                return tok(s, TokenKind::FatArrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::EqualEqual);
            }
            return tok(s, TokenKind::Equal);
        }
        case '<' => {
            if consume(s, '>') {
                return tok(s, TokenKind::LtGt);
            }
            if consume(s, '<') {
                if consume(s, '=') {
                    return tok(s, TokenKind::LtLtEqual);
                }
                return tok(s, TokenKind::LtLt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::LtEqual);
            }
            return tok(s, TokenKind::Lt);
        }
        case '>' => {
            if consume(s, '>') {
                if consume(s, '=') {
                    return tok(s, TokenKind::GtGtEqual);
                }
                return tok(s, TokenKind::GtGt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::GtEqual);
            }
            return tok(s, TokenKind::Gt);
        }
        case '@' => {
            // Scan `@identifier` as a single token.
            let ch = current(s) else {
                return invalid(s.token, "expected identifier after `@`");
            };
            if not char::isAlpha(ch) and ch <> '_' {
                return invalid(s.token, "expected identifier after `@`");
            }
            while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
                advance(s);
            }
            let name = &s.source[s.token..s.cursor];
            return Token {
                kind: TokenKind::AtIdent,
                source: strings::intern(s.pool, name),
                offset: s.token,
            };
        }
        case '_' => {
            if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
                // This is part of an identifier like `_foo` or `__start`
                return scanIdentifier(s);
            }
            return tok(s, TokenKind::Underscore);
        }
        else => return invalid(s.token, "unexpected character"),
    }
}

/// Get the source code location from a byte offset.
export fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
    let mut l: u16 = 1;
    let mut c: u16 = 1;

    if offset >= source.len {
        return nil;
    }
    for ch in &source[..offset] {
        if ch == '\n' {
            set c = 1;
            set l += 1;
        } else {
            set c += 1;
        }
    }
    return Location { source: sourceLoc, line: l, col: c };
}

1	//! Lexical scanner for the Radiance programming language.
2	//!
3	//! This module implements a hand-written scanner that tokenizes Radiance
4	//! source code into a stream of tokens for consumption by the parser.
5	@test mod tests;
6
7	use std::char;
8	use std::mem;
9	use std::lang::strings;
10
11	/// Token kinds representing all lexical elements in Radiance.
12	///
13	/// This enum covers operators, keywords, literals, and structural
14	/// elements used by the parser to build the AST.
15	export union TokenKind: Copy {
16	/// Special end of file token generated when the input is exhausted.
17	Eof,
18	/// Special invalid token.
19	Invalid,
20
21	LParen, // (
22	RParen, // )
23	LBrace, // {
24	RBrace, // }
25	LBracket, // [
26	RBracket, // ]
27	Comma, // ,
28	Dot, // .
29	DotDot, // ..
30	Minus, // -
31	Plus, // +
32	Colon, // :
33	ColonColon, // ::
34	Semicolon, // ;
35	Slash, // /
36	Star, // *
37	Percent, // %
38	Amp, // &
39	Pipe, // \|
40	Caret, // ^
41	Tilde, // ~
42	Underscore, // _
43	Question, // ?
44	Bang, // !
45	LtGt, // <>
46	Equal, // =
47	EqualEqual, // ==
48	Gt, // >
49	GtEqual, // >=
50	Lt, // <
51	LtEqual, // <=
52	LtLt, // <<
53	GtGt, // >>
54	Arrow, // ->
55	FatArrow, // =>
56
57	// Compound assignment operators.
58	PlusEqual, // +=
59	MinusEqual, // -=
60	StarEqual, // *=
61	SlashEqual, // /=
62	PercentEqual, // %=
63	AmpEqual, // &=
64	PipeEqual, // \|=
65	CaretEqual, // ^=
66	LtLtEqual, // <<=
67	GtGtEqual, // >>=
68
69	// Boolean operators.
70	Not, And, Or,
71
72	/// Eg. `fnord`
73	Ident,
74	/// Eg. `@default`
75	AtIdent,
76	/// The `log` keyword.
77	Log,
78
79	// Literals.
80	String, // "fnord"
81	Char, // 'f'
82	Number, // 42
83	True, // true
84	False, // false
85	Nil, // nil
86	Undefined, // undefined
87
88	// Control flow tokens.
89	If, Else, Return, Break,
90	Continue, While, For, In,
91	Loop, Match, Case, Try, Catch,
92	Throw, Throws, Panic, Assert,
93
94	// Variable binding tokens.
95	Let, Mut, Set, Constant, Align,
96
97	// Module-related tokens.
98	Mod, Use, Super,
99
100	// Type or function attributes.
101	Export, Static,
102
103	// Trait-related tokens.
104	Trait, Instance,
105
106	// Type-related tokens.
107	I8, I16, I32, I64, U8, U16, U32, U64,
108	Opaque, Fn, Bool, Union, Record, As, Unsafe
109	}
110
111	/// A reserved keyword.
112	record Keyword: Copy {
113	/// Keyword string.
114	name: *[u8],
115	/// Corresponding token.
116	tok: TokenKind,
117	}
118
119	/// Sorted keyword table for binary search.
120	constant KEYWORDS: [Keyword; 52] = [
121	{ name: "align", tok: TokenKind::Align },
122	{ name: "and", tok: TokenKind::And },
123	{ name: "as", tok: TokenKind::As },
124	{ name: "assert", tok: TokenKind::Assert },
125	{ name: "bool", tok: TokenKind::Bool },
126	{ name: "break", tok: TokenKind::Break },
127	{ name: "case", tok: TokenKind::Case },
128	{ name: "catch", tok: TokenKind::Catch },
129	{ name: "constant", tok: TokenKind::Constant },
130	{ name: "continue", tok: TokenKind::Continue },
131	{ name: "else", tok: TokenKind::Else },
132	{ name: "export", tok: TokenKind::Export },
133	{ name: "false", tok: TokenKind::False },
134	{ name: "fn", tok: TokenKind::Fn },
135	{ name: "for", tok: TokenKind::For },
136	{ name: "i16", tok: TokenKind::I16 },
137	{ name: "i32", tok: TokenKind::I32 },
138	{ name: "i64", tok: TokenKind::I64 },
139	{ name: "i8", tok: TokenKind::I8 },
140	{ name: "if", tok: TokenKind::If },
141	{ name: "in", tok: TokenKind::In },
142	{ name: "instance", tok: TokenKind::Instance },
143	{ name: "let", tok: TokenKind::Let },
144	{ name: "log", tok: TokenKind::Log },
145	{ name: "loop", tok: TokenKind::Loop },
146	{ name: "match", tok: TokenKind::Match },
147	{ name: "mod", tok: TokenKind::Mod },
148	{ name: "mut", tok: TokenKind::Mut },
149	{ name: "nil", tok: TokenKind::Nil },
150	{ name: "not", tok: TokenKind::Not },
151	{ name: "opaque", tok: TokenKind::Opaque },
152	{ name: "or", tok: TokenKind::Or },
153	{ name: "panic", tok: TokenKind::Panic },
154	{ name: "record", tok: TokenKind::Record },
155	{ name: "return", tok: TokenKind::Return },
156	{ name: "set", tok: TokenKind::Set },
157	{ name: "static", tok: TokenKind::Static },
158	{ name: "super", tok: TokenKind::Super },
159	{ name: "throw", tok: TokenKind::Throw },
160	{ name: "throws", tok: TokenKind::Throws },
161	{ name: "trait", tok: TokenKind::Trait },
162	{ name: "true", tok: TokenKind::True },
163	{ name: "try", tok: TokenKind::Try },
164	{ name: "u16", tok: TokenKind::U16 },
165	{ name: "u32", tok: TokenKind::U32 },
166	{ name: "u64", tok: TokenKind::U64 },
167	{ name: "u8", tok: TokenKind::U8 },
168	{ name: "undefined", tok: TokenKind::Undefined },
169	{ name: "union", tok: TokenKind::Union },
170	{ name: "unsafe", tok: TokenKind::Unsafe },
171	{ name: "use", tok: TokenKind::Use },
172	{ name: "while", tok: TokenKind::While },
173	];
174
175	/// Describes where source code originated from.
176	export union SourceLoc: Copy {
177	/// Source loaded from a file at the given path.
178	File(*[u8]),
179	/// Source provided as an inline string (no file path).
180	String,
181	}
182
183	/// Lexical scanner state for tokenizing Radiance source code.
184	///
185	/// Maintains position information and source buffer reference.
186	export record Scanner: Copy {
187	/// Origin of the source being scanned.
188	sourceLoc: SourceLoc,
189	/// Source buffer.
190	source: *[u8],
191	/// Offset of current token into buffer.
192	token: u32,
193	/// Offset of current character being scanned.
194	cursor: u32,
195	/// Interned string pool.
196	pool: *mut strings::Pool,
197	}
198
199	/// Individual token with kind, source text, and position.
200	///
201	/// Represents a single lexical element extracted from source,
202	/// including its original text and byte offset for error reporting.
203	export record Token: Copy {
204	/// Token kind.
205	kind: TokenKind,
206	/// Token source string.
207	source: *[u8],
208	/// Byte offset of `source` in input buffer.
209	offset: u32,
210	}
211
212	/// Source code location with line/column information.
213	///
214	/// Used for error reporting and debugging.
215	export record Location: Copy {
216	/// Origin of the source.
217	source: SourceLoc,
218	/// Line number.
219	line: u16,
220	/// Column number.
221	col: u16,
222	}
223
224	/// Create a new scanner object.
225	export fn scanner(sourceLoc: SourceLoc, source: [u8], pool: mut strings::Pool) -> Scanner {
226	// Intern built-in functions and attributes.
227	strings::intern(pool, "@sizeOf");
228	strings::intern(pool, "@alignOf");
229	strings::intern(pool, "@sliceOf");
230	strings::intern(pool, "@default");
231	strings::intern(pool, "@intrinsic");
232	strings::intern(pool, "@test");
233	// Intern built-in slice methods.
234	strings::intern(pool, "append");
235	strings::intern(pool, "delete");
236
237	return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
238	}
239
240	/// Check if we've reached the end of input.
241	export fn isEof(s: *Scanner) -> bool {
242	return s.cursor >= s.source.len;
243	}
244
245	/// Get the current character, if any.
246	export fn current(s: *Scanner) -> ?u8 {
247	if isEof(s) {
248	return nil;
249	}
250	return s.source[s.cursor];
251	}
252
253	/// Peek at the next character without advancing the scanner.
254	fn peek(s: *Scanner) -> ?u8 {
255	if s.cursor + 1 >= s.source.len {
256	return nil;
257	}
258	return s.source[s.cursor + 1];
259	}
260
261	/// Advance scanner and return the character that was consumed.
262	fn advance(s: *mut Scanner) -> u8 {
263	let ch = s.source[s.cursor];
264	set s.cursor += 1;
265	return ch;
266	}
267
268	/// Consume the expected character if it matches the current position.
269	fn consume(s: *mut Scanner, expected: u8) -> bool {
270	if let c = current(s); c == expected {
271	advance(s);
272	return true;
273	}
274	return false;
275	}
276
277	/// Create a token from the current scanner state.
278	fn tok(s: *Scanner, kind: TokenKind) -> Token {
279	return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
280	}
281
282	/// Create an invalid token with the given message.
283	export fn invalid(offset: u32, message: *[u8]) -> Token {
284	return Token { kind: TokenKind::Invalid, source: message, offset };
285	}
286
287	/// Skip whitespace characters and line comments.
288	fn skipWhitespace(s: *mut Scanner) {
289	while let ch = current(s) {
290	match ch {
291	case ' ', '\n', '\r', '\t' => advance(s),
292	case '/' => {
293	if let c = peek(s); c == '/' {
294	while let ch = current(s); ch <> '\n' {
295	advance(s);
296	}
297	} else {
298	return;
299	}
300	}
301	else => return,
302	}
303	}
304	}
305
306	/// Scan numeric literal (decimal, hex, or binary).
307	fn scanNumber(s: *mut Scanner) -> Token {
308	// Check for hex literal (`0x` or `0X` prefix).
309	if s.source[s.cursor - 1] == '0' {
310	if let ch = current(s); ch == 'x' or ch == 'X' {
311	advance(s);
312	// Must have at least one hex digit after `0x`.
313	if let ch = current(s); not char::isHexDigit(ch) {
314	return invalid(s.token, "invalid hex literal");
315	}
316	while let ch = current(s); char::isHexDigit(ch) {
317	advance(s);
318	}
319	return tok(s, TokenKind::Number);
320	}
321	// Check for binary literal (`0b` or `0B` prefix).
322	if let ch = current(s); ch == 'b' or ch == 'B' {
323	advance(s);
324	// Must have at least one binary digit after `0b`.
325	if let ch = current(s); not char::isBinDigit(ch) {
326	return invalid(s.token, "invalid binary literal");
327	}
328	while let ch = current(s); char::isBinDigit(ch) {
329	advance(s);
330	}
331	return tok(s, TokenKind::Number);
332	}
333	}
334	// Regular decimal number.
335	while let ch = current(s); char::isDigit(ch) {
336	advance(s);
337	}
338	// Look for decimal part.
339	if let ch = current(s); ch == '.' {
340	if let p = peek(s); char::isDigit(p) {
341	advance(s); // Consume the "."
342	while let ch = current(s); char::isDigit(ch) {
343	advance(s);
344	}
345	}
346	}
347	return tok(s, TokenKind::Number);
348	}
349
350	fn scanDelimited(s: *mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
351	while let ch = current(s); ch <> delim {
352	if not char::isPrint(ch) {
353	return invalid(s.token, "invalid character");
354	}
355	if consume(s, '\\') { // Consume escapes
356	if isEof(s) {
357	return nil;
358	}
359	}
360	advance(s);
361	}
362	if not consume(s, delim) {
363	return nil;
364	}
365	return tok(s, kind);
366	}
367
368	/// Scan string literal enclosed in double quotes.
369	fn scanString(s: *mut Scanner) -> Token {
370	if let tok = scanDelimited(s, '"', TokenKind::String) {
371	return tok;
372	}
373	return invalid(s.token, "unterminated string");
374	}
375
376	/// Scan character literal enclosed in single quotes.
377	fn scanChar(s: *mut Scanner) -> Token {
378	if let tok = scanDelimited(s, '\'', TokenKind::Char) {
379	return tok;
380	}
381	return invalid(s.token, "unterminated character");
382	}
383
384	/// Scan a keyword or an identifier.
385	fn keywordOrIdent(src: *[u8]) -> TokenKind {
386	let mut left: u32 = 0;
387	let mut right: u32 = KEYWORDS.len;
388
389	while left < right {
390	let mid = left + ((right - left) / 2);
391	let kw = &KEYWORDS[mid];
392	let cmp = mem::cmp(src, kw.name);
393
394	match cmp {
395	case -1 => set right = mid,
396	case 1 => set left = mid + 1,
397	else => return kw.tok,
398	}
399	}
400	return TokenKind::Ident;
401	}
402
403	/// Scan an identifier, keyword, or label.
404	fn scanIdentifier(s: *mut Scanner) -> Token {
405	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
406	advance(s);
407	}
408	let ident = &s.source[s.token..s.cursor];
409	let kind = keywordOrIdent(ident);
410
411	// Only intern actual identifiers, not keywords.
412	if kind == TokenKind::Ident {
413	return Token { kind, source: strings::intern(s.pool, ident), offset: s.token };
414	}
415	return tok(s, kind);
416	}
417
418	/// Scan the next token.
419	export fn next(s: *mut Scanner) -> Token {
420	skipWhitespace(s); // Skip any whitespace between tokens.
421	set s.token = s.cursor; // Token starts at current position.
422
423	if isEof(s) {
424	return tok(s, TokenKind::Eof);
425	}
426	let c: u8 = advance(s);
427
428	if char::isDigit(c) {
429	return scanNumber(s);
430	}
431	if char::isAlpha(c) {
432	return scanIdentifier(s);
433	}
434	match c {
435	case '\'' => return scanChar(s),
436	case '"' => return scanString(s),
437	case '(' => return tok(s, TokenKind::LParen),
438	case ')' => return tok(s, TokenKind::RParen),
439	case '{' => return tok(s, TokenKind::LBrace),
440	case '}' => return tok(s, TokenKind::RBrace),
441	case '[' => return tok(s, TokenKind::LBracket),
442	case ']' => return tok(s, TokenKind::RBracket),
443	case ';' => return tok(s, TokenKind::Semicolon),
444	case ',' => return tok(s, TokenKind::Comma),
445	case '.' => {
446	if consume(s, '.') {
447	return tok(s, TokenKind::DotDot);
448	}
449	return tok(s, TokenKind::Dot);
450	}
451	case ':' => {
452	if consume(s, ':') {
453	return tok(s, TokenKind::ColonColon);
454	}
455	return tok(s, TokenKind::Colon);
456	}
457	case '-' => {
458	if consume(s, '>') {
459	return tok(s, TokenKind::Arrow);
460	}
461	if consume(s, '=') {
462	return tok(s, TokenKind::MinusEqual);
463	}
464	return tok(s, TokenKind::Minus);
465	}
466	case '+' => {
467	if consume(s, '=') {
468	return tok(s, TokenKind::PlusEqual);
469	}
470	return tok(s, TokenKind::Plus);
471	}
472	case '/' => {
473	if consume(s, '=') {
474	return tok(s, TokenKind::SlashEqual);
475	}
476	return tok(s, TokenKind::Slash);
477	}
478	case '*' => {
479	if consume(s, '=') {
480	return tok(s, TokenKind::StarEqual);
481	}
482	return tok(s, TokenKind::Star);
483	}
484	case '%' => {
485	if consume(s, '=') {
486	return tok(s, TokenKind::PercentEqual);
487	}
488	return tok(s, TokenKind::Percent);
489	}
490	case '&' => {
491	if consume(s, '=') {
492	return tok(s, TokenKind::AmpEqual);
493	}
494	return tok(s, TokenKind::Amp);
495	}
496	case '?' => return tok(s, TokenKind::Question),
497	case '\|' => {
498	if consume(s, '=') {
499	return tok(s, TokenKind::PipeEqual);
500	}
501	return tok(s, TokenKind::Pipe);
502	}
503	case '^' => {
504	if consume(s, '=') {
505	return tok(s, TokenKind::CaretEqual);
506	}
507	return tok(s, TokenKind::Caret);
508	}
509	case '~' => return tok(s, TokenKind::Tilde),
510	case '!' => return tok(s, TokenKind::Bang),
511	case '=' => {
512	if consume(s, '>') {
513	return tok(s, TokenKind::FatArrow);
514	}
515	if consume(s, '=') {
516	return tok(s, TokenKind::EqualEqual);
517	}
518	return tok(s, TokenKind::Equal);
519	}
520	case '<' => {
521	if consume(s, '>') {
522	return tok(s, TokenKind::LtGt);
523	}
524	if consume(s, '<') {
525	if consume(s, '=') {
526	return tok(s, TokenKind::LtLtEqual);
527	}
528	return tok(s, TokenKind::LtLt);
529	}
530	if consume(s, '=') {
531	return tok(s, TokenKind::LtEqual);
532	}
533	return tok(s, TokenKind::Lt);
534	}
535	case '>' => {
536	if consume(s, '>') {
537	if consume(s, '=') {
538	return tok(s, TokenKind::GtGtEqual);
539	}
540	return tok(s, TokenKind::GtGt);
541	}
542	if consume(s, '=') {
543	return tok(s, TokenKind::GtEqual);
544	}
545	return tok(s, TokenKind::Gt);
546	}
547	case '@' => {
548	// Scan `@identifier` as a single token.
549	let ch = current(s) else {
550	return invalid(s.token, "expected identifier after `@`");
551	};
552	if not char::isAlpha(ch) and ch <> '_' {
553	return invalid(s.token, "expected identifier after `@`");
554	}
555	while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
556	advance(s);
557	}
558	let name = &s.source[s.token..s.cursor];
559	return Token {
560	kind: TokenKind::AtIdent,
561	source: strings::intern(s.pool, name),
562	offset: s.token,
563	};
564	}
565	case '_' => {
566	if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) {
567	// This is part of an identifier like `_foo` or `__start`
568	return scanIdentifier(s);
569	}
570	return tok(s, TokenKind::Underscore);
571	}
572	else => return invalid(s.token, "unexpected character"),
573	}
574	}
575
576	/// Get the source code location from a byte offset.
577	export fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
578	let mut l: u16 = 1;
579	let mut c: u16 = 1;
580
581	if offset >= source.len {
582	return nil;
583	}
584	for ch in &source[..offset] {
585	if ch == '\n' {
586	set c = 1;
587	set l += 1;
588	} else {
589	set c += 1;
590	}
591	}
592	return Location { source: sourceLoc, line: l, col: c };
593	}