radiance · Radiance self-hosting compiler

.ai/ compiler/ lib/ examples/ std/ arch/ collections/ lang/ alloc/ ast/ gen/ il/ module/ parser/ resolver/ scanner/ alloc.rad 4.2 KiB ast.rad 22.6 KiB gen.rad 489 B il.rad 18.0 KiB lower.rad 261.8 KiB module.rad 13.4 KiB package.rad 1.2 KiB parser.rad 79.7 KiB resolver.rad 246.8 KiB scanner.rad 18.1 KiB sexpr.rad 6.3 KiB strings.rad 2.2 KiB sys/ arch.rad 65 B collections.rad 36 B fmt.rad 3.8 KiB intrinsics.rad 399 B io.rad 1.2 KiB lang.rad 222 B mem.rad 2.1 KiB sys.rad 167 B testing.rad 2.3 KiB tests.rad 11.6 KiB vec.rad 3.1 KiB std.rad 231 B scripts/ seed/ test/ vim/ .gitignore 353 B .gitsigners 112 B LICENSE 1.1 KiB Makefile 3.0 KiB README 2.5 KiB std.lib 1.0 KiB std.lib.test 252 B

lib/std/lang/scanner.rad 18.1 KiB raw

//! Lexical scanner for the Radiance programming language.
//!
//! This module implements a hand-written scanner that tokenizes Radiance
//! source code into a stream of tokens for consumption by the parser.
@test mod tests;

use std::mem;
use std::lang::strings;

/// Token kinds representing all lexical elements in Radiance.
///
/// This enum covers operators, keywords, literals, and structural
/// elements used by the parser to build the AST.
pub union TokenKind {
    /// Special end of file token generated when the input is exhausted.
    Eof,
    /// Special invalid token.
    Invalid,

    LParen,     // (
    RParen,     // )
    LBrace,     // {
    RBrace,     // }
    LBracket,   // [
    RBracket,   // ]
    Comma,      // ,
    Dot,        // .
    DotDot,     // ..
    Minus,      // -
    Plus,       // +
    Colon,      // :
    ColonColon, // ::
    Semicolon,  // ;
    Slash,      // /
    Star,       // *
    Percent,    // %
    Amp,        // &
    Pipe,       // |
    Caret,      // ^
    Tilde,      // ~
    Underscore, // _
    Question,   // ?
    Bang,       // !
    BangEqual,  // !=
    Equal,      // =
    EqualEqual, // ==
    Gt,         // >
    GtEqual,    // >=
    Lt,         // <
    LtEqual,    // <=
    LtLt,       // <<
    GtGt,       // >>
    Arrow,      // ->
    FatArrow,   // =>

    // Compound assignment operators.
    PlusEqual,    // +=
    MinusEqual,   // -=
    StarEqual,    // *=
    SlashEqual,   // /=
    PercentEqual, // %=
    AmpEqual,     // &=
    PipeEqual,    // |=
    CaretEqual,   // ^=
    LtLtEqual,    // <<=
    GtGtEqual,    // >>=

    // Boolean operators.
    Not, And, Or,

    /// Eg. `fnord`
    Ident,
    /// Eg. `@default`
    AtIdent,
    /// The `log` keyword.
    Log,

    // Literals.
    String,     // "fnord"
    Char,       // 'f'
    Number,     // 42
    True,       // true
    False,      // false
    Nil,        // nil
    Undefined,  // undefined

    // Control flow tokens.
    If, Else, Return, Break,
    Continue, While, For, In,
    Loop, Match, Case, Try, Catch,
    Throw, Throws, Panic, Assert,

    // Variable binding tokens.
    Let, Mut, Const, Align,

    // Module-related tokens.
    Mod, Use, Super,

    // Type or function attributes.
    Pub, Extern, Static,

    // Trait-related tokens.
    Trait, Instance,

    // Type-related tokens.
    I8, I16, I32, I64, U8, U16, U32, U64,
    Opaque, Fn, Bool, Union, Record, As,

    // Safety.
    Unsafe,
}

/// A reserved keyword.
record Keyword {
    /// Keyword string.
    name: *[u8],
    /// Corresponding token.
    tok: TokenKind,
}

/// Sorted keyword table for binary search.
const KEYWORDS: [Keyword; 52] = [
    { name: "align", tok: TokenKind::Align },
    { name: "and", tok: TokenKind::And },
    { name: "as", tok: TokenKind::As },
    { name: "assert", tok: TokenKind::Assert },
    { name: "bool", tok: TokenKind::Bool },
    { name: "break", tok: TokenKind::Break },
    { name: "case", tok: TokenKind::Case },
    { name: "catch", tok: TokenKind::Catch },
    { name: "const", tok: TokenKind::Const },
    { name: "continue", tok: TokenKind::Continue },
    { name: "else", tok: TokenKind::Else },
    { name: "extern", tok: TokenKind::Extern },
    { name: "false", tok: TokenKind::False },
    { name: "fn", tok: TokenKind::Fn },
    { name: "for", tok: TokenKind::For },
    { name: "i16", tok: TokenKind::I16 },
    { name: "i32", tok: TokenKind::I32 },
    { name: "i64", tok: TokenKind::I64 },
    { name: "i8", tok: TokenKind::I8 },
    { name: "if", tok: TokenKind::If },
    { name: "in", tok: TokenKind::In },
    { name: "instance", tok: TokenKind::Instance },
    { name: "let", tok: TokenKind::Let },
    { name: "log", tok: TokenKind::Log },
    { name: "loop", tok: TokenKind::Loop },
    { name: "match", tok: TokenKind::Match },
    { name: "mod", tok: TokenKind::Mod },
    { name: "mut", tok: TokenKind::Mut },
    { name: "nil", tok: TokenKind::Nil },
    { name: "not", tok: TokenKind::Not },
    { name: "opaque", tok: TokenKind::Opaque },
    { name: "or", tok: TokenKind::Or },
    { name: "panic", tok: TokenKind::Panic },
    { name: "pub", tok: TokenKind::Pub },
    { name: "record", tok: TokenKind::Record },
    { name: "return", tok: TokenKind::Return },
    { name: "static", tok: TokenKind::Static },
    { name: "super", tok: TokenKind::Super },
    { name: "throw", tok: TokenKind::Throw },
    { name: "throws", tok: TokenKind::Throws },
    { name: "trait", tok: TokenKind::Trait },
    { name: "true", tok: TokenKind::True },
    { name: "try", tok: TokenKind::Try },
    { name: "u16", tok: TokenKind::U16 },
    { name: "u32", tok: TokenKind::U32 },
    { name: "u64", tok: TokenKind::U64 },
    { name: "u8", tok: TokenKind::U8 },
    { name: "undefined", tok: TokenKind::Undefined },
    { name: "union", tok: TokenKind::Union },
    { name: "unsafe", tok: TokenKind::Unsafe },
    { name: "use", tok: TokenKind::Use },
    { name: "while", tok: TokenKind::While },
];

/// Describes where source code originated from.
pub union SourceLoc {
    /// Source loaded from a file at the given path.
    File(*[u8]),
    /// Source provided as an inline string (no file path).
    String,
}

/// Lexical scanner state for tokenizing Radiance source code.
///
/// Maintains position information and source buffer reference.
pub record Scanner {
    /// Origin of the source being scanned.
    sourceLoc: SourceLoc,
    /// Source buffer.
    source: *[u8],
    /// Offset of current token into buffer.
    token: u32,
    /// Offset of current character being scanned.
    cursor: u32,
    /// Interned string pool.
    pool: *mut strings::Pool,
}

/// Individual token with kind, source text, and position.
///
/// Represents a single lexical element extracted from source,
/// including its original text and byte offset for error reporting.
pub record Token {
    /// Token kind.
    kind: TokenKind,
    /// Token source string.
    source: *[u8],
    /// Byte offset of `source` in input buffer.
    offset: u32,
}

/// Source code location with line/column information.
///
/// Used for error reporting and debugging.
pub record Location {
    /// Origin of the source.
    source: SourceLoc,
    /// Line number.
    line: u16,
    /// Column number.
    col: u16,
}

/// Create a new scanner object.
pub fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: *mut strings::Pool) -> Scanner {
    // Intern built-in functions and attributes.
    strings::intern(pool, "@sizeOf");
    strings::intern(pool, "@alignOf");
    strings::intern(pool, "@sliceOf");
    strings::intern(pool, "@default");
    strings::intern(pool, "@intrinsic");
    strings::intern(pool, "@test");
    // Intern built-in slice methods.
    strings::intern(pool, "append");
    strings::intern(pool, "delete");

    return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
}

/// Check if we've reached the end of input.
pub fn isEof(s: *Scanner) -> bool {
    return s.cursor >= s.source.len;
}

/// Get the current character, if any.
pub fn current(s: *Scanner) -> ?u8 {
    if isEof(s) {
        return nil;
    }
    return s.source[s.cursor];
}

/// Peek at the next character without advancing the scanner.
fn peek(s: *Scanner) -> ?u8 {
    if s.cursor + 1 >= s.source.len {
        return nil;
    }
    return s.source[s.cursor + 1];
}

/// Advance scanner and return the character that was consumed.
fn advance(s: *mut Scanner) -> u8 {
    s.cursor += 1;
    return s.source[s.cursor - 1];
}

/// Consume the expected character if it matches the current position.
fn consume(s: *mut Scanner, expected: u8) -> bool {
    if let c = current(s); c == expected {
        advance(s);
        return true;
    }
    return false;
}

/// Create a token from the current scanner state.
fn tok(s: *Scanner, kind: TokenKind) -> Token {
    return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
}

/// Create an invalid token with the given message.
pub fn invalid(offset: u32, message: *[u8]) -> Token {
    return Token { kind: TokenKind::Invalid, source: message, offset };
}

/// Skip whitespace characters and line comments.
fn skipWhitespace(s: *mut Scanner) {
    while let ch = current(s) {
        match ch {
            case ' ', '\n', '\r', '\t' => advance(s),
            case '/' => {
                if let c = peek(s); c == '/' {
                    while let ch = current(s); ch != '\n' {
                        advance(s);
                    }
                } else {
                    return;
                }
            }
            else => return,
        }
    }
}

/// Check if character is an ASCII digit (0-9).
fn isDigit(c: u8) -> bool {
    return c >= '0' and c <= '9';
}

/// Check if character is a hexadecimal digit (0-9, a-f, A-F).
fn isHexDigit(c: u8) -> bool {
    return (c >= '0' and c <= '9')
        or (c >= 'a' and c <= 'f')
        or (c >= 'A' and c <= 'F');
}

/// Check if character is a binary digit (0 or 1).
fn isBinDigit(c: u8) -> bool {
    return c == '0' or c == '1';
}

/// Check if character is alphabetic.
fn isAlpha(c: u8) -> bool {
    return (c >= 'a' and c <= 'z')
        or (c >= 'A' and c <= 'Z');
}

/// Check if character is printable ASCII.
fn isPrint(c: u8) -> bool {
    return c >= ' ' and c <= '~';
}

/// Scan numeric literal (decimal, hex, or binary).
fn scanNumber(s: *mut Scanner) -> Token {
    let first = s.source[s.cursor - 1];
    if first == '-' or first == '+' {
        advance(s);
    }
    // Check for hex literal (`0x` or `0X` prefix).
    if s.source[s.cursor - 1] == '0' {
        if let ch = current(s); ch == 'x' or ch == 'X' {
            advance(s);
            // Must have at least one hex digit after `0x`.
            if let ch = current(s); not isHexDigit(ch) {
                return invalid(s.token, "invalid hex literal");
            }
            while let ch = current(s); isHexDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
        // Check for binary literal (`0b` or `0B` prefix).
        if let ch = current(s); ch == 'b' or ch == 'B' {
            advance(s);
            // Must have at least one binary digit after `0b`.
            if let ch = current(s); not isBinDigit(ch) {
                return invalid(s.token, "invalid binary literal");
            }
            while let ch = current(s); isBinDigit(ch) {
                advance(s);
            }
            return tok(s, TokenKind::Number);
        }
    }

    // Regular decimal number.
    while let ch = current(s); isDigit(ch) {
        advance(s);
    }

    // Look for decimal part.
    if let ch = current(s); ch == '.' {
        if let p = peek(s); isDigit(p) {
            advance(s); // Consume the "."
            while let ch = current(s); isDigit(ch) {
                advance(s);
            }
        }
    }
    return tok(s, TokenKind::Number);
}

fn scanDelimited(s: *mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
    while let ch = current(s); ch != delim {
        if not isPrint(ch) {
            return invalid(s.token, "invalid character");
        }
        consume(s, '\\'); // Consume escapes
        advance(s);
    }
    if not consume(s, delim) {
        return nil;
    }
    return tok(s, kind);
}

/// Scan string literal enclosed in double quotes.
fn scanString(s: *mut Scanner) -> Token {
    if let tok = scanDelimited(s, '"', TokenKind::String) {
        return tok;
    }
    return invalid(s.token, "unterminated string");
}

/// Scan character literal enclosed in single quotes.
fn scanChar(s: *mut Scanner) -> Token {
    if let tok = scanDelimited(s, '\'', TokenKind::Char) {
        return tok;
    }
    return invalid(s.token, "unterminated character");
}

/// Scan a keyword or an identifier.
fn keywordOrIdent(src: *[u8]) -> TokenKind {
    let mut left: u32 = 0;
    let mut right: u32 = KEYWORDS.len;

    while left < right {
        let mid = left + ((right - left) / 2);
        let kw = KEYWORDS[mid];
        let cmp = mem::cmp(src, kw.name);

        match cmp {
            case -1 => right = mid,
            case 1 => left = mid + 1,
            else => return kw.tok,
        }
    }
    return TokenKind::Ident;
}

/// Scan an identifier, keyword, or label.
fn scanIdentifier(s: *mut Scanner) -> Token {
    while let ch = current(s); isAlpha(ch) or isDigit(ch) or ch == '_' or ch == '#' {
        advance(s);
    }
    let ident = &s.source[s.token..s.cursor];
    let kind = keywordOrIdent(ident);

    // Only intern actual identifiers, not keywords.
    if kind == TokenKind::Ident {
        return Token { kind, source: strings::intern(s.pool, ident), offset: s.token };
    }
    return tok(s, kind);
}

/// Scan the next token.
pub fn next(s: *mut Scanner) -> Token {
    skipWhitespace(s);  // Skip any whitespace between tokens.
    s.token = s.cursor; // Token starts at current position.

    if isEof(s) {
        return tok(s, TokenKind::Eof);
    }
    let c: u8 = advance(s);

    if isDigit(c) {
        return scanNumber(s);
    }
    if isAlpha(c) {
        return scanIdentifier(s);
    }
    match c {
        case '\'' => return scanChar(s),
        case '"'  => return scanString(s),
        case '('  => return tok(s, TokenKind::LParen),
        case ')'  => return tok(s, TokenKind::RParen),
        case '{'  => return tok(s, TokenKind::LBrace),
        case '}'  => return tok(s, TokenKind::RBrace),
        case '['  => return tok(s, TokenKind::LBracket),
        case ']'  => return tok(s, TokenKind::RBracket),
        case ';'  => return tok(s, TokenKind::Semicolon),
        case ','  => return tok(s, TokenKind::Comma),
        case '.'  => {
            if consume(s, '.') {
                return tok(s, TokenKind::DotDot);
            }
            return tok(s, TokenKind::Dot);
        }
        case ':'  => {
            if consume(s, ':') {
                return tok(s, TokenKind::ColonColon);
            }
            return tok(s, TokenKind::Colon);
        }
        case '-'  => {
            if consume(s, '>') {
                return tok(s, TokenKind::Arrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::MinusEqual);
            }
            // If followed by a digit, scan as negative number
            if let ch = current(s); isDigit(ch) {
                return scanNumber(s);
            }
            return tok(s, TokenKind::Minus);
        }
        case '+' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PlusEqual);
            }
            if let ch = current(s); isDigit(ch) {
                return scanNumber(s);
            }
            return tok(s, TokenKind::Plus);
        }
        case '/' => {
            if consume(s, '=') {
                return tok(s, TokenKind::SlashEqual);
            }
            return tok(s, TokenKind::Slash);
        }
        case '*' => {
            if consume(s, '=') {
                return tok(s, TokenKind::StarEqual);
            }
            return tok(s, TokenKind::Star);
        }
        case '%' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PercentEqual);
            }
            return tok(s, TokenKind::Percent);
        }
        case '&' => {
            if consume(s, '=') {
                return tok(s, TokenKind::AmpEqual);
            }
            return tok(s, TokenKind::Amp);
        }
        case '?' => return tok(s, TokenKind::Question),
        case '|' => {
            if consume(s, '=') {
                return tok(s, TokenKind::PipeEqual);
            }
            return tok(s, TokenKind::Pipe);
        }
        case '^' => {
            if consume(s, '=') {
                return tok(s, TokenKind::CaretEqual);
            }
            return tok(s, TokenKind::Caret);
        }
        case '~' => return tok(s, TokenKind::Tilde),
        case '!' => {
            if consume(s, '=') {
                return tok(s, TokenKind::BangEqual);
            }
            return tok(s, TokenKind::Bang);
        }
        case '=' => {
            if consume(s, '>') {
                return tok(s, TokenKind::FatArrow);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::EqualEqual);
            }
            return tok(s, TokenKind::Equal);
        }
        case '<' => {
            if consume(s, '<') {
                if consume(s, '=') {
                    return tok(s, TokenKind::LtLtEqual);
                }
                return tok(s, TokenKind::LtLt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::LtEqual);
            }
            return tok(s, TokenKind::Lt);
        }
        case '>' => {
            if consume(s, '>') {
                if consume(s, '=') {
                    return tok(s, TokenKind::GtGtEqual);
                }
                return tok(s, TokenKind::GtGt);
            }
            if consume(s, '=') {
                return tok(s, TokenKind::GtEqual);
            }
            return tok(s, TokenKind::Gt);
        }
        case '@' => {
            // Scan `@identifier` as a single token.
            while let ch = current(s); isAlpha(ch) {
                advance(s);
            }
            // Must have at least one character after `@`.
            if s.cursor - s.token <= 1 {
                return invalid(s.token, "expected identifier after `@`");
            }
            let name = &s.source[s.token..s.cursor];
            return Token {
                kind: TokenKind::AtIdent,
                source: strings::intern(s.pool, name),
                offset: s.token,
            };
        }
        case '_' => {
            if let ch = current(s); isAlpha(ch) or isDigit(ch) or ch == '_' {
                // This is part of an identifier like `_foo` or `__start`
                return scanIdentifier(s);
            }
            return tok(s, TokenKind::Underscore);
        }
        else => return invalid(s.token, "unexpected character"),
    }
}

/// Get the source code location from a byte offset.
pub fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
    let mut l: u16 = 1;
    let mut c: u16 = 1;

    if offset >= source.len {
        return nil;
    }
    for ch in &source[..offset] {
        if ch == '\n' {
            c = 1;
            l += 1;
        } else {
            c += 1;
        }
    }
    return Location { source: sourceLoc, line: l, col: c };
}

1	//! Lexical scanner for the Radiance programming language.
2	//!
3	//! This module implements a hand-written scanner that tokenizes Radiance
4	//! source code into a stream of tokens for consumption by the parser.
5	@test mod tests;
6
7	use std::mem;
8	use std::lang::strings;
9
10	/// Token kinds representing all lexical elements in Radiance.
11	///
12	/// This enum covers operators, keywords, literals, and structural
13	/// elements used by the parser to build the AST.
14	pub union TokenKind {
15	/// Special end of file token generated when the input is exhausted.
16	Eof,
17	/// Special invalid token.
18	Invalid,
19
20	LParen, // (
21	RParen, // )
22	LBrace, // {
23	RBrace, // }
24	LBracket, // [
25	RBracket, // ]
26	Comma, // ,
27	Dot, // .
28	DotDot, // ..
29	Minus, // -
30	Plus, // +
31	Colon, // :
32	ColonColon, // ::
33	Semicolon, // ;
34	Slash, // /
35	Star, // *
36	Percent, // %
37	Amp, // &
38	Pipe, // \|
39	Caret, // ^
40	Tilde, // ~
41	Underscore, // _
42	Question, // ?
43	Bang, // !
44	BangEqual, // !=
45	Equal, // =
46	EqualEqual, // ==
47	Gt, // >
48	GtEqual, // >=
49	Lt, // <
50	LtEqual, // <=
51	LtLt, // <<
52	GtGt, // >>
53	Arrow, // ->
54	FatArrow, // =>
55
56	// Compound assignment operators.
57	PlusEqual, // +=
58	MinusEqual, // -=
59	StarEqual, // *=
60	SlashEqual, // /=
61	PercentEqual, // %=
62	AmpEqual, // &=
63	PipeEqual, // \|=
64	CaretEqual, // ^=
65	LtLtEqual, // <<=
66	GtGtEqual, // >>=
67
68	// Boolean operators.
69	Not, And, Or,
70
71	/// Eg. `fnord`
72	Ident,
73	/// Eg. `@default`
74	AtIdent,
75	/// The `log` keyword.
76	Log,
77
78	// Literals.
79	String, // "fnord"
80	Char, // 'f'
81	Number, // 42
82	True, // true
83	False, // false
84	Nil, // nil
85	Undefined, // undefined
86
87	// Control flow tokens.
88	If, Else, Return, Break,
89	Continue, While, For, In,
90	Loop, Match, Case, Try, Catch,
91	Throw, Throws, Panic, Assert,
92
93	// Variable binding tokens.
94	Let, Mut, Const, Align,
95
96	// Module-related tokens.
97	Mod, Use, Super,
98
99	// Type or function attributes.
100	Pub, Extern, Static,
101
102	// Trait-related tokens.
103	Trait, Instance,
104
105	// Type-related tokens.
106	I8, I16, I32, I64, U8, U16, U32, U64,
107	Opaque, Fn, Bool, Union, Record, As,
108
109	// Safety.
110	Unsafe,
111	}
112
113	/// A reserved keyword.
114	record Keyword {
115	/// Keyword string.
116	name: *[u8],
117	/// Corresponding token.
118	tok: TokenKind,
119	}
120
121	/// Sorted keyword table for binary search.
122	const KEYWORDS: [Keyword; 52] = [
123	{ name: "align", tok: TokenKind::Align },
124	{ name: "and", tok: TokenKind::And },
125	{ name: "as", tok: TokenKind::As },
126	{ name: "assert", tok: TokenKind::Assert },
127	{ name: "bool", tok: TokenKind::Bool },
128	{ name: "break", tok: TokenKind::Break },
129	{ name: "case", tok: TokenKind::Case },
130	{ name: "catch", tok: TokenKind::Catch },
131	{ name: "const", tok: TokenKind::Const },
132	{ name: "continue", tok: TokenKind::Continue },
133	{ name: "else", tok: TokenKind::Else },
134	{ name: "extern", tok: TokenKind::Extern },
135	{ name: "false", tok: TokenKind::False },
136	{ name: "fn", tok: TokenKind::Fn },
137	{ name: "for", tok: TokenKind::For },
138	{ name: "i16", tok: TokenKind::I16 },
139	{ name: "i32", tok: TokenKind::I32 },
140	{ name: "i64", tok: TokenKind::I64 },
141	{ name: "i8", tok: TokenKind::I8 },
142	{ name: "if", tok: TokenKind::If },
143	{ name: "in", tok: TokenKind::In },
144	{ name: "instance", tok: TokenKind::Instance },
145	{ name: "let", tok: TokenKind::Let },
146	{ name: "log", tok: TokenKind::Log },
147	{ name: "loop", tok: TokenKind::Loop },
148	{ name: "match", tok: TokenKind::Match },
149	{ name: "mod", tok: TokenKind::Mod },
150	{ name: "mut", tok: TokenKind::Mut },
151	{ name: "nil", tok: TokenKind::Nil },
152	{ name: "not", tok: TokenKind::Not },
153	{ name: "opaque", tok: TokenKind::Opaque },
154	{ name: "or", tok: TokenKind::Or },
155	{ name: "panic", tok: TokenKind::Panic },
156	{ name: "pub", tok: TokenKind::Pub },
157	{ name: "record", tok: TokenKind::Record },
158	{ name: "return", tok: TokenKind::Return },
159	{ name: "static", tok: TokenKind::Static },
160	{ name: "super", tok: TokenKind::Super },
161	{ name: "throw", tok: TokenKind::Throw },
162	{ name: "throws", tok: TokenKind::Throws },
163	{ name: "trait", tok: TokenKind::Trait },
164	{ name: "true", tok: TokenKind::True },
165	{ name: "try", tok: TokenKind::Try },
166	{ name: "u16", tok: TokenKind::U16 },
167	{ name: "u32", tok: TokenKind::U32 },
168	{ name: "u64", tok: TokenKind::U64 },
169	{ name: "u8", tok: TokenKind::U8 },
170	{ name: "undefined", tok: TokenKind::Undefined },
171	{ name: "union", tok: TokenKind::Union },
172	{ name: "unsafe", tok: TokenKind::Unsafe },
173	{ name: "use", tok: TokenKind::Use },
174	{ name: "while", tok: TokenKind::While },
175	];
176
177	/// Describes where source code originated from.
178	pub union SourceLoc {
179	/// Source loaded from a file at the given path.
180	File(*[u8]),
181	/// Source provided as an inline string (no file path).
182	String,
183	}
184
185	/// Lexical scanner state for tokenizing Radiance source code.
186	///
187	/// Maintains position information and source buffer reference.
188	pub record Scanner {
189	/// Origin of the source being scanned.
190	sourceLoc: SourceLoc,
191	/// Source buffer.
192	source: *[u8],
193	/// Offset of current token into buffer.
194	token: u32,
195	/// Offset of current character being scanned.
196	cursor: u32,
197	/// Interned string pool.
198	pool: *mut strings::Pool,
199	}
200
201	/// Individual token with kind, source text, and position.
202	///
203	/// Represents a single lexical element extracted from source,
204	/// including its original text and byte offset for error reporting.
205	pub record Token {
206	/// Token kind.
207	kind: TokenKind,
208	/// Token source string.
209	source: *[u8],
210	/// Byte offset of `source` in input buffer.
211	offset: u32,
212	}
213
214	/// Source code location with line/column information.
215	///
216	/// Used for error reporting and debugging.
217	pub record Location {
218	/// Origin of the source.
219	source: SourceLoc,
220	/// Line number.
221	line: u16,
222	/// Column number.
223	col: u16,
224	}
225
226	/// Create a new scanner object.
227	pub fn scanner(sourceLoc: SourceLoc, source: [u8], pool: mut strings::Pool) -> Scanner {
228	// Intern built-in functions and attributes.
229	strings::intern(pool, "@sizeOf");
230	strings::intern(pool, "@alignOf");
231	strings::intern(pool, "@sliceOf");
232	strings::intern(pool, "@default");
233	strings::intern(pool, "@intrinsic");
234	strings::intern(pool, "@test");
235	// Intern built-in slice methods.
236	strings::intern(pool, "append");
237	strings::intern(pool, "delete");
238
239	return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
240	}
241
242	/// Check if we've reached the end of input.
243	pub fn isEof(s: *Scanner) -> bool {
244	return s.cursor >= s.source.len;
245	}
246
247	/// Get the current character, if any.
248	pub fn current(s: *Scanner) -> ?u8 {
249	if isEof(s) {
250	return nil;
251	}
252	return s.source[s.cursor];
253	}
254
255	/// Peek at the next character without advancing the scanner.
256	fn peek(s: *Scanner) -> ?u8 {
257	if s.cursor + 1 >= s.source.len {
258	return nil;
259	}
260	return s.source[s.cursor + 1];
261	}
262
263	/// Advance scanner and return the character that was consumed.
264	fn advance(s: *mut Scanner) -> u8 {
265	s.cursor += 1;
266	return s.source[s.cursor - 1];
267	}
268
269	/// Consume the expected character if it matches the current position.
270	fn consume(s: *mut Scanner, expected: u8) -> bool {
271	if let c = current(s); c == expected {
272	advance(s);
273	return true;
274	}
275	return false;
276	}
277
278	/// Create a token from the current scanner state.
279	fn tok(s: *Scanner, kind: TokenKind) -> Token {
280	return Token { kind, source: &s.source[s.token..s.cursor], offset: s.token };
281	}
282
283	/// Create an invalid token with the given message.
284	pub fn invalid(offset: u32, message: *[u8]) -> Token {
285	return Token { kind: TokenKind::Invalid, source: message, offset };
286	}
287
288	/// Skip whitespace characters and line comments.
289	fn skipWhitespace(s: *mut Scanner) {
290	while let ch = current(s) {
291	match ch {
292	case ' ', '\n', '\r', '\t' => advance(s),
293	case '/' => {
294	if let c = peek(s); c == '/' {
295	while let ch = current(s); ch != '\n' {
296	advance(s);
297	}
298	} else {
299	return;
300	}
301	}
302	else => return,
303	}
304	}
305	}
306
307	/// Check if character is an ASCII digit (0-9).
308	fn isDigit(c: u8) -> bool {
309	return c >= '0' and c <= '9';
310	}
311
312	/// Check if character is a hexadecimal digit (0-9, a-f, A-F).
313	fn isHexDigit(c: u8) -> bool {
314	return (c >= '0' and c <= '9')
315	or (c >= 'a' and c <= 'f')
316	or (c >= 'A' and c <= 'F');
317	}
318
319	/// Check if character is a binary digit (0 or 1).
320	fn isBinDigit(c: u8) -> bool {
321	return c == '0' or c == '1';
322	}
323
324	/// Check if character is alphabetic.
325	fn isAlpha(c: u8) -> bool {
326	return (c >= 'a' and c <= 'z')
327	or (c >= 'A' and c <= 'Z');
328	}
329
330	/// Check if character is printable ASCII.
331	fn isPrint(c: u8) -> bool {
332	return c >= ' ' and c <= '~';
333	}
334
335	/// Scan numeric literal (decimal, hex, or binary).
336	fn scanNumber(s: *mut Scanner) -> Token {
337	let first = s.source[s.cursor - 1];
338	if first == '-' or first == '+' {
339	advance(s);
340	}
341	// Check for hex literal (`0x` or `0X` prefix).
342	if s.source[s.cursor - 1] == '0' {
343	if let ch = current(s); ch == 'x' or ch == 'X' {
344	advance(s);
345	// Must have at least one hex digit after `0x`.
346	if let ch = current(s); not isHexDigit(ch) {
347	return invalid(s.token, "invalid hex literal");
348	}
349	while let ch = current(s); isHexDigit(ch) {
350	advance(s);
351	}
352	return tok(s, TokenKind::Number);
353	}
354	// Check for binary literal (`0b` or `0B` prefix).
355	if let ch = current(s); ch == 'b' or ch == 'B' {
356	advance(s);
357	// Must have at least one binary digit after `0b`.
358	if let ch = current(s); not isBinDigit(ch) {
359	return invalid(s.token, "invalid binary literal");
360	}
361	while let ch = current(s); isBinDigit(ch) {
362	advance(s);
363	}
364	return tok(s, TokenKind::Number);
365	}
366	}
367
368	// Regular decimal number.
369	while let ch = current(s); isDigit(ch) {
370	advance(s);
371	}
372
373	// Look for decimal part.
374	if let ch = current(s); ch == '.' {
375	if let p = peek(s); isDigit(p) {
376	advance(s); // Consume the "."
377	while let ch = current(s); isDigit(ch) {
378	advance(s);
379	}
380	}
381	}
382	return tok(s, TokenKind::Number);
383	}
384
385	fn scanDelimited(s: *mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
386	while let ch = current(s); ch != delim {
387	if not isPrint(ch) {
388	return invalid(s.token, "invalid character");
389	}
390	consume(s, '\\'); // Consume escapes
391	advance(s);
392	}
393	if not consume(s, delim) {
394	return nil;
395	}
396	return tok(s, kind);
397	}
398
399	/// Scan string literal enclosed in double quotes.
400	fn scanString(s: *mut Scanner) -> Token {
401	if let tok = scanDelimited(s, '"', TokenKind::String) {
402	return tok;
403	}
404	return invalid(s.token, "unterminated string");
405	}
406
407	/// Scan character literal enclosed in single quotes.
408	fn scanChar(s: *mut Scanner) -> Token {
409	if let tok = scanDelimited(s, '\'', TokenKind::Char) {
410	return tok;
411	}
412	return invalid(s.token, "unterminated character");
413	}
414
415	/// Scan a keyword or an identifier.
416	fn keywordOrIdent(src: *[u8]) -> TokenKind {
417	let mut left: u32 = 0;
418	let mut right: u32 = KEYWORDS.len;
419
420	while left < right {
421	let mid = left + ((right - left) / 2);
422	let kw = KEYWORDS[mid];
423	let cmp = mem::cmp(src, kw.name);
424
425	match cmp {
426	case -1 => right = mid,
427	case 1 => left = mid + 1,
428	else => return kw.tok,
429	}
430	}
431	return TokenKind::Ident;
432	}
433
434	/// Scan an identifier, keyword, or label.
435	fn scanIdentifier(s: *mut Scanner) -> Token {
436	while let ch = current(s); isAlpha(ch) or isDigit(ch) or ch == '_' or ch == '#' {
437	advance(s);
438	}
439	let ident = &s.source[s.token..s.cursor];
440	let kind = keywordOrIdent(ident);
441
442	// Only intern actual identifiers, not keywords.
443	if kind == TokenKind::Ident {
444	return Token { kind, source: strings::intern(s.pool, ident), offset: s.token };
445	}
446	return tok(s, kind);
447	}
448
449	/// Scan the next token.
450	pub fn next(s: *mut Scanner) -> Token {
451	skipWhitespace(s); // Skip any whitespace between tokens.
452	s.token = s.cursor; // Token starts at current position.
453
454	if isEof(s) {
455	return tok(s, TokenKind::Eof);
456	}
457	let c: u8 = advance(s);
458
459	if isDigit(c) {
460	return scanNumber(s);
461	}
462	if isAlpha(c) {
463	return scanIdentifier(s);
464	}
465	match c {
466	case '\'' => return scanChar(s),
467	case '"' => return scanString(s),
468	case '(' => return tok(s, TokenKind::LParen),
469	case ')' => return tok(s, TokenKind::RParen),
470	case '{' => return tok(s, TokenKind::LBrace),
471	case '}' => return tok(s, TokenKind::RBrace),
472	case '[' => return tok(s, TokenKind::LBracket),
473	case ']' => return tok(s, TokenKind::RBracket),
474	case ';' => return tok(s, TokenKind::Semicolon),
475	case ',' => return tok(s, TokenKind::Comma),
476	case '.' => {
477	if consume(s, '.') {
478	return tok(s, TokenKind::DotDot);
479	}
480	return tok(s, TokenKind::Dot);
481	}
482	case ':' => {
483	if consume(s, ':') {
484	return tok(s, TokenKind::ColonColon);
485	}
486	return tok(s, TokenKind::Colon);
487	}
488	case '-' => {
489	if consume(s, '>') {
490	return tok(s, TokenKind::Arrow);
491	}
492	if consume(s, '=') {
493	return tok(s, TokenKind::MinusEqual);
494	}
495	// If followed by a digit, scan as negative number
496	if let ch = current(s); isDigit(ch) {
497	return scanNumber(s);
498	}
499	return tok(s, TokenKind::Minus);
500	}
501	case '+' => {
502	if consume(s, '=') {
503	return tok(s, TokenKind::PlusEqual);
504	}
505	if let ch = current(s); isDigit(ch) {
506	return scanNumber(s);
507	}
508	return tok(s, TokenKind::Plus);
509	}
510	case '/' => {
511	if consume(s, '=') {
512	return tok(s, TokenKind::SlashEqual);
513	}
514	return tok(s, TokenKind::Slash);
515	}
516	case '*' => {
517	if consume(s, '=') {
518	return tok(s, TokenKind::StarEqual);
519	}
520	return tok(s, TokenKind::Star);
521	}
522	case '%' => {
523	if consume(s, '=') {
524	return tok(s, TokenKind::PercentEqual);
525	}
526	return tok(s, TokenKind::Percent);
527	}
528	case '&' => {
529	if consume(s, '=') {
530	return tok(s, TokenKind::AmpEqual);
531	}
532	return tok(s, TokenKind::Amp);
533	}
534	case '?' => return tok(s, TokenKind::Question),
535	case '\|' => {
536	if consume(s, '=') {
537	return tok(s, TokenKind::PipeEqual);
538	}
539	return tok(s, TokenKind::Pipe);
540	}
541	case '^' => {
542	if consume(s, '=') {
543	return tok(s, TokenKind::CaretEqual);
544	}
545	return tok(s, TokenKind::Caret);
546	}
547	case '~' => return tok(s, TokenKind::Tilde),
548	case '!' => {
549	if consume(s, '=') {
550	return tok(s, TokenKind::BangEqual);
551	}
552	return tok(s, TokenKind::Bang);
553	}
554	case '=' => {
555	if consume(s, '>') {
556	return tok(s, TokenKind::FatArrow);
557	}
558	if consume(s, '=') {
559	return tok(s, TokenKind::EqualEqual);
560	}
561	return tok(s, TokenKind::Equal);
562	}
563	case '<' => {
564	if consume(s, '<') {
565	if consume(s, '=') {
566	return tok(s, TokenKind::LtLtEqual);
567	}
568	return tok(s, TokenKind::LtLt);
569	}
570	if consume(s, '=') {
571	return tok(s, TokenKind::LtEqual);
572	}
573	return tok(s, TokenKind::Lt);
574	}
575	case '>' => {
576	if consume(s, '>') {
577	if consume(s, '=') {
578	return tok(s, TokenKind::GtGtEqual);
579	}
580	return tok(s, TokenKind::GtGt);
581	}
582	if consume(s, '=') {
583	return tok(s, TokenKind::GtEqual);
584	}
585	return tok(s, TokenKind::Gt);
586	}
587	case '@' => {
588	// Scan `@identifier` as a single token.
589	while let ch = current(s); isAlpha(ch) {
590	advance(s);
591	}
592	// Must have at least one character after `@`.
593	if s.cursor - s.token <= 1 {
594	return invalid(s.token, "expected identifier after `@`");
595	}
596	let name = &s.source[s.token..s.cursor];
597	return Token {
598	kind: TokenKind::AtIdent,
599	source: strings::intern(s.pool, name),
600	offset: s.token,
601	};
602	}
603	case '_' => {
604	if let ch = current(s); isAlpha(ch) or isDigit(ch) or ch == '_' {
605	// This is part of an identifier like `_foo` or `__start`
606	return scanIdentifier(s);
607	}
608	return tok(s, TokenKind::Underscore);
609	}
610	else => return invalid(s.token, "unexpected character"),
611	}
612	}
613
614	/// Get the source code location from a byte offset.
615	pub fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
616	let mut l: u16 = 1;
617	let mut c: u16 = 1;
618
619	if offset >= source.len {
620	return nil;
621	}
622	for ch in &source[..offset] {
623	if ch == '\n' {
624	c = 1;
625	l += 1;
626	} else {
627	c += 1;
628	}
629	}
630	return Location { source: sourceLoc, line: l, col: c };
631	}