Print the actual parse errors if any

c8b1ee9e80b4d280fe13cbc5a325474aebd26e8e1b3dd218bc39a55fdcf8b247
Alexis Sellier committed ago 1 parent c4b1fccc
compiler/radiance.rad +2 -2
3 3
use std::fmt;
4 4
use std::io;
5 5
use std::lang::alloc;
6 6
use std::lang::ast;
7 7
use std::lang::parser;
8 +
use std::lang::scanner;
8 9
use std::lang::resolver;
9 10
use std::lang::module;
10 11
use std::lang::strings;
11 12
use std::lang::package;
12 13
use std::lang::il;
176 177
        throw Error::Other;
177 178
    }
178 179
    // Commit only what was read.
179 180
    alloc::commit(sourceArena, source.len);
180 181
181 -
    let ast = try parser::parse(source, nodeArena, &mut STRING_POOL) catch {
182 -
        io::printError("radiance: error parsing module\n");
182 +
    let ast = try parser::parse(scanner::SourceLoc::File(path), source, nodeArena, &mut STRING_POOL) catch {
183 183
        throw Error::Other;
184 184
    };
185 185
    try module::setAst(graph, moduleId, ast) catch {
186 186
        io::printError("radiance: error setting AST\n");
187 187
        throw Error::Other;
lib/std/lang/parser.rad +28 -13
144 144
    allocator: alloc::Allocator,
145 145
    /// Current parsing context (normal or conditional).
146 146
    context: Context,
147 147
}
148 148
149 -
/// Create a new parser initialized with the given source and node arena.
150 -
pub fn mkParser(source: *[u8], arena: *mut ast::NodeArena, pool: *mut strings::Pool) -> Parser {
149 +
/// Create a new parser initialized with the given source kind, source and node arena.
150 +
pub fn mkParser(sourceLoc: scanner::SourceLoc, source: *[u8], arena: *mut ast::NodeArena, pool: *mut strings::Pool) -> Parser {
151 151
    return Parser {
152 -
        scanner: scanner::scanner("", source, pool),
152 +
        scanner: scanner::scanner(sourceLoc, source, pool),
153 153
        current: scanner::invalid(0, ""),
154 154
        previous: scanner::invalid(0, ""),
155 155
        errors: ErrorList { list: undefined, count: 0 },
156 156
        arena,
157 157
        allocator: ast::nodeAllocator(arena),
1156 1156
    }
1157 1157
}
1158 1158
1159 1159
/// Fail the parsing process with the given error.
1160 1160
fn failParsing(p: *mut Parser, err: *[u8]) -> ParseError {
1161 -
    reportError(p, p.previous, err);
1161 +
    reportError(p, p.current, err);
1162 1162
    return ParseError::UnexpectedToken;
1163 1163
}
1164 1164
1165 1165
/// Print all errors that have been collected during parsing.
1166 -
fn printErrors(p: *Parser) {
1166 +
pub fn printErrors(p: *Parser) {
1167 1167
    for i in 0..p.errors.count {
1168 1168
        let e = p.errors.list[i];
1169 1169
        if let loc = scanner::getLocation(
1170 -
            p.scanner.file, p.scanner.source, e.token.offset
1170 +
            p.scanner.sourceLoc, p.scanner.source, e.token.offset
1171 1171
        ) {
1172 +
            if let case scanner::SourceLoc::File(path) = loc.source {
1173 +
                io::print(path);
1174 +
                io::print(":");
1175 +
            }
1172 1176
            io::printU32(loc.line as u32);
1173 1177
            io::print(":");
1174 1178
            io::printU32(loc.col as u32);
1175 -
            io::print(": ");
1179 +
            io::print(": error: ");
1180 +
        } else {
1181 +
            io::print("error: ");
1176 1182
        }
1177 1183
        io::print(e.message);
1178 -
        io::print(", got `");
1179 -
        io::print(scanner::tokenKindToString(e.token.kind));
1180 -
        io::printLn("`");
1184 +
        if e.token.kind == scanner::TokenKind::Invalid {
1185 +
            io::print(": ");
1186 +
            io::print(e.token.source);
1187 +
        } else {
1188 +
            io::print(", got `");
1189 +
            io::print(e.token.source);
1190 +
            io::print("`");
1191 +
        }
1192 +
        io::print("\n");
1181 1193
    }
1182 1194
}
1183 1195
1184 1196
/// Check whether the current token matches the expected kind.
1185 1197
pub fn check(p: *Parser, kind: scanner::TokenKind) -> bool {
2246 2258
        ident: binding.name, type: binding.type, value, alignment: binding.alignment, mutable,
2247 2259
    }));
2248 2260
}
2249 2261
2250 2262
/// Parse a module from source text using the provided arena for node storage.
2251 -
pub fn parse(input: *[u8], arena: *mut ast::NodeArena, pool: *mut strings::Pool) -> *mut ast::Node
2263 +
pub fn parse(sourceLoc: scanner::SourceLoc, input: *[u8], arena: *mut ast::NodeArena, pool: *mut strings::Pool) -> *mut ast::Node
2252 2264
    throws (ParseError)
2253 2265
{
2254 -
    let mut p = mkParser(input, arena, pool);
2255 -
    return try parseModule(&mut p);
2266 +
    let mut p = mkParser(sourceLoc, input, arena, pool);
2267 +
    return try parseModule(&mut p) catch {
2268 +
        printErrors(&p);
2269 +
        throw ParseError::UnexpectedToken;
2270 +
    };
2256 2271
}
2257 2272
2258 2273
/// Parse a complete module into a block of top-level statements.
2259 2274
///
2260 2275
/// This is the main entry point for parsing an entire Radiance source file.
lib/std/lang/parser/tests.rad +6 -6
69 69
/// Parse multiple statements from a string.
70 70
fn parseStmtsStr(input: *[u8]) -> *ast::Node
71 71
    throws (testing::TestError)
72 72
{
73 73
    let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]);
74 -
    let mut parser = super::mkParser(input, &mut arena, &mut STRING_POOL);
74 +
    let mut parser = super::mkParser(scanner::SourceLoc::String, input, &mut arena, &mut STRING_POOL);
75 75
    return try super::parseModule(&mut parser) catch {
76 76
        throw testing::TestError::Failed;
77 77
    };
78 78
}
79 79
80 80
/// Parse a single type from a string.
81 81
fn parseTypeStr(input: *[u8]) -> *ast::Node
82 82
    throws (super::ParseError)
83 83
{
84 84
    let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]);
85 -
    let mut parser = super::mkParser(input, &mut arena, &mut STRING_POOL);
85 +
    let mut parser = super::mkParser(scanner::SourceLoc::String, input, &mut arena, &mut STRING_POOL);
86 86
    super::advance(&mut parser);
87 87
    let root = try super::parseType(&mut parser);
88 88
    try super::expect(&mut parser, scanner::TokenKind::Eof, "expected end of type");
89 89
90 90
    return root;
93 93
/// Parse a single expression from a string.
94 94
pub fn parseExprStr(input: *[u8]) -> *ast::Node
95 95
    throws (super::ParseError)
96 96
{
97 97
    let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]);
98 -
    let mut parser = super::mkParser(input, &mut arena, &mut STRING_POOL);
98 +
    let mut parser = super::mkParser(scanner::SourceLoc::String, input, &mut arena, &mut STRING_POOL);
99 99
    super::advance(&mut parser);
100 100
    return try super::parseExpr(&mut parser);
101 101
}
102 102
103 103
/// Parse a single statement from a string.
104 104
fn parseStmtStr(input: *[u8]) -> *ast::Node
105 105
    throws (super::ParseError)
106 106
{
107 107
    let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]);
108 -
    let mut parser = super::mkParser(input, &mut arena, &mut STRING_POOL);
108 +
    let mut parser = super::mkParser(scanner::SourceLoc::String, input, &mut arena, &mut STRING_POOL);
109 109
    super::advance(&mut parser);
110 110
    let root = try super::parseStmt(&mut parser);
111 111
    while super::consume(&mut parser, scanner::TokenKind::Semicolon) {}
112 112
    try super::expect(&mut parser, scanner::TokenKind::Eof, "expected end of statement");
113 113
117 117
/// Parse an expression expected to be a number literal and return its payload.
118 118
fn parseNumberLiteral(text: *[u8]) -> ast::IntLiteral
119 119
    throws (testing::TestError)
120 120
{
121 121
    let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]);
122 -
    let mut parser = super::mkParser(text, &mut arena, &mut STRING_POOL);
122 +
    let mut parser = super::mkParser(scanner::SourceLoc::String, text, &mut arena, &mut STRING_POOL);
123 123
    super::advance(&mut parser);
124 124
125 125
    let node = try! super::parseExpr(&mut parser);
126 126
127 127
    if not super::check(&parser, scanner::TokenKind::Eof) {
136 136
/// Ensure that parsing the supplied literal source fails.
137 137
fn expectNumberLiteralFail(text: *[u8])
138 138
    throws (testing::TestError)
139 139
{
140 140
    let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]);
141 -
    let mut parser = super::mkParser(text, &mut arena, &mut STRING_POOL);
141 +
    let mut parser = super::mkParser(scanner::SourceLoc::String, text, &mut arena, &mut STRING_POOL);
142 142
    super::advance(&mut parser);
143 143
144 144
    try super::parseExpr(&mut parser) catch {
145 145
        return;
146 146
    };
lib/std/lang/resolver/printer.rad +5 -3
302 302
        // Find the module containing this error.
303 303
        if let moduleEntry = module::get(res.moduleGraph, err.moduleId) {
304 304
            // Get the source text if available.
305 305
            if let source = moduleEntry.source {
306 306
                // Convert offset to location.
307 -
                if let loc = scanner::getLocation(moduleEntry.filePath, source, node.span.offset) {
307 +
                if let loc = scanner::getLocation(scanner::SourceLoc::File(moduleEntry.filePath), source, node.span.offset) {
308 308
                    // Print: filename:line:col: error: message
309 -
                    io::print(loc.file);
310 -
                    io::print(":");
309 +
                    if let case scanner::SourceLoc::File(path) = loc.source {
310 +
                        io::print(path);
311 +
                        io::print(":");
312 +
                    }
311 313
                    io::printU32(loc.line as u32);
312 314
                    io::print(":");
313 315
                    io::printU32(loc.col as u32);
314 316
                    io::print(": error: ");
315 317
                } else {
lib/std/lang/resolver/tests.rad +4 -4
76 76
}
77 77
78 78
/// Parse and analyze an expression string for testing.
79 79
fn resolveExprStr(self: *mut super::Resolver, stmt: *[u8]) -> TestResult throws (testing::TestError) {
80 80
    let mut arena = ast::nodeArena(&mut AST_ARENA[..]);
81 -
    let mut p = parser::mkParser(stmt, &mut arena, &mut STRING_POOL);
81 +
    let mut p = parser::mkParser(scanner::SourceLoc::String, stmt, &mut arena, &mut STRING_POOL);
82 82
    parser::advance(&mut p);
83 83
84 84
    let expr = try parser::parseExpr(&mut p) catch {
85 85
        panic "resolveExprStr: parsing failed";
86 86
    };
92 92
93 93
/// Parse and analyze a module string for testing.
94 94
/// Use this for code with `fn`, `record`, `union`, etc. at the top level.
95 95
fn resolveProgramStr(self: *mut super::Resolver, stmt: *[u8]) -> TestResult throws (testing::TestError) {
96 96
    let mut arena = ast::nodeArena(&mut AST_ARENA[..]);
97 -
    let stmt = try parser::parse(stmt, &mut arena, &mut STRING_POOL) catch {
97 +
    let stmt = try parser::parse(scanner::SourceLoc::String, stmt, &mut arena, &mut STRING_POOL) catch {
98 98
        panic "resolveProgramStr: parsing failed";
99 99
    };
100 100
    let diagnostics = try super::resolveModuleRoot(self, stmt) catch {
101 101
        throw testing::TestError::Failed;
102 102
    };
105 105
106 106
/// Parse and analyze a block of statements (eg. inside a function body) for testing.
107 107
/// Use this for code with `let` bindings and expressions, not module-level declarations.
108 108
fn resolveBlockStr(self: *mut super::Resolver, stmt: *[u8]) -> TestResult throws (testing::TestError) {
109 109
    let mut arena = ast::nodeArena(&mut AST_ARENA[..]);
110 -
    let parsed = try parser::parse(stmt, &mut arena, &mut STRING_POOL) catch {
110 +
    let parsed = try parser::parse(scanner::SourceLoc::String, stmt, &mut arena, &mut STRING_POOL) catch {
111 111
        panic "resolveBlockStr: parsing failed";
112 112
    };
113 113
    let case ast::NodeValue::Block(block) = parsed.value
114 114
        else panic "resolveBlockStr: expected block root";
115 115
159 159
    } else {
160 160
        modId = try module::registerRootWithName(graph, 0, name, filePath) catch {
161 161
            throw testing::TestError::Failed;
162 162
        };
163 163
    }
164 -
    let root = try parser::parse(code, arena, &mut STRING_POOL) catch {
164 +
    let root = try parser::parse(scanner::SourceLoc::String, code, arena, &mut STRING_POOL) catch {
165 165
        panic "registerModule: parsing failed";
166 166
    };
167 167
    try module::setAst(graph, modId, root) catch {
168 168
        panic "registerModule: module not found";
169 169
    };
lib/std/lang/scanner.rad +18 -10
282 282
    { name: "use", tok: TokenKind::Use },
283 283
    { name: "void", tok: TokenKind::Void },
284 284
    { name: "while", tok: TokenKind::While },
285 285
];
286 286
287 +
/// Describes where source code originated from.
288 +
pub union SourceLoc {
289 +
    /// Source loaded from a file at the given path.
290 +
    File(*[u8]),
291 +
    /// Source provided as an inline string (no file path).
292 +
    String,
293 +
}
294 +
287 295
/// Lexical scanner state for tokenizing Radiance source code.
288 296
///
289 297
/// Maintains position information and source buffer reference.
290 298
pub record Scanner {
291 -
    /// File path.
292 -
    file: *[u8],
299 +
    /// Origin of the source being scanned.
300 +
    sourceLoc: SourceLoc,
293 301
    /// Source buffer.
294 302
    source: *[u8],
295 303
    /// Offset of current token into buffer.
296 304
    token: u32,
297 305
    /// Offset of current character being scanned.
311 319
    source: *[u8],
312 320
    /// Byte offset of `source` in input buffer.
313 321
    offset: u32,
314 322
}
315 323
316 -
/// Source code location with file and line/column information.
324 +
/// Source code location with line/column information.
317 325
///
318 326
/// Used for error reporting and debugging.
319 327
pub record Location {
320 -
    /// File path.
321 -
    file: *[u8],
322 -
    /// line number.
328 +
    /// Origin of the source.
329 +
    source: SourceLoc,
330 +
    /// Line number.
323 331
    line: u16,
324 332
    /// Column number.
325 333
    col: u16,
326 334
}
327 335
328 336
/// Create a new scanner object.
329 -
pub fn scanner(file: *[u8], source: *[u8], pool: *mut strings::Pool) -> Scanner {
337 +
pub fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: *mut strings::Pool) -> Scanner {
330 338
    // Intern built-in functions and attributes.
331 339
    strings::intern(pool, "@sizeOf");
332 340
    strings::intern(pool, "@alignOf");
333 341
    strings::intern(pool, "@sliceOf");
334 342
    strings::intern(pool, "@default");
336 344
    strings::intern(pool, "@test");
337 345
    // Intern built-in slice methods.
338 346
    strings::intern(pool, "append");
339 347
    strings::intern(pool, "delete");
340 348
341 -
    return Scanner { file, source, token: 0, cursor: 0, pool };
349 +
    return Scanner { sourceLoc, source, token: 0, cursor: 0, pool };
342 350
}
343 351
344 352
/// Check if we've reached the end of input.
345 353
pub fn isEof(s: *Scanner) -> bool {
346 354
    return s.cursor >= s.source.len;
712 720
        else => return invalid(s.token, "unexpected character"),
713 721
    }
714 722
}
715 723
716 724
/// Get the source code location from a byte offset.
717 -
pub fn getLocation(file: *[u8], source: *[u8], offset: u32) -> ?Location {
725 +
pub fn getLocation(sourceLoc: SourceLoc, source: *[u8], offset: u32) -> ?Location {
718 726
    let mut l: u16 = 1;
719 727
    let mut c: u16 = 1;
720 728
721 729
    if offset >= source.len {
722 730
        return nil;
727 735
            l += 1;
728 736
        } else {
729 737
            c += 1;
730 738
        }
731 739
    }
732 -
    return Location { file, line: l, col: c };
740 +
    return Location { source: sourceLoc, line: l, col: c };
733 741
}
lib/std/lang/scanner/tests.rad +5 -5
4 4
5 5
/// String pool for testing.
6 6
static TEST_STRING_POOL: strings::Pool = strings::Pool { table: undefined, count: 0 };
7 7
8 8
fn testScanner(source: *[u8]) -> super::Scanner {
9 -
    return super::scanner("test.r", source, &mut TEST_STRING_POOL);
9 +
    return super::scanner(super::SourceLoc::File("test.r"), source, &mut TEST_STRING_POOL);
10 10
}
11 11
12 12
@test fn testScanTokens() throws (testing::TestError) {
13 13
    let mut s = testScanner(
14 14
        "'x' < fnord fnord: 0 => >> mod and nil"
98 98
    );
99 99
    let mut tok: super::Token = super::next(&mut s);
100 100
    try testing::expect(tok.kind == super::TokenKind::Ident);
101 101
    try testing::expect(tok.source.len == 3);
102 102
103 -
    if let loc = super::getLocation(s.file, s.source, tok.offset) {
103 +
    if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) {
104 104
        try testing::expect(loc.line == 1);
105 105
        try testing::expect(loc.col == 1);
106 106
    } else {
107 107
        try testing::expect(false);
108 108
    }
109 109
    tok = super::next(&mut s);
110 110
    try testing::expect(tok.kind == super::TokenKind::Ident);
111 111
    try testing::expect(tok.source.len == 3);
112 112
113 -
    if let loc = super::getLocation(s.file, s.source, tok.offset) {
113 +
    if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) {
114 114
        try testing::expect(loc.line == 2);
115 115
        try testing::expect(loc.col == 3);
116 116
    } else {
117 117
        try testing::expect(false);
118 118
    }
119 119
    tok = super::next(&mut s);
120 120
    try testing::expect(tok.kind == super::TokenKind::Ident);
121 121
    try testing::expect(tok.source.len == 3);
122 122
123 -
    if let loc = super::getLocation(s.file, s.source, tok.offset) {
123 +
    if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) {
124 124
        try testing::expect(loc.line == 3);
125 125
        try testing::expect(loc.col == 5);
126 126
    } else {
127 127
        try testing::expect(false);
128 128
    }
129 129
}
130 130
131 131
@test fn testScanEmptyInput() throws (testing::TestError) {
132 132
    let mut s = super::Scanner {
133 -
        file: "test.r",
133 +
        sourceLoc: super::SourceLoc::File("test.r"),
134 134
        source: "",
135 135
        token: 0,
136 136
        cursor: 0,
137 137
        pool: &mut TEST_STRING_POOL,
138 138
    };
test/lower/lower-test.rad +2 -1
11 11
use std::lang::alloc;
12 12
use std::lang::ast;
13 13
use std::lang::il;
14 14
use std::lang::il::printer;
15 15
use std::lang::parser;
16 +
use std::lang::scanner;
16 17
use std::lang::module;
17 18
use std::lang::resolver;
18 19
use std::lang::strings;
19 20
use std::lang::lower;
20 21
161 162
        io::printLn(expectedPath);
162 163
        return false;
163 164
    };
164 165
    // Parse source.
165 166
    let mut astArena = ast::nodeArena(&mut astArenaStorage[..]);
166 -
    let root = try parser::parse(source, &mut astArena, &mut STRING_POOL) catch {
167 +
    let root = try parser::parse(scanner::SourceLoc::String, source, &mut astArena, &mut STRING_POOL) catch {
167 168
        io::printLn("error: Parsing failed");
168 169
        return false;
169 170
    };
170 171
    // Run resolver.
171 172
    let mut moduleArena = ast::nodeArena(&mut moduleArenaStorage[..]);