compiler: Retain a checked parser arena borrow
e2a0b14e501e92b5cde2379ce6d5cf6363c3cb1d667644a06b9b596004ab4e8a
1 parent
929c447c
lib/std/lang/parser.rad
+9 -8
| 163 | 163 | current: scanner::Token, |
|
| 164 | 164 | /// The most recently consumed token. |
|
| 165 | 165 | previous: scanner::Token, |
|
| 166 | 166 | /// Collection of errors encountered during parsing. |
|
| 167 | 167 | errors: ErrorList, |
|
| 168 | - | /// Arena for node allocations. It must outlive the parser. |
|
| 169 | - | arena: *unsafe mut ast::NodeArena, |
|
| 168 | + | /// Arena for node allocations, borrowed for the parser's region. |
|
| 169 | + | arena: &'pool mut ast::NodeArena, |
|
| 170 | 170 | /// Allocator backed by the node arena. |
|
| 171 | 171 | allocator: alloc::Allocator, |
|
| 172 | 172 | /// Current parsing context (normal or conditional). |
|
| 173 | 173 | context: Context, |
|
| 174 | 174 | } |
|
| 175 | 175 | ||
| 176 | 176 | /// Create a new parser initialized with the given source kind, source and node arena. |
|
| 177 | 177 | /// The node arena and string pool must outlive the parser. |
|
| 178 | - | export unsafe fn mkParser 'pool (sourceLoc: scanner::SourceLoc, source: *[u8], arena: &mut ast::NodeArena, pool: &'pool mut strings::Pool) -> Parser 'pool { |
|
| 178 | + | export unsafe fn mkParser 'pool (sourceLoc: scanner::SourceLoc, source: *[u8], arena: &'pool mut ast::NodeArena, pool: &'pool mut strings::Pool) -> Parser 'pool { |
|
| 179 | + | let allocator = alloc::arenaAllocator(&mut arena.arena); |
|
| 179 | 180 | return Parser 'pool { |
|
| 180 | 181 | scanner: scanner::scanner(sourceLoc, source, pool), |
|
| 181 | 182 | pool, |
|
| 182 | 183 | current: scanner::invalid(0, ""), |
|
| 183 | 184 | previous: scanner::invalid(0, ""), |
|
| 184 | 185 | errors: ErrorList { list: [Error { message: "", token: scanner::invalid(0, "") }; MAX_ERRORS], count: 0 }, |
|
| 185 | - | arena: (&mut *arena) as *unsafe mut ast::NodeArena, |
|
| 186 | - | allocator: alloc::arenaAllocator(&mut arena.arena), |
|
| 186 | + | arena, |
|
| 187 | + | allocator, |
|
| 187 | 188 | context: Context::Normal, |
|
| 188 | 189 | }; |
|
| 189 | 190 | } |
|
| 190 | 191 | ||
| 191 | 192 | /// Emit a `true` or `false` literal node. |
| 1110 | 1111 | set node.span.length = 0; |
|
| 1111 | 1112 | } |
|
| 1112 | 1113 | } |
|
| 1113 | 1114 | ||
| 1114 | 1115 | /// Save parser state for speculative parsing. |
|
| 1115 | - | unsafe fn saveState 'pool (p: &Parser 'pool) -> SavedState { |
|
| 1116 | + | fn saveState 'pool (p: &Parser 'pool) -> SavedState { |
|
| 1116 | 1117 | return SavedState { |
|
| 1117 | 1118 | scanner: p.scanner, |
|
| 1118 | 1119 | current: p.current, |
|
| 1119 | 1120 | previous: p.previous, |
|
| 1120 | 1121 | errors: p.errors, |
| 2447 | 2448 | ||
| 2448 | 2449 | /// Parse a module from source text using the provided arena for node storage. |
|
| 2449 | 2450 | export unsafe fn parse(sourceLoc: scanner::SourceLoc, input: *[u8], arena: &mut ast::NodeArena, pool: &mut strings::Pool) -> *mut ast::Node |
|
| 2450 | 2451 | throws (ParseError) |
|
| 2451 | 2452 | { |
|
| 2452 | - | let poolRef: 'pool = &mut *pool in { |
|
| 2453 | - | let mut p = mkParser(sourceLoc, input, arena, poolRef); |
|
| 2453 | + | let poolRef: 'pool = &mut *pool, arenaRef = &mut *arena in { |
|
| 2454 | + | let mut p = mkParser(sourceLoc, input, arenaRef, poolRef); |
|
| 2454 | 2455 | return try parseModule(&mut p) catch { |
|
| 2455 | 2456 | printErrors(&p); |
|
| 2456 | 2457 | throw ParseError::UnexpectedToken; |
|
| 2457 | 2458 | }; |
|
| 2458 | 2459 | } |
lib/std/lang/parser/tests.rad
+32 -32
| 85 | 85 | } |
|
| 86 | 86 | ||
| 87 | 87 | /// Scanner state can advance without an unsafe token operation. |
|
| 88 | 88 | @test unsafe fn testSafeTokenOperations() throws (testing::TestError) { |
|
| 89 | 89 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 90 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 91 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, "alpha, beta", &mut arena, poolRef); |
|
| 90 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 91 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, "alpha, beta", arenaRef, poolRef); |
|
| 92 | 92 | try checkTokenOperations(&mut parser); |
|
| 93 | 93 | } |
|
| 94 | 94 | } |
|
| 95 | 95 | ||
| 96 | 96 | /// Parse multiple statements from a string. |
|
| 97 | 97 | unsafe fn parseStmtsStr(input: *[u8]) -> *ast::Node |
|
| 98 | 98 | throws (testing::TestError) |
|
| 99 | 99 | { |
|
| 100 | 100 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 101 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 102 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, input, &mut arena, poolRef); |
|
| 101 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 102 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, input, arenaRef, poolRef); |
|
| 103 | 103 | return try super::parseModule(&mut parser) catch { |
|
| 104 | 104 | throw testing::TestError::Failed; |
|
| 105 | 105 | }; |
|
| 106 | 106 | } |
|
| 107 | 107 | } |
| 109 | 109 | /// Parse a single type from a string. |
|
| 110 | 110 | unsafe fn parseTypeStr(input: *[u8]) -> *ast::Node |
|
| 111 | 111 | throws (super::ParseError) |
|
| 112 | 112 | { |
|
| 113 | 113 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 114 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 115 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, input, &mut arena, poolRef); |
|
| 114 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 115 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, input, arenaRef, poolRef); |
|
| 116 | 116 | super::advance(&mut parser); |
|
| 117 | 117 | let root = try super::parseType(&mut parser); |
|
| 118 | 118 | try super::expect(&mut parser, scanner::TokenKind::Eof, "expected end of type"); |
|
| 119 | 119 | ||
| 120 | 120 | return root; |
| 124 | 124 | /// Parse a single expression from a string. |
|
| 125 | 125 | export unsafe fn parseExprStr(input: *[u8]) -> *ast::Node |
|
| 126 | 126 | throws (super::ParseError) |
|
| 127 | 127 | { |
|
| 128 | 128 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 129 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 130 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, input, &mut arena, poolRef); |
|
| 129 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 130 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, input, arenaRef, poolRef); |
|
| 131 | 131 | super::advance(&mut parser); |
|
| 132 | 132 | return try super::parseExpr(&mut parser); |
|
| 133 | 133 | } |
|
| 134 | 134 | } |
|
| 135 | 135 | ||
| 136 | 136 | /// Parse a single statement from a string. |
|
| 137 | 137 | unsafe fn parseStmtStr(input: *[u8]) -> *ast::Node |
|
| 138 | 138 | throws (super::ParseError) |
|
| 139 | 139 | { |
|
| 140 | 140 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 141 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 142 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, input, &mut arena, poolRef); |
|
| 141 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 142 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, input, arenaRef, poolRef); |
|
| 143 | 143 | super::advance(&mut parser); |
|
| 144 | 144 | let root = try super::parseStmt(&mut parser); |
|
| 145 | 145 | while super::consume(&mut parser, scanner::TokenKind::Semicolon) {} |
|
| 146 | 146 | try super::expect(&mut parser, scanner::TokenKind::Eof, "expected end of statement"); |
|
| 147 | 147 |
| 152 | 152 | /// Parse an expression expected to be a number literal and return its payload. |
|
| 153 | 153 | unsafe fn parseNumberLiteral(text: *[u8]) -> fmt::IntLiteral |
|
| 154 | 154 | throws (testing::TestError) |
|
| 155 | 155 | { |
|
| 156 | 156 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 157 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 158 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, text, &mut arena, poolRef); |
|
| 157 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 158 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, text, arenaRef, poolRef); |
|
| 159 | 159 | super::advance(&mut parser); |
|
| 160 | 160 | ||
| 161 | 161 | let node = try! super::parseExpr(&mut parser); |
|
| 162 | 162 | ||
| 163 | 163 | if not super::check(&parser, scanner::TokenKind::Eof) { |
| 173 | 173 | /// Ensure that parsing the supplied literal source fails. |
|
| 174 | 174 | unsafe fn expectNumberLiteralFail(text: *[u8]) |
|
| 175 | 175 | throws (testing::TestError) |
|
| 176 | 176 | { |
|
| 177 | 177 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 178 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 179 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, text, &mut arena, poolRef); |
|
| 178 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 179 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, text, arenaRef, poolRef); |
|
| 180 | 180 | super::advance(&mut parser); |
|
| 181 | 181 | ||
| 182 | 182 | try super::parseExpr(&mut parser) catch { |
|
| 183 | 183 | return; |
|
| 184 | 184 | }; |
| 1158 | 1158 | } |
|
| 1159 | 1159 | ||
| 1160 | 1160 | /// Test scanning source-level `void` produces an identifier, not a type keyword. |
|
| 1161 | 1161 | @test unsafe fn testParseTypeVoidRejected() throws (testing::TestError) { |
|
| 1162 | 1162 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 1163 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 1164 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, "void", &mut arena, poolRef); |
|
| 1163 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 1164 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, "void", arenaRef, poolRef); |
|
| 1165 | 1165 | super::advance(&mut parser); |
|
| 1166 | 1166 | try testing::expect(super::check(&parser, scanner::TokenKind::Ident)); |
|
| 1167 | 1167 | } |
|
| 1168 | 1168 | } |
|
| 1169 | 1169 |
| 3246 | 3246 | /// Measure committed storage for a statement after an existing node. |
|
| 3247 | 3247 | unsafe fn statementStorageUsed(source: *[u8]) -> u32 { |
|
| 3248 | 3248 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 3249 | 3249 | ast::allocNode(&mut arena, ast::Span { offset: 0, length: 0 }, ast::NodeValue::Bool(true)); |
|
| 3250 | 3250 | let start = alloc::used(&arena.arena); |
|
| 3251 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 3252 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, source, &mut arena, poolRef); |
|
| 3251 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 3252 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, source, arenaRef, poolRef); |
|
| 3253 | 3253 | super::advance(&mut parser); |
|
| 3254 | 3254 | try! super::parseStmt(&mut parser); |
|
| 3255 | - | return alloc::used(&arena.arena) - start; |
|
| 3255 | + | return alloc::used(&parser.arena.arena) - start; |
|
| 3256 | 3256 | } |
|
| 3257 | 3257 | } |
|
| 3258 | 3258 | ||
| 3259 | 3259 | /// Rewinding a failed expression preserves published nodes and source tokens. |
|
| 3260 | 3260 | @test unsafe fn testSpeculativeRestoreStorage() throws (testing::TestError) { |
|
| 3261 | 3261 | let expectedBytes = statementStorageUsed("return"); |
|
| 3262 | 3262 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 3263 | 3263 | let retained = ast::allocNode(&mut arena, ast::Span { offset: 3, length: 1 }, ast::NodeValue::Bool(true)); |
|
| 3264 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 3264 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 3265 | 3265 | let mut parser = super::mkParser(scanner::SourceLoc::String, |
|
| 3266 | - | "return [speculativeRestoreIdentifier, 1 +", &mut arena, poolRef); |
|
| 3266 | + | "return [speculativeRestoreIdentifier, 1 +", arenaRef, poolRef); |
|
| 3267 | 3267 | super::advance(&mut parser); |
|
| 3268 | 3268 | try super::expect(&mut parser, scanner::TokenKind::Eof, "retained diagnostic") catch { |
|
| 3269 | 3269 | }; |
|
| 3270 | 3270 | let mut expectedScanner = parser.scanner; |
|
| 3271 | 3271 | let expectedCurrent = scanner::next(&mut expectedScanner, parser.pool); |
|
| 3272 | 3272 | let expectedPrevious = parser.current; |
|
| 3273 | 3273 | let expectedContext = parser.context; |
|
| 3274 | - | let startOffset = alloc::used(&arena.arena); |
|
| 3275 | - | let startId = arena.nextId; |
|
| 3274 | + | let startOffset = alloc::used(&parser.arena.arena); |
|
| 3275 | + | let startId = parser.arena.nextId; |
|
| 3276 | 3276 | let statement = try! super::parseStmt(&mut parser); |
|
| 3277 | 3277 | let case ast::NodeValue::Return { value } = statement.value else throw testing::TestError::Failed; |
|
| 3278 | 3278 | try testing::expect(value == nil); |
|
| 3279 | 3279 | try testing::expect(statement.id == startId); |
|
| 3280 | - | try testing::expect(alloc::used(&arena.arena) == startOffset + expectedBytes); |
|
| 3281 | - | try testing::expect(arena.nextId == startId + 1); |
|
| 3280 | + | try testing::expect(alloc::used(&parser.arena.arena) == startOffset + expectedBytes); |
|
| 3281 | + | try testing::expect(parser.arena.nextId == startId + 1); |
|
| 3282 | 3282 | try testing::expect(parser.scanner.cursor == expectedScanner.cursor); |
|
| 3283 | 3283 | try testing::expect(parser.scanner.token == expectedScanner.token); |
|
| 3284 | 3284 | try testing::expect(parser.current.kind == expectedCurrent.kind); |
|
| 3285 | 3285 | try testing::expect(parser.current.offset == expectedCurrent.offset); |
|
| 3286 | 3286 | try testing::expect(parser.previous.kind == expectedPrevious.kind); |
|
| 3287 | 3287 | try testing::expect(parser.previous.offset == expectedPrevious.offset); |
|
| 3288 | 3288 | try testing::expect(parser.context == expectedContext); |
|
| 3289 | 3289 | try testing::expect(parser.errors.count == 1); |
|
| 3290 | - | for i in alloc::used(&arena.arena)..ARENA_STORAGE.len { |
|
| 3290 | + | for i in alloc::used(&parser.arena.arena)..ARENA_STORAGE.len { |
|
| 3291 | 3291 | set ARENA_STORAGE[i] = 0xA5; |
|
| 3292 | 3292 | } |
|
| 3293 | 3293 | let case ast::NodeValue::Bool(true) = retained.value else throw testing::TestError::Failed; |
|
| 3294 | 3294 | try testing::expect(retained.span.offset == 3); |
|
| 3295 | 3295 | try testing::expect(retained.span.length == 1); |
|
| 3296 | 3296 | try testing::expect(mem::eq(parser.errors.list[0].message, "retained diagnostic")); |
|
| 3297 | 3297 | try testing::expect(mem::eq(parser.errors.list[0].token.source, "return")); |
|
| 3298 | 3298 | let interned = strings::find(parser.pool, "speculativeRestoreIdentifier") else throw testing::TestError::Failed; |
|
| 3299 | 3299 | try testing::expect(mem::eq(interned, "speculativeRestoreIdentifier")); |
|
| 3300 | - | let replacement = ast::allocNode(&mut arena, ast::Span { offset: 0, length: 0 }, ast::NodeValue::Bool(false)); |
|
| 3300 | + | let replacement = ast::allocNode(parser.arena, ast::Span { offset: 0, length: 0 }, ast::NodeValue::Bool(false)); |
|
| 3301 | 3301 | try testing::expect(replacement.id == startId + 1); |
|
| 3302 | 3302 | let case ast::NodeValue::Bool(true) = retained.value else throw testing::TestError::Failed; |
|
| 3303 | 3303 | } |
|
| 3304 | 3304 | } |
|
| 3305 | 3305 |
| 3308 | 3308 | let expectedBytes = statementStorageUsed("return"); |
|
| 3309 | 3309 | for source in ["return (speculativeReturnIdentifier +", "panic (speculativePanicIdentifier +"] { |
|
| 3310 | 3310 | let mut arena = ast::nodeArena(&mut ARENA_STORAGE[..]); |
|
| 3311 | 3311 | ast::allocNode(&mut arena, ast::Span { offset: 0, length: 0 }, ast::NodeValue::Bool(true)); |
|
| 3312 | 3312 | let startOffset = alloc::used(&arena.arena); |
|
| 3313 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 3314 | - | let mut parser = super::mkParser(scanner::SourceLoc::String, source, &mut arena, poolRef); |
|
| 3313 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 3314 | + | let mut parser = super::mkParser(scanner::SourceLoc::String, source, arenaRef, poolRef); |
|
| 3315 | 3315 | super::advance(&mut parser); |
|
| 3316 | - | let startId = arena.nextId; |
|
| 3316 | + | let startId = parser.arena.nextId; |
|
| 3317 | 3317 | let statement = try! super::parseStmt(&mut parser); |
|
| 3318 | 3318 | try testing::expect(statement.id == startId); |
|
| 3319 | - | try testing::expect(arena.nextId == startId + 1); |
|
| 3320 | - | try testing::expect(alloc::used(&arena.arena) == startOffset + expectedBytes); |
|
| 3319 | + | try testing::expect(parser.arena.nextId == startId + 1); |
|
| 3320 | + | try testing::expect(alloc::used(&parser.arena.arena) == startOffset + expectedBytes); |
|
| 3321 | 3321 | try testing::expect(parser.errors.count == 0); |
|
| 3322 | 3322 | try testing::expect(parser.current.kind == scanner::TokenKind::LParen); |
|
| 3323 | 3323 | match statement.value { |
|
| 3324 | 3324 | case ast::NodeValue::Return { value } => try testing::expect(value == nil), |
|
| 3325 | 3325 | case ast::NodeValue::Panic { message } => try testing::expect(message == nil), |
lib/std/lang/resolver/tests.rad
+3 -3
| 105 | 105 | } |
|
| 106 | 106 | ||
| 107 | 107 | /// Parse and analyze an expression string for testing. |
|
| 108 | 108 | unsafe fn resolveExprStr 'arena (self: &mut super::Resolver 'arena, stmt: *[u8]) -> TestResult throws (testing::TestError) { |
|
| 109 | 109 | let mut arena = ast::nodeArena(&mut AST_ARENA[..]); |
|
| 110 | - | let poolRef: 'pool = &mut STRING_POOL in { |
|
| 111 | - | let mut p = parser::mkParser(scanner::SourceLoc::String, stmt, &mut arena, poolRef); |
|
| 110 | + | let poolRef: 'pool = &mut STRING_POOL, arenaRef = &mut arena in { |
|
| 111 | + | let mut p = parser::mkParser(scanner::SourceLoc::String, stmt, arenaRef, poolRef); |
|
| 112 | 112 | parser::advance(&mut p); |
|
| 113 | 113 | ||
| 114 | 114 | let expr = try parser::parseExpr(&mut p) catch { |
|
| 115 | 115 | panic "resolveExprStr: parsing failed"; |
|
| 116 | 116 | }; |
|
| 117 | - | let diagnostics = try super::resolveExpr(self, expr, &mut arena) catch { |
|
| 117 | + | let diagnostics = try super::resolveExpr(self, expr, p.arena) catch { |
|
| 118 | 118 | throw testing::TestError::Failed; |
|
| 119 | 119 | }; |
|
| 120 | 120 | return TestResult { diagnostics, root: expr }; |
|
| 121 | 121 | } |
|
| 122 | 122 | } |