compiler: Scan tokens through checked string pool borrows
2c67cf477eaa49a130b9e43e27f375f15be2415de9feef02cc62e6265e837ee4
1 parent
6d74348f
lib/std/lang/parser.rad
+7 -4
| 147 | 147 | ||
| 148 | 148 | /// Parser state. |
|
| 149 | 149 | export record Parser: Copy { |
|
| 150 | 150 | /// The scanner that provides tokens. |
|
| 151 | 151 | scanner: scanner::Scanner, |
|
| 152 | + | /// Interned string pool. It must remain valid while parsing. |
|
| 153 | + | pool: *unsafe mut strings::Pool, |
|
| 152 | 154 | /// The current token being examined. |
|
| 153 | 155 | current: scanner::Token, |
|
| 154 | 156 | /// The most recently consumed token. |
|
| 155 | 157 | previous: scanner::Token, |
|
| 156 | 158 | /// Collection of errors encountered during parsing. |
| 166 | 168 | /// Create a new parser initialized with the given source kind, source and node arena. |
|
| 167 | 169 | /// The node arena and string pool must outlive every copy of the parser. |
|
| 168 | 170 | export unsafe fn mkParser(sourceLoc: scanner::SourceLoc, source: *[u8], arena: &mut ast::NodeArena, pool: *unsafe mut strings::Pool) -> Parser { |
|
| 169 | 171 | return Parser { |
|
| 170 | 172 | scanner: scanner::scanner(sourceLoc, source, pool), |
|
| 173 | + | pool, |
|
| 171 | 174 | current: scanner::invalid(0, ""), |
|
| 172 | 175 | previous: scanner::invalid(0, ""), |
|
| 173 | 176 | errors: ErrorList { list: undefined, count: 0 }, |
|
| 174 | 177 | arena: (&mut *arena) as *unsafe mut ast::NodeArena, |
|
| 175 | 178 | allocator: alloc::arenaAllocator(&mut arena.arena), |
| 985 | 988 | } |
|
| 986 | 989 | ||
| 987 | 990 | /// Return whether the current `let` statement starts a regional block. |
|
| 988 | 991 | unsafe fn isRegionBlock(p: &Parser) -> bool { |
|
| 989 | 992 | let mut lookahead = p.scanner; |
|
| 990 | - | return scanner::next(&mut lookahead).kind == scanner::TokenKind::Ident |
|
| 991 | - | and scanner::next(&mut lookahead).kind == scanner::TokenKind::Colon |
|
| 992 | - | and scanner::next(&mut lookahead).kind == scanner::TokenKind::Region; |
|
| 993 | + | return scanner::next(&mut lookahead, p.pool).kind == scanner::TokenKind::Ident |
|
| 994 | + | and scanner::next(&mut lookahead, p.pool).kind == scanner::TokenKind::Colon |
|
| 995 | + | and scanner::next(&mut lookahead, p.pool).kind == scanner::TokenKind::Region; |
|
| 993 | 996 | } |
|
| 994 | 997 | ||
| 995 | 998 | /// Return whether the current `use` statement has an allocation-session header. |
|
| 996 | 999 | unsafe fn isSessionBlock(p: &mut Parser) -> bool { |
|
| 997 | 1000 | let saved = saveState(p); |
| 1173 | 1176 | } |
|
| 1174 | 1177 | ||
| 1175 | 1178 | /// Advance the parser by one token. |
|
| 1176 | 1179 | export unsafe fn advance(p: &mut Parser) { |
|
| 1177 | 1180 | set p.previous = p.current; |
|
| 1178 | - | set p.current = scanner::next(&mut p.scanner); |
|
| 1181 | + | set p.current = scanner::next(&mut p.scanner, p.pool); |
|
| 1179 | 1182 | } |
|
| 1180 | 1183 | ||
| 1181 | 1184 | /// Parse an `if let` pattern matching statement. |
|
| 1182 | 1185 | /// |
|
| 1183 | 1186 | /// Syntax: `if let binding = scrutinee { ... }` |
lib/std/lang/scanner.rad
+9 -11
| 193 | 193 | source: *[u8], |
|
| 194 | 194 | /// Offset of current token into buffer. |
|
| 195 | 195 | token: u32, |
|
| 196 | 196 | /// Offset of current character being scanned. |
|
| 197 | 197 | cursor: u32, |
|
| 198 | - | /// Interned string pool. It must remain valid while scanning. |
|
| 199 | - | pool: *unsafe mut strings::Pool, |
|
| 200 | 198 | } |
|
| 201 | 199 | ||
| 202 | 200 | /// Individual token with kind, source text, and position. |
|
| 203 | 201 | /// |
|
| 204 | 202 | /// Represents a single lexical element extracted from source, |
| 223 | 221 | /// Column number. |
|
| 224 | 222 | col: u16, |
|
| 225 | 223 | } |
|
| 226 | 224 | ||
| 227 | 225 | /// Create a new scanner object. |
|
| 228 | - | export unsafe fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: *unsafe mut strings::Pool) -> Scanner { |
|
| 226 | + | export fn scanner(sourceLoc: SourceLoc, source: *[u8], pool: &mut strings::Pool) -> Scanner { |
|
| 229 | 227 | // Intern built-in functions and attributes. |
|
| 230 | 228 | strings::intern(pool, "@sizeOf"); |
|
| 231 | 229 | strings::intern(pool, "@alignOf"); |
|
| 232 | 230 | strings::intern(pool, "@sliceOf"); |
|
| 233 | 231 | strings::intern(pool, "@default"); |
| 235 | 233 | strings::intern(pool, "@test"); |
|
| 236 | 234 | // Intern built-in slice methods. |
|
| 237 | 235 | strings::intern(pool, "append"); |
|
| 238 | 236 | strings::intern(pool, "delete"); |
|
| 239 | 237 | ||
| 240 | - | return Scanner { sourceLoc, source, token: 0, cursor: 0, pool }; |
|
| 238 | + | return Scanner { sourceLoc, source, token: 0, cursor: 0 }; |
|
| 241 | 239 | } |
|
| 242 | 240 | ||
| 243 | 241 | /// Check if we've reached the end of input. |
|
| 244 | 242 | export fn isEof(s: &Scanner) -> bool { |
|
| 245 | 243 | return s.cursor >= s.source.len; |
| 411 | 409 | } |
|
| 412 | 410 | return TokenKind::Ident; |
|
| 413 | 411 | } |
|
| 414 | 412 | ||
| 415 | 413 | /// Scan an identifier, keyword, or label. |
|
| 416 | - | unsafe fn scanIdentifier(s: &mut Scanner) -> Token { |
|
| 414 | + | fn scanIdentifier(s: &mut Scanner, pool: &mut strings::Pool) -> Token { |
|
| 417 | 415 | while let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) { |
|
| 418 | 416 | advance(s); |
|
| 419 | 417 | } |
|
| 420 | 418 | let ident = &s.source[s.token..s.cursor]; |
|
| 421 | 419 | let kind = keywordOrIdent(ident); |
|
| 422 | 420 | ||
| 423 | 421 | // Only intern actual identifiers, not keywords. |
|
| 424 | 422 | if kind == TokenKind::Ident { |
|
| 425 | - | return Token { kind, source: strings::intern(s.pool, ident), offset: s.token }; |
|
| 423 | + | return Token { kind, source: strings::intern(pool, ident), offset: s.token }; |
|
| 426 | 424 | } |
|
| 427 | 425 | return tok(s, kind); |
|
| 428 | 426 | } |
|
| 429 | 427 | ||
| 430 | - | /// Scan the next token. The retained string pool must be valid and writable. |
|
| 431 | - | export unsafe fn next(s: &mut Scanner) -> Token { |
|
| 428 | + | /// Scan the next token and intern identifiers in the supplied pool. |
|
| 429 | + | export fn next(s: &mut Scanner, pool: &mut strings::Pool) -> Token { |
|
| 432 | 430 | skipWhitespace(s); // Skip any whitespace between tokens. |
|
| 433 | 431 | set s.token = s.cursor; // Token starts at current position. |
|
| 434 | 432 | ||
| 435 | 433 | if isEof(s) { |
|
| 436 | 434 | return tok(s, TokenKind::Eof); |
| 439 | 437 | ||
| 440 | 438 | if char::isDigit(c) { |
|
| 441 | 439 | return scanNumber(s); |
|
| 442 | 440 | } |
|
| 443 | 441 | if char::isAlpha(c) { |
|
| 444 | - | return scanIdentifier(s); |
|
| 442 | + | return scanIdentifier(s, pool); |
|
| 445 | 443 | } |
|
| 446 | 444 | match c { |
|
| 447 | 445 | case '\'' => return scanChar(s), |
|
| 448 | 446 | case '"' => return scanString(s), |
|
| 449 | 447 | case '(' => return tok(s, TokenKind::LParen), |
| 568 | 566 | advance(s); |
|
| 569 | 567 | } |
|
| 570 | 568 | let name = &s.source[s.token..s.cursor]; |
|
| 571 | 569 | return Token { |
|
| 572 | 570 | kind: TokenKind::AtIdent, |
|
| 573 | - | source: strings::intern(s.pool, name), |
|
| 571 | + | source: strings::intern(pool, name), |
|
| 574 | 572 | offset: s.token, |
|
| 575 | 573 | }; |
|
| 576 | 574 | } |
|
| 577 | 575 | case '_' => { |
|
| 578 | 576 | if let ch = current(s); char::isAlpha(ch) or ch == '_' or char::isDigit(ch) { |
|
| 579 | 577 | // This is part of an identifier like `_foo` or `__start` |
|
| 580 | - | return scanIdentifier(s); |
|
| 578 | + | return scanIdentifier(s, pool); |
|
| 581 | 579 | } |
|
| 582 | 580 | return tok(s, TokenKind::Underscore); |
|
| 583 | 581 | } |
|
| 584 | 582 | else => return invalid(s.token, "unexpected character"), |
|
| 585 | 583 | } |
lib/std/lang/scanner/tests.rad
+135 -137
| 1 | 1 | use std::lang::strings; |
|
| 2 | 2 | use std::mem; |
|
| 3 | 3 | use std::testing; |
|
| 4 | 4 | ||
| 5 | 5 | /// String pool for testing. |
|
| 6 | - | unsafe static TEST_STRING_POOL: strings::Pool = strings::Pool { table: undefined, count: 0 }; |
|
| 6 | + | static TEST_STRING_POOL: strings::Pool = strings::Pool { table: [""; 32768], count: 0 }; |
|
| 7 | 7 | ||
| 8 | 8 | /// Create a scanner backed by the test string pool. |
|
| 9 | 9 | fn testScanner(source: *[u8]) -> super::Scanner { |
|
| 10 | - | unsafe { |
|
| 11 | - | return super::scanner(super::SourceLoc::File("test.r"), source, &mut TEST_STRING_POOL); |
|
| 12 | - | } |
|
| 10 | + | return super::scanner(super::SourceLoc::File("test.r"), source, &mut TEST_STRING_POOL); |
|
| 13 | 11 | } |
|
| 14 | 12 | ||
| 15 | - | @test unsafe fn testScanTokens() throws (testing::TestError) { |
|
| 13 | + | @test fn testScanTokens() throws (testing::TestError) { |
|
| 16 | 14 | let mut s = testScanner( |
|
| 17 | 15 | "'x' < fnord fnord: 0 => >> mod and nil" |
|
| 18 | 16 | ); |
|
| 19 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Char); |
|
| 20 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Lt); |
|
| 21 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 22 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 23 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Colon); |
|
| 24 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Number); |
|
| 25 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::FatArrow); |
|
| 26 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::GtGt); |
|
| 27 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Mod); |
|
| 28 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::And); |
|
| 29 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Nil); |
|
| 30 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 17 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Char); |
|
| 18 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Lt); |
|
| 19 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 20 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 21 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Colon); |
|
| 22 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Number); |
|
| 23 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::FatArrow); |
|
| 24 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::GtGt); |
|
| 25 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Mod); |
|
| 26 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::And); |
|
| 27 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Nil); |
|
| 28 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 31 | 29 | } |
|
| 32 | 30 | ||
| 33 | - | @test unsafe fn testScanChars() throws (testing::TestError) { |
|
| 31 | + | @test fn testScanChars() throws (testing::TestError) { |
|
| 34 | 32 | let mut s = testScanner( |
|
| 35 | 33 | "'\\0'" |
|
| 36 | 34 | ); |
|
| 37 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Char); |
|
| 38 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 35 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Char); |
|
| 36 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 39 | 37 | } |
|
| 40 | 38 | ||
| 41 | - | @test unsafe fn testScanWhitespace() throws (testing::TestError) { |
|
| 39 | + | @test fn testScanWhitespace() throws (testing::TestError) { |
|
| 42 | 40 | let mut s = testScanner( |
|
| 43 | 41 | " X\n\nY //\n Z//1 \n" |
|
| 44 | 42 | ); |
|
| 45 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 46 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 47 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 48 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 43 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 44 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 45 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 46 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 49 | 47 | } |
|
| 50 | 48 | ||
| 51 | - | @test unsafe fn testScanIdentsAndKeywords() throws (testing::TestError) { |
|
| 49 | + | @test fn testScanIdentsAndKeywords() throws (testing::TestError) { |
|
| 52 | 50 | let mut s = testScanner( |
|
| 53 | 51 | "m mo mod modo" |
|
| 54 | 52 | ); |
|
| 55 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 56 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 57 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Mod); |
|
| 58 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 59 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 53 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 54 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 55 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Mod); |
|
| 56 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 57 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 60 | 58 | } |
|
| 61 | 59 | ||
| 62 | - | @test unsafe fn testScanIdentifiers() throws (testing::TestError) { |
|
| 60 | + | @test fn testScanIdentifiers() throws (testing::TestError) { |
|
| 63 | 61 | let mut s = testScanner( |
|
| 64 | 62 | "fnord::yikes no1 4j" |
|
| 65 | 63 | ); |
|
| 66 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 67 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::ColonColon); |
|
| 68 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 69 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 70 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Number); |
|
| 71 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 72 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 64 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 65 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::ColonColon); |
|
| 66 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 67 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 68 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Number); |
|
| 69 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 70 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 73 | 71 | } |
|
| 74 | 72 | ||
| 75 | - | @test unsafe fn testScanAtIdentifiers() throws (testing::TestError) { |
|
| 73 | + | @test fn testScanAtIdentifiers() throws (testing::TestError) { |
|
| 76 | 74 | let mut s = testScanner( |
|
| 77 | 75 | "@default @" |
|
| 78 | 76 | ); |
|
| 79 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::AtIdent); |
|
| 80 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Invalid); |
|
| 77 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::AtIdent); |
|
| 78 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Invalid); |
|
| 81 | 79 | } |
|
| 82 | 80 | ||
| 83 | - | @test unsafe fn testScanStrings() throws (testing::TestError) { |
|
| 81 | + | @test fn testScanStrings() throws (testing::TestError) { |
|
| 84 | 82 | let mut s = testScanner( |
|
| 85 | 83 | "\"Hello World!\" \"\\\"\"" |
|
| 86 | 84 | ); |
|
| 87 | - | let mut t: super::Token = super::next(&mut s); |
|
| 85 | + | let mut t: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 88 | 86 | try testing::expect(t.kind == super::TokenKind::String); |
|
| 89 | 87 | try testing::expect(t.source.len == 14); // Includes quotes. |
|
| 90 | 88 | try testing::expect(mem::eq(t.source, "\"Hello World!\"")); |
|
| 91 | 89 | ||
| 92 | - | set t = super::next(&mut s); |
|
| 90 | + | set t = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 93 | 91 | try testing::expect(t.kind == super::TokenKind::String); |
|
| 94 | 92 | try testing::expect(t.source.len == 4); |
|
| 95 | 93 | try testing::expect(mem::eq(t.source, "\"\\\"\"")); |
|
| 96 | 94 | } |
|
| 97 | 95 | ||
| 98 | - | @test unsafe fn testGetLocation() throws (testing::TestError) { |
|
| 96 | + | @test fn testGetLocation() throws (testing::TestError) { |
|
| 99 | 97 | let mut s = testScanner( |
|
| 100 | 98 | "abc\n def\n ghi" |
|
| 101 | 99 | ); |
|
| 102 | - | let mut tok: super::Token = super::next(&mut s); |
|
| 100 | + | let mut tok: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 103 | 101 | try testing::expect(tok.kind == super::TokenKind::Ident); |
|
| 104 | 102 | try testing::expect(tok.source.len == 3); |
|
| 105 | 103 | ||
| 106 | 104 | if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) { |
|
| 107 | 105 | try testing::expect(loc.line == 1); |
|
| 108 | 106 | try testing::expect(loc.col == 1); |
|
| 109 | 107 | } else { |
|
| 110 | 108 | try testing::expect(false); |
|
| 111 | 109 | } |
|
| 112 | - | set tok = super::next(&mut s); |
|
| 110 | + | set tok = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 113 | 111 | try testing::expect(tok.kind == super::TokenKind::Ident); |
|
| 114 | 112 | try testing::expect(tok.source.len == 3); |
|
| 115 | 113 | ||
| 116 | 114 | if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) { |
|
| 117 | 115 | try testing::expect(loc.line == 2); |
|
| 118 | 116 | try testing::expect(loc.col == 3); |
|
| 119 | 117 | } else { |
|
| 120 | 118 | try testing::expect(false); |
|
| 121 | 119 | } |
|
| 122 | - | set tok = super::next(&mut s); |
|
| 120 | + | set tok = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 123 | 121 | try testing::expect(tok.kind == super::TokenKind::Ident); |
|
| 124 | 122 | try testing::expect(tok.source.len == 3); |
|
| 125 | 123 | ||
| 126 | 124 | if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) { |
|
| 127 | 125 | try testing::expect(loc.line == 3); |
| 129 | 127 | } else { |
|
| 130 | 128 | try testing::expect(false); |
|
| 131 | 129 | } |
|
| 132 | 130 | } |
|
| 133 | 131 | ||
| 134 | - | @test unsafe fn testScanEmptyInput() throws (testing::TestError) { |
|
| 132 | + | @test fn testScanEmptyInput() throws (testing::TestError) { |
|
| 135 | 133 | let mut s = testScanner(""); |
|
| 136 | - | let tok: super::Token = super::next(&mut s); |
|
| 134 | + | let tok: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 137 | 135 | ||
| 138 | 136 | try testing::expect(tok.kind == super::TokenKind::Eof); |
|
| 139 | 137 | try testing::expect(tok.source.len == 0); |
|
| 140 | 138 | try testing::expect(tok.offset == 0); |
|
| 141 | 139 | } |
|
| 142 | 140 | ||
| 143 | - | @test unsafe fn testScanSingleCharTokens() throws (testing::TestError) { |
|
| 141 | + | @test fn testScanSingleCharTokens() throws (testing::TestError) { |
|
| 144 | 142 | let mut s = testScanner( |
|
| 145 | 143 | "(){}[] ,;+-*~|&" |
|
| 146 | 144 | ); |
|
| 147 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::LParen); |
|
| 148 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::RParen); |
|
| 149 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::LBrace); |
|
| 150 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::RBrace); |
|
| 151 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::LBracket); |
|
| 152 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::RBracket); |
|
| 153 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Comma); |
|
| 154 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Semicolon); |
|
| 155 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Plus); |
|
| 156 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Minus); |
|
| 157 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Star); |
|
| 158 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Tilde); |
|
| 159 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Pipe); |
|
| 160 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Amp); |
|
| 161 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 145 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::LParen); |
|
| 146 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::RParen); |
|
| 147 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::LBrace); |
|
| 148 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::RBrace); |
|
| 149 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::LBracket); |
|
| 150 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::RBracket); |
|
| 151 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Comma); |
|
| 152 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Semicolon); |
|
| 153 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Plus); |
|
| 154 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Minus); |
|
| 155 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Star); |
|
| 156 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Tilde); |
|
| 157 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Pipe); |
|
| 158 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Amp); |
|
| 159 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 162 | 160 | } |
|
| 163 | 161 | ||
| 164 | - | @test unsafe fn testScanDoubleCharTokens() throws (testing::TestError) { |
|
| 162 | + | @test fn testScanDoubleCharTokens() throws (testing::TestError) { |
|
| 165 | 163 | let mut s = testScanner( |
|
| 166 | 164 | "== <> <= >= < > >> << -> =>" |
|
| 167 | 165 | ); |
|
| 168 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::EqualEqual); |
|
| 169 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::LtGt); |
|
| 170 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::LtEqual); |
|
| 171 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::GtEqual); |
|
| 172 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Lt); |
|
| 173 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Gt); |
|
| 174 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::GtGt); |
|
| 175 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::LtLt); |
|
| 176 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Arrow); |
|
| 177 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::FatArrow); |
|
| 178 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 166 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::EqualEqual); |
|
| 167 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::LtGt); |
|
| 168 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::LtEqual); |
|
| 169 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::GtEqual); |
|
| 170 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Lt); |
|
| 171 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Gt); |
|
| 172 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::GtGt); |
|
| 173 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::LtLt); |
|
| 174 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Arrow); |
|
| 175 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::FatArrow); |
|
| 176 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 179 | 177 | } |
|
| 180 | 178 | ||
| 181 | - | @test unsafe fn testScanIdentifierLengths() throws (testing::TestError) { |
|
| 179 | + | @test fn testScanIdentifierLengths() throws (testing::TestError) { |
|
| 182 | 180 | let mut s = testScanner( |
|
| 183 | 181 | "sho xyz981 lal_lel" |
|
| 184 | 182 | ); |
|
| 185 | - | let mut tok: super::Token = super::next(&mut s); |
|
| 183 | + | let mut tok: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 186 | 184 | try testing::expect(tok.kind == super::TokenKind::Ident); |
|
| 187 | 185 | try testing::expect(tok.source.len == 3); |
|
| 188 | 186 | ||
| 189 | - | set tok = super::next(&mut s); |
|
| 187 | + | set tok = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 190 | 188 | try testing::expect(tok.kind == super::TokenKind::Ident); |
|
| 191 | 189 | try testing::expect(tok.source.len == 6); |
|
| 192 | 190 | ||
| 193 | - | set tok = super::next(&mut s); |
|
| 191 | + | set tok = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 194 | 192 | try testing::expect(tok.kind == super::TokenKind::Ident); |
|
| 195 | 193 | try testing::expect(tok.source.len == 7); |
|
| 196 | 194 | } |
|
| 197 | 195 | ||
| 198 | - | @test unsafe fn testScanNumbers() throws (testing::TestError) { |
|
| 196 | + | @test fn testScanNumbers() throws (testing::TestError) { |
|
| 199 | 197 | let mut s = testScanner( |
|
| 200 | 198 | "2 -2 +2 value-2 value+2 value--2 value+-2 value-+2 f(-2)" |
|
| 201 | 199 | ); |
|
| 202 | 200 | let expected: [super::TokenKind; 29] = [ |
|
| 203 | 201 | super::TokenKind::Number, |
| 214 | 212 | super::TokenKind::Ident, super::TokenKind::LParen, super::TokenKind::Minus, |
|
| 215 | 213 | super::TokenKind::Number, super::TokenKind::RParen, |
|
| 216 | 214 | super::TokenKind::Eof, |
|
| 217 | 215 | ]; |
|
| 218 | 216 | for expectedKind in expected { |
|
| 219 | - | try testing::expect(super::next(&mut s).kind == expectedKind); |
|
| 217 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == expectedKind); |
|
| 220 | 218 | } |
|
| 221 | 219 | } |
|
| 222 | 220 | ||
| 223 | - | @test unsafe fn testScanKeywords() throws (testing::TestError) { |
|
| 221 | + | @test fn testScanKeywords() throws (testing::TestError) { |
|
| 224 | 222 | let mut s = testScanner("nil mod not static unsafe where"); |
|
| 225 | - | let tok1: super::Token = super::next(&mut s); |
|
| 223 | + | let tok1: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 226 | 224 | ||
| 227 | 225 | try testing::expect(tok1.kind == super::TokenKind::Nil); |
|
| 228 | 226 | try testing::expect(tok1.source.len == 3); |
|
| 229 | 227 | ||
| 230 | - | let tok2: super::Token = super::next(&mut s); |
|
| 228 | + | let tok2: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 231 | 229 | try testing::expect(tok2.kind == super::TokenKind::Mod); |
|
| 232 | 230 | try testing::expect(tok2.source.len == 3); |
|
| 233 | 231 | ||
| 234 | - | let tok3: super::Token = super::next(&mut s); |
|
| 232 | + | let tok3: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 235 | 233 | try testing::expect(tok3.kind == super::TokenKind::Not); |
|
| 236 | 234 | try testing::expect(tok3.source.len == 3); |
|
| 237 | 235 | ||
| 238 | - | let tok4: super::Token = super::next(&mut s); |
|
| 236 | + | let tok4: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 239 | 237 | try testing::expect(tok4.kind == super::TokenKind::Static); |
|
| 240 | 238 | try testing::expect(tok4.source.len == 6); |
|
| 241 | 239 | ||
| 242 | - | let tok5: super::Token = super::next(&mut s); |
|
| 240 | + | let tok5: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 243 | 241 | try testing::expect(tok5.kind == super::TokenKind::Unsafe); |
|
| 244 | 242 | try testing::expect(tok5.source.len == 6); |
|
| 245 | 243 | ||
| 246 | - | let tok6: super::Token = super::next(&mut s); |
|
| 244 | + | let tok6: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 247 | 245 | try testing::expect(tok6.kind == super::TokenKind::Where); |
|
| 248 | 246 | try testing::expect(tok6.source.len == 5); |
|
| 249 | 247 | } |
|
| 250 | 248 | ||
| 251 | - | @test unsafe fn testScanVoidAsIdent() throws (testing::TestError) { |
|
| 249 | + | @test fn testScanVoidAsIdent() throws (testing::TestError) { |
|
| 252 | 250 | let mut s = testScanner("void"); |
|
| 253 | - | let tok: super::Token = super::next(&mut s); |
|
| 251 | + | let tok: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 254 | 252 | try testing::expect(tok.kind == super::TokenKind::Ident); |
|
| 255 | 253 | try testing::expect(tok.source.len == 4); |
|
| 256 | 254 | } |
|
| 257 | 255 | ||
| 258 | - | @test unsafe fn testScanWhitespaceAndComments() throws (testing::TestError) { |
|
| 256 | + | @test fn testScanWhitespaceAndComments() throws (testing::TestError) { |
|
| 259 | 257 | let mut s = testScanner( |
|
| 260 | 258 | " \t\n // This is a comment..\n 42" |
|
| 261 | 259 | ); |
|
| 262 | - | let tok: super::Token = super::next(&mut s); |
|
| 260 | + | let tok: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 263 | 261 | try testing::expect(tok.kind == super::TokenKind::Number); |
|
| 264 | 262 | try testing::expect(tok.source.len == 2); |
|
| 265 | 263 | try testing::expect(super::isEof(&s)); |
|
| 266 | 264 | } |
|
| 267 | 265 | ||
| 268 | - | @test unsafe fn testScanInvalidCharacters() throws (testing::TestError) { |
|
| 266 | + | @test fn testScanInvalidCharacters() throws (testing::TestError) { |
|
| 269 | 267 | let mut s = testScanner( |
|
| 270 | 268 | "`#$" |
|
| 271 | 269 | ); |
|
| 272 | - | let mut tok: super::Token = super::next(&mut s); |
|
| 270 | + | let mut tok: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 273 | 271 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
|
| 274 | 272 | try testing::expect(tok.offset == 0); |
|
| 275 | 273 | ||
| 276 | - | set tok = super::next(&mut s); |
|
| 274 | + | set tok = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 277 | 275 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
|
| 278 | 276 | try testing::expect(tok.offset == 1); |
|
| 279 | 277 | ||
| 280 | - | set tok = super::next(&mut s); |
|
| 278 | + | set tok = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 281 | 279 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
|
| 282 | 280 | try testing::expect(tok.offset == 2); |
|
| 283 | 281 | } |
|
| 284 | 282 | ||
| 285 | - | @test unsafe fn testScanUnterminatedString() throws (testing::TestError) { |
|
| 283 | + | @test fn testScanUnterminatedString() throws (testing::TestError) { |
|
| 286 | 284 | let mut s = testScanner("\"Hello World!"); |
|
| 287 | - | let tok: super::Token = super::next(&mut s); |
|
| 285 | + | let tok: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 288 | 286 | ||
| 289 | 287 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
|
| 290 | 288 | try testing::expect(tok.source == "unterminated string"); |
|
| 291 | 289 | try testing::expect(tok.offset == 0); |
|
| 292 | 290 | } |
|
| 293 | 291 | ||
| 294 | - | @test unsafe fn testScanUnderminatedStringEscape() throws (testing::TestError) { |
|
| 292 | + | @test fn testScanUnderminatedStringEscape() throws (testing::TestError) { |
|
| 295 | 293 | let mut s = testScanner("\"\\"); |
|
| 296 | - | let tok: super::Token = super::next(&mut s); |
|
| 294 | + | let tok: super::Token = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 297 | 295 | ||
| 298 | 296 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
|
| 299 | 297 | } |
|
| 300 | 298 | ||
| 301 | - | @test unsafe fn testScanFunctionDefinition() throws (testing::TestError) { |
|
| 299 | + | @test fn testScanFunctionDefinition() throws (testing::TestError) { |
|
| 302 | 300 | let mut s = testScanner( |
|
| 303 | 301 | "fn add(a: i32, b: i32) -> i32 { return a + b; }" |
|
| 304 | 302 | ); |
|
| 305 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Fn); |
|
| 306 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 307 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::LParen); |
|
| 308 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 309 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Colon); |
|
| 310 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::I32); |
|
| 311 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Comma); |
|
| 312 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 313 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Colon); |
|
| 314 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::I32); |
|
| 315 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::RParen); |
|
| 316 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Arrow); |
|
| 317 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::I32); |
|
| 318 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::LBrace); |
|
| 319 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Return); |
|
| 320 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 321 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Plus); |
|
| 322 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
|
| 323 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Semicolon); |
|
| 324 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::RBrace); |
|
| 325 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 303 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Fn); |
|
| 304 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 305 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::LParen); |
|
| 306 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 307 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Colon); |
|
| 308 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::I32); |
|
| 309 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Comma); |
|
| 310 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 311 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Colon); |
|
| 312 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::I32); |
|
| 313 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::RParen); |
|
| 314 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Arrow); |
|
| 315 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::I32); |
|
| 316 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::LBrace); |
|
| 317 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Return); |
|
| 318 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 319 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Plus); |
|
| 320 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident); |
|
| 321 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Semicolon); |
|
| 322 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::RBrace); |
|
| 323 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 326 | 324 | } |
|
| 327 | 325 | ||
| 328 | 326 | /// Region tokens preserve their names and do not consume character literals. |
|
| 329 | - | @test unsafe fn testScanRegions() throws (testing::TestError) { |
|
| 327 | + | @test fn testScanRegions() throws (testing::TestError) { |
|
| 330 | 328 | let mut s = testScanner("'x 'long_2 'x' '\\n' 'outer<'inner"); |
|
| 331 | - | let first = super::next(&mut s); |
|
| 329 | + | let first = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 332 | 330 | try testing::expect(first.kind == super::TokenKind::Region); |
|
| 333 | 331 | try testing::expect(mem::eq(first.source, "'x")); |
|
| 334 | - | let second = super::next(&mut s); |
|
| 332 | + | let second = super::next(&mut s, &mut TEST_STRING_POOL); |
|
| 335 | 333 | try testing::expect(second.kind == super::TokenKind::Region); |
|
| 336 | 334 | try testing::expect(mem::eq(second.source, "'long_2")); |
|
| 337 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Char); |
|
| 338 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Char); |
|
| 339 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Region); |
|
| 340 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Lt); |
|
| 341 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Region); |
|
| 342 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
|
| 335 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Char); |
|
| 336 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Char); |
|
| 337 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Region); |
|
| 338 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Lt); |
|
| 339 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Region); |
|
| 340 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof); |
|
| 343 | 341 | } |
|
| 344 | 342 | ||
| 345 | 343 | /// An apostrophe without a name or a closing quote is invalid. |
|
| 346 | - | @test unsafe fn testScanMalformedRegions() throws (testing::TestError) { |
|
| 344 | + | @test fn testScanMalformedRegions() throws (testing::TestError) { |
|
| 347 | 345 | for text in ["'", "'1", "'\\", "'\n"] { |
|
| 348 | 346 | let mut s = testScanner(text); |
|
| 349 | - | try testing::expect(super::next(&mut s).kind == super::TokenKind::Invalid); |
|
| 347 | + | try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Invalid); |
|
| 350 | 348 | } |
|
| 351 | 349 | } |