compiler/
kernel/
lib/
examples/
std/
arch/
char/
collections/
lang/
alloc/
ast/
gen/
il/
module/
parser/
resolver/
scanner/
tests.rad
15.5 KiB
alloc.rad
7.1 KiB
ast.rad
26.7 KiB
gen.rad
513 B
il.rad
19.5 KiB
lower.rad
308.1 KiB
module.rad
14.9 KiB
package.rad
1.3 KiB
parser.rad
89.5 KiB
resolver.rad
439.6 KiB
scanner.rad
18.0 KiB
sexpr.rad
6.4 KiB
strings.rad
2.2 KiB
types.rad
1.6 KiB
sys/
arch.rad
68 B
char.rad
855 B
collections.rad
39 B
fmt.rad
8.3 KiB
intrinsics.rad
467 B
io.rad
1.7 KiB
lang.rad
276 B
mem.rad
2.3 KiB
sys.rad
179 B
testing.rad
2.4 KiB
tests.rad
15.7 KiB
vec.rad
3.2 KiB
std.rad
281 B
scripts/
seed/
sublime/
test/
vim/
.gitignore
336 B
.gitsigners
112 B
CONTRIBUTING
2.1 KiB
LICENSE
1.1 KiB
Makefile
9.2 KiB
README
2.5 KiB
STYLE
2.5 KiB
std.lib
1.5 KiB
std.lib.test
662 B
lib/std/lang/scanner/tests.rad
raw
| 1 | use std::lang::strings; |
| 2 | use std::mem; |
| 3 | use std::testing; |
| 4 | |
| 5 | /// String pool for testing. |
| 6 | unsafe static TEST_STRING_POOL: strings::Pool = strings::Pool { table: undefined, count: 0 }; |
| 7 | |
| 8 | /// Create a scanner backed by the test string pool. |
| 9 | fn testScanner(source: *[u8]) -> super::Scanner { |
| 10 | unsafe { |
| 11 | return super::scanner(super::SourceLoc::File("test.r"), source, &mut TEST_STRING_POOL); |
| 12 | } |
| 13 | } |
| 14 | |
| 15 | @test unsafe fn testScanTokens() throws (testing::TestError) { |
| 16 | let mut s = testScanner( |
| 17 | "'x' < fnord fnord: 0 => >> mod and nil" |
| 18 | ); |
| 19 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Char); |
| 20 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Lt); |
| 21 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 22 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 23 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Colon); |
| 24 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Number); |
| 25 | try testing::expect(super::next(&mut s).kind == super::TokenKind::FatArrow); |
| 26 | try testing::expect(super::next(&mut s).kind == super::TokenKind::GtGt); |
| 27 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Mod); |
| 28 | try testing::expect(super::next(&mut s).kind == super::TokenKind::And); |
| 29 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Nil); |
| 30 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 31 | } |
| 32 | |
| 33 | @test unsafe fn testScanChars() throws (testing::TestError) { |
| 34 | let mut s = testScanner( |
| 35 | "'\\0'" |
| 36 | ); |
| 37 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Char); |
| 38 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 39 | } |
| 40 | |
| 41 | @test unsafe fn testScanWhitespace() throws (testing::TestError) { |
| 42 | let mut s = testScanner( |
| 43 | " X\n\nY //\n Z//1 \n" |
| 44 | ); |
| 45 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 46 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 47 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 48 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 49 | } |
| 50 | |
| 51 | @test unsafe fn testScanIdentsAndKeywords() throws (testing::TestError) { |
| 52 | let mut s = testScanner( |
| 53 | "m mo mod modo" |
| 54 | ); |
| 55 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 56 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 57 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Mod); |
| 58 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 59 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 60 | } |
| 61 | |
| 62 | @test unsafe fn testScanIdentifiers() throws (testing::TestError) { |
| 63 | let mut s = testScanner( |
| 64 | "fnord::yikes no1 4j" |
| 65 | ); |
| 66 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 67 | try testing::expect(super::next(&mut s).kind == super::TokenKind::ColonColon); |
| 68 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 69 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 70 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Number); |
| 71 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 72 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 73 | } |
| 74 | |
| 75 | @test unsafe fn testScanAtIdentifiers() throws (testing::TestError) { |
| 76 | let mut s = testScanner( |
| 77 | "@default @" |
| 78 | ); |
| 79 | try testing::expect(super::next(&mut s).kind == super::TokenKind::AtIdent); |
| 80 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Invalid); |
| 81 | } |
| 82 | |
| 83 | @test unsafe fn testScanStrings() throws (testing::TestError) { |
| 84 | let mut s = testScanner( |
| 85 | "\"Hello World!\" \"\\\"\"" |
| 86 | ); |
| 87 | let mut t: super::Token = super::next(&mut s); |
| 88 | try testing::expect(t.kind == super::TokenKind::String); |
| 89 | try testing::expect(t.source.len == 14); // Includes quotes. |
| 90 | try testing::expect(mem::eq(t.source, "\"Hello World!\"")); |
| 91 | |
| 92 | set t = super::next(&mut s); |
| 93 | try testing::expect(t.kind == super::TokenKind::String); |
| 94 | try testing::expect(t.source.len == 4); |
| 95 | try testing::expect(mem::eq(t.source, "\"\\\"\"")); |
| 96 | } |
| 97 | |
| 98 | @test unsafe fn testGetLocation() throws (testing::TestError) { |
| 99 | let mut s = testScanner( |
| 100 | "abc\n def\n ghi" |
| 101 | ); |
| 102 | let mut tok: super::Token = super::next(&mut s); |
| 103 | try testing::expect(tok.kind == super::TokenKind::Ident); |
| 104 | try testing::expect(tok.source.len == 3); |
| 105 | |
| 106 | if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) { |
| 107 | try testing::expect(loc.line == 1); |
| 108 | try testing::expect(loc.col == 1); |
| 109 | } else { |
| 110 | try testing::expect(false); |
| 111 | } |
| 112 | set tok = super::next(&mut s); |
| 113 | try testing::expect(tok.kind == super::TokenKind::Ident); |
| 114 | try testing::expect(tok.source.len == 3); |
| 115 | |
| 116 | if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) { |
| 117 | try testing::expect(loc.line == 2); |
| 118 | try testing::expect(loc.col == 3); |
| 119 | } else { |
| 120 | try testing::expect(false); |
| 121 | } |
| 122 | set tok = super::next(&mut s); |
| 123 | try testing::expect(tok.kind == super::TokenKind::Ident); |
| 124 | try testing::expect(tok.source.len == 3); |
| 125 | |
| 126 | if let loc = super::getLocation(s.sourceLoc, s.source, tok.offset) { |
| 127 | try testing::expect(loc.line == 3); |
| 128 | try testing::expect(loc.col == 5); |
| 129 | } else { |
| 130 | try testing::expect(false); |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | @test unsafe fn testScanEmptyInput() throws (testing::TestError) { |
| 135 | let mut s = testScanner(""); |
| 136 | let tok: super::Token = super::next(&mut s); |
| 137 | |
| 138 | try testing::expect(tok.kind == super::TokenKind::Eof); |
| 139 | try testing::expect(tok.source.len == 0); |
| 140 | try testing::expect(tok.offset == 0); |
| 141 | } |
| 142 | |
| 143 | @test unsafe fn testScanSingleCharTokens() throws (testing::TestError) { |
| 144 | let mut s = testScanner( |
| 145 | "(){}[] ,;+-*~|&" |
| 146 | ); |
| 147 | try testing::expect(super::next(&mut s).kind == super::TokenKind::LParen); |
| 148 | try testing::expect(super::next(&mut s).kind == super::TokenKind::RParen); |
| 149 | try testing::expect(super::next(&mut s).kind == super::TokenKind::LBrace); |
| 150 | try testing::expect(super::next(&mut s).kind == super::TokenKind::RBrace); |
| 151 | try testing::expect(super::next(&mut s).kind == super::TokenKind::LBracket); |
| 152 | try testing::expect(super::next(&mut s).kind == super::TokenKind::RBracket); |
| 153 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Comma); |
| 154 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Semicolon); |
| 155 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Plus); |
| 156 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Minus); |
| 157 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Star); |
| 158 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Tilde); |
| 159 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Pipe); |
| 160 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Amp); |
| 161 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 162 | } |
| 163 | |
| 164 | @test unsafe fn testScanDoubleCharTokens() throws (testing::TestError) { |
| 165 | let mut s = testScanner( |
| 166 | "== <> <= >= < > >> << -> =>" |
| 167 | ); |
| 168 | try testing::expect(super::next(&mut s).kind == super::TokenKind::EqualEqual); |
| 169 | try testing::expect(super::next(&mut s).kind == super::TokenKind::LtGt); |
| 170 | try testing::expect(super::next(&mut s).kind == super::TokenKind::LtEqual); |
| 171 | try testing::expect(super::next(&mut s).kind == super::TokenKind::GtEqual); |
| 172 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Lt); |
| 173 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Gt); |
| 174 | try testing::expect(super::next(&mut s).kind == super::TokenKind::GtGt); |
| 175 | try testing::expect(super::next(&mut s).kind == super::TokenKind::LtLt); |
| 176 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Arrow); |
| 177 | try testing::expect(super::next(&mut s).kind == super::TokenKind::FatArrow); |
| 178 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 179 | } |
| 180 | |
| 181 | @test unsafe fn testScanIdentifierLengths() throws (testing::TestError) { |
| 182 | let mut s = testScanner( |
| 183 | "sho xyz981 lal_lel" |
| 184 | ); |
| 185 | let mut tok: super::Token = super::next(&mut s); |
| 186 | try testing::expect(tok.kind == super::TokenKind::Ident); |
| 187 | try testing::expect(tok.source.len == 3); |
| 188 | |
| 189 | set tok = super::next(&mut s); |
| 190 | try testing::expect(tok.kind == super::TokenKind::Ident); |
| 191 | try testing::expect(tok.source.len == 6); |
| 192 | |
| 193 | set tok = super::next(&mut s); |
| 194 | try testing::expect(tok.kind == super::TokenKind::Ident); |
| 195 | try testing::expect(tok.source.len == 7); |
| 196 | } |
| 197 | |
| 198 | @test unsafe fn testScanNumbers() throws (testing::TestError) { |
| 199 | let mut s = testScanner( |
| 200 | "2 -2 +2 value-2 value+2 value--2 value+-2 value-+2 f(-2)" |
| 201 | ); |
| 202 | let expected: [super::TokenKind; 29] = [ |
| 203 | super::TokenKind::Number, |
| 204 | super::TokenKind::Minus, super::TokenKind::Number, |
| 205 | super::TokenKind::Plus, super::TokenKind::Number, |
| 206 | super::TokenKind::Ident, super::TokenKind::Minus, super::TokenKind::Number, |
| 207 | super::TokenKind::Ident, super::TokenKind::Plus, super::TokenKind::Number, |
| 208 | super::TokenKind::Ident, super::TokenKind::Minus, super::TokenKind::Minus, |
| 209 | super::TokenKind::Number, |
| 210 | super::TokenKind::Ident, super::TokenKind::Plus, super::TokenKind::Minus, |
| 211 | super::TokenKind::Number, |
| 212 | super::TokenKind::Ident, super::TokenKind::Minus, super::TokenKind::Plus, |
| 213 | super::TokenKind::Number, |
| 214 | super::TokenKind::Ident, super::TokenKind::LParen, super::TokenKind::Minus, |
| 215 | super::TokenKind::Number, super::TokenKind::RParen, |
| 216 | super::TokenKind::Eof, |
| 217 | ]; |
| 218 | for expectedKind in expected { |
| 219 | try testing::expect(super::next(&mut s).kind == expectedKind); |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | @test unsafe fn testScanKeywords() throws (testing::TestError) { |
| 224 | let mut s = testScanner("nil mod not static unsafe where"); |
| 225 | let tok1: super::Token = super::next(&mut s); |
| 226 | |
| 227 | try testing::expect(tok1.kind == super::TokenKind::Nil); |
| 228 | try testing::expect(tok1.source.len == 3); |
| 229 | |
| 230 | let tok2: super::Token = super::next(&mut s); |
| 231 | try testing::expect(tok2.kind == super::TokenKind::Mod); |
| 232 | try testing::expect(tok2.source.len == 3); |
| 233 | |
| 234 | let tok3: super::Token = super::next(&mut s); |
| 235 | try testing::expect(tok3.kind == super::TokenKind::Not); |
| 236 | try testing::expect(tok3.source.len == 3); |
| 237 | |
| 238 | let tok4: super::Token = super::next(&mut s); |
| 239 | try testing::expect(tok4.kind == super::TokenKind::Static); |
| 240 | try testing::expect(tok4.source.len == 6); |
| 241 | |
| 242 | let tok5: super::Token = super::next(&mut s); |
| 243 | try testing::expect(tok5.kind == super::TokenKind::Unsafe); |
| 244 | try testing::expect(tok5.source.len == 6); |
| 245 | |
| 246 | let tok6: super::Token = super::next(&mut s); |
| 247 | try testing::expect(tok6.kind == super::TokenKind::Where); |
| 248 | try testing::expect(tok6.source.len == 5); |
| 249 | } |
| 250 | |
| 251 | @test unsafe fn testScanVoidAsIdent() throws (testing::TestError) { |
| 252 | let mut s = testScanner("void"); |
| 253 | let tok: super::Token = super::next(&mut s); |
| 254 | try testing::expect(tok.kind == super::TokenKind::Ident); |
| 255 | try testing::expect(tok.source.len == 4); |
| 256 | } |
| 257 | |
| 258 | @test unsafe fn testScanWhitespaceAndComments() throws (testing::TestError) { |
| 259 | let mut s = testScanner( |
| 260 | " \t\n // This is a comment..\n 42" |
| 261 | ); |
| 262 | let tok: super::Token = super::next(&mut s); |
| 263 | try testing::expect(tok.kind == super::TokenKind::Number); |
| 264 | try testing::expect(tok.source.len == 2); |
| 265 | try testing::expect(super::isEof(&s)); |
| 266 | } |
| 267 | |
| 268 | @test unsafe fn testScanInvalidCharacters() throws (testing::TestError) { |
| 269 | let mut s = testScanner( |
| 270 | "`#$" |
| 271 | ); |
| 272 | let mut tok: super::Token = super::next(&mut s); |
| 273 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
| 274 | try testing::expect(tok.offset == 0); |
| 275 | |
| 276 | set tok = super::next(&mut s); |
| 277 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
| 278 | try testing::expect(tok.offset == 1); |
| 279 | |
| 280 | set tok = super::next(&mut s); |
| 281 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
| 282 | try testing::expect(tok.offset == 2); |
| 283 | } |
| 284 | |
| 285 | @test unsafe fn testScanUnterminatedString() throws (testing::TestError) { |
| 286 | let mut s = testScanner("\"Hello World!"); |
| 287 | let tok: super::Token = super::next(&mut s); |
| 288 | |
| 289 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
| 290 | try testing::expect(tok.source == "unterminated string"); |
| 291 | try testing::expect(tok.offset == 0); |
| 292 | } |
| 293 | |
| 294 | @test unsafe fn testScanUnderminatedStringEscape() throws (testing::TestError) { |
| 295 | let mut s = testScanner("\"\\"); |
| 296 | let tok: super::Token = super::next(&mut s); |
| 297 | |
| 298 | try testing::expect(tok.kind == super::TokenKind::Invalid); |
| 299 | } |
| 300 | |
| 301 | @test unsafe fn testScanFunctionDefinition() throws (testing::TestError) { |
| 302 | let mut s = testScanner( |
| 303 | "fn add(a: i32, b: i32) -> i32 { return a + b; }" |
| 304 | ); |
| 305 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Fn); |
| 306 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 307 | try testing::expect(super::next(&mut s).kind == super::TokenKind::LParen); |
| 308 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 309 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Colon); |
| 310 | try testing::expect(super::next(&mut s).kind == super::TokenKind::I32); |
| 311 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Comma); |
| 312 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 313 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Colon); |
| 314 | try testing::expect(super::next(&mut s).kind == super::TokenKind::I32); |
| 315 | try testing::expect(super::next(&mut s).kind == super::TokenKind::RParen); |
| 316 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Arrow); |
| 317 | try testing::expect(super::next(&mut s).kind == super::TokenKind::I32); |
| 318 | try testing::expect(super::next(&mut s).kind == super::TokenKind::LBrace); |
| 319 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Return); |
| 320 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 321 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Plus); |
| 322 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident); |
| 323 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Semicolon); |
| 324 | try testing::expect(super::next(&mut s).kind == super::TokenKind::RBrace); |
| 325 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 326 | } |
| 327 | |
| 328 | /// Region tokens preserve their names and do not consume character literals. |
| 329 | @test unsafe fn testScanRegions() throws (testing::TestError) { |
| 330 | let mut s = testScanner("'x 'long_2 'x' '\\n' 'outer<'inner"); |
| 331 | let first = super::next(&mut s); |
| 332 | try testing::expect(first.kind == super::TokenKind::Region); |
| 333 | try testing::expect(mem::eq(first.source, "'x")); |
| 334 | let second = super::next(&mut s); |
| 335 | try testing::expect(second.kind == super::TokenKind::Region); |
| 336 | try testing::expect(mem::eq(second.source, "'long_2")); |
| 337 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Char); |
| 338 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Char); |
| 339 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Region); |
| 340 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Lt); |
| 341 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Region); |
| 342 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof); |
| 343 | } |
| 344 | |
| 345 | /// An apostrophe without a name or a closing quote is invalid. |
| 346 | @test unsafe fn testScanMalformedRegions() throws (testing::TestError) { |
| 347 | for text in ["'", "'1", "'\\", "'\n"] { |
| 348 | let mut s = testScanner(text); |
| 349 | try testing::expect(super::next(&mut s).kind == super::TokenKind::Invalid); |
| 350 | } |
| 351 | } |