radiance · commit

Fix small scanner edge-case

6458522743aaef92984b59c9d9a15a6774f89ed0f44b758c5a33640e05bf3548

Alexis Sellier committed 1 month ago 1 parent afd27f21

lib/std/lang/scanner.rad +5 -1

fn scanDelimited(s: *mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
    while let ch = current(s); ch <> delim {
        if not isPrint(ch) {
            return invalid(s.token, "invalid character");
        }
        consume(s, '\\'); // Consume escapes
        if consume(s, '\\') { // Consume escapes
            if isEof(s) {
                return nil;
            }
        }
        advance(s);
    }
    if not consume(s, delim) {
        return nil;
    }

lib/std/lang/scanner/tests.rad +7 -1

    let tok: super::Token = super::next(&mut s);
    try testing::expect(tok.kind == super::TokenKind::Ident);
    try testing::expect(tok.source.len == 4);
}


@test fn testScanWhitespaceAndComments() throws (testing::TestError) {
    let mut s = testScanner(
        "  \t\n  // This is a comment..\n  42"
    );
    let tok: super::Token = super::next(&mut s);

    try testing::expect(tok.kind == super::TokenKind::Invalid);
    try testing::expect(tok.source == "unterminated string");
    try testing::expect(tok.offset == 0);
}

@test fn testScanUnderminatedStringEscape() throws (testing::TestError) {
    let mut s = testScanner("\"\\");
    let tok: super::Token = super::next(&mut s);

    try testing::expect(tok.kind == super::TokenKind::Invalid);
}

@test fn testScanFunctionDefinition() throws (testing::TestError) {
    let mut s = testScanner(
        "fn add(a: i32, b: i32) -> i32 { return a + b; }"
    );
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Fn);

381	381		fn scanDelimited(s: *mut Scanner, delim: u8, kind: TokenKind) -> ?Token {
382	382		while let ch = current(s); ch <> delim {
383	383		if not isPrint(ch) {
384	384		return invalid(s.token, "invalid character");
385	385		}
386		-	consume(s, '\\'); // Consume escapes
	386	+	if consume(s, '\\') { // Consume escapes
	387	+	if isEof(s) {
	388	+	return nil;
	389	+	}
	390	+	}
387	391		advance(s);
388	392		}
389	393		if not consume(s, delim) {
390	394		return nil;
391	395		}

252	252		let tok: super::Token = super::next(&mut s);
253	253		try testing::expect(tok.kind == super::TokenKind::Ident);
254	254		try testing::expect(tok.source.len == 4);
255	255		}
256	256
257		-
258	257		@test fn testScanWhitespaceAndComments() throws (testing::TestError) {
259	258		let mut s = testScanner(
260	259		" \t\n // This is a comment..\n 42"
261	260		);
262	261		let tok: super::Token = super::next(&mut s);

289	288		try testing::expect(tok.kind == super::TokenKind::Invalid);
290	289		try testing::expect(tok.source == "unterminated string");
291	290		try testing::expect(tok.offset == 0);
292	291		}
293	292
	293	+	@test fn testScanUnderminatedStringEscape() throws (testing::TestError) {
	294	+	let mut s = testScanner("\"\\");
	295	+	let tok: super::Token = super::next(&mut s);
	296	+
	297	+	try testing::expect(tok.kind == super::TokenKind::Invalid);
	298	+	}
	299	+
294	300		@test fn testScanFunctionDefinition() throws (testing::TestError) {
295	301		let mut s = testScanner(
296	302		"fn add(a: i32, b: i32) -> i32 { return a + b; }"
297	303		);
298	304		try testing::expect(super::next(&mut s).kind == super::TokenKind::Fn);