compiler: Scan assembly through checked string pool borrows

e4ef0fa52ae072dded070bed36862c41dad8bf67227baeb8edc940bc09b5ef06
Alexis Sellier committed ago 1 parent 38affd56
lib/std/arch/rv64/asm.rad +7 -4
457 457
458 458
/// Parser and emission state.
459 459
export record Assembler {
460 460
    /// Allocation arena for temporary assembler state.
461 461
    arena: *unsafe mut alloc::Arena,
462 +
    /// Intern pool for identifier-shaped token text. It must remain valid while parsing.
463 +
    pool: *unsafe mut strings::Pool,
462 464
    /// Assembler lexical scanner.
463 465
    scan: scanner::Scanner,
464 466
    /// Output text buffer.
465 467
    text: *mut [u32],
466 468
    /// Number of emitted text words.
516 518
    let symbolBuf = symbols as *mut [Symbol];
517 519
    let fixupBuf = fixups as *mut [Fixup];
518 520
    let externalBuf = externalFixups as *mut [Fixup];
519 521
    let mut a = Assembler {
520 522
        arena: arena as *unsafe mut alloc::Arena,
521 -
        scan: scanner::scanner(sourceKind, source, pool),
523 +
        pool,
524 +
        scan: scanner::scanner(sourceKind, source),
522 525
        text: textBuf,
523 526
        textLen: 0,
524 527
        data: dataBuf,
525 528
        dataLen: 0,
526 529
        section: Section::Text,
552 555
        externalFixups: &pendingFixups[..externalFixupsLen],
553 556
    };
554 557
}
555 558
556 559
/// Bound symbol and fixup counts by lexical tokens, including one spare slot.
557 -
unsafe fn tokenCapacity(kind: scanner::SourceKind, source: *[u8], pool: *unsafe mut strings::Pool) -> u32 {
558 -
    let mut scan = scanner::scanner(kind, source, pool);
560 +
fn tokenCapacity(kind: scanner::SourceKind, source: *[u8], pool: &mut strings::Pool) -> u32 {
561 +
    let mut scan = scanner::scanner(kind, source);
559 562
    let mut count = SOURCE_CAP_PADDING;
560 563
    while true {
561 -
        let token = scanner::next(&mut scan);
564 +
        let token = scanner::next(&mut scan, pool);
562 565
        if token.kind == scanner::TokenKind::Eof {
563 566
            break;
564 567
        }
565 568
        set count += 1;
566 569
        if token.kind == scanner::TokenKind::Invalid {
lib/std/arch/rv64/asm/parser.rad +3 -3
40 40
}
41 41
42 42
/// Advance the parser by one token, preserving the previous token.
43 43
unsafe fn advance(a: &mut super::Assembler) {
44 44
    set a.scan.previous = a.scan.current;
45 -
    set a.scan.current = scanner::next(&mut a.scan);
45 +
    set a.scan.current = scanner::next(&mut a.scan, a.pool);
46 46
}
47 47
48 48
/// Consume the current token when it has `kind`.
49 49
unsafe fn consume(a: &mut super::Assembler, kind: scanner::TokenKind) -> bool {
50 50
    if a.scan.current.kind == kind {
139 139
    while consume(a, scanner::TokenKind::ColonColon) {
140 140
        let segment = try expectToken(a, scanner::TokenKind::Ident, "expected identifier after `::`");
141 141
        set end = segment.offset + segment.source.len;
142 142
    }
143 143
    let source = &a.scan.source[start..end];
144 -
    return strings::intern(a.scan.pool, source);
144 +
    return strings::intern(a.pool, source);
145 145
}
146 146
147 147
/// Parse a bare symbol name.
148 148
unsafe fn parseSymbolName(a: &mut super::Assembler) -> *[u8] throws (super::Error) {
149 149
    return try parseScopedName(a, scanner::TokenKind::Ident, "expected symbol name", 0);
162 162
    let storage = try alloc::allocSlice(a.arena, 1, 1, raw.len) catch {
163 163
        panic "asm: out of memory allocating quoted label";
164 164
    } as *mut [u8];
165 165
    let len = fmt::unescapeString(raw, storage);
166 166
167 -
    return strings::intern(a.scan.pool, &storage[..len]);
167 +
    return strings::intern(a.pool, &storage[..len]);
168 168
}
169 169
170 170
/// Parse a label reference or definition name.
171 171
unsafe fn parseLabelName(a: &mut super::Assembler) -> *[u8] throws (super::Error) {
172 172
    if a.scan.current.kind == scanner::TokenKind::QuotedLabel {
lib/std/arch/rv64/asm/scanner.rad +13 -16
61 61
    cursor: u32,
62 62
    /// Current token observed by the parser.
63 63
    current: Token,
64 64
    /// Previously consumed token observed by the parser.
65 65
    previous: Token,
66 -
    /// Intern pool for identifier-shaped token text. It must remain valid while scanning.
67 -
    pool: *unsafe mut strings::Pool,
68 66
}
69 67
70 68
/// Individual token with kind, source text, and byte offset.
71 69
export record Token: Copy {
72 70
    /// Token kind.
76 74
    /// Byte offset of `source` in the input buffer.
77 75
    offset: u32,
78 76
}
79 77
80 78
/// Create a new assembler scanner.
81 -
export fn scanner(sourceKind: SourceKind, source: *[u8], pool: *unsafe mut strings::Pool) -> Scanner {
79 +
export fn scanner(sourceKind: SourceKind, source: *[u8]) -> Scanner {
82 80
    let invalidToken = invalid(0, "");
83 81
    return Scanner {
84 82
        sourceKind,
85 83
        source,
86 84
        token: 0,
87 85
        cursor: 0,
88 86
        current: invalidToken,
89 87
        previous: invalidToken,
90 -
        pool,
91 88
    };
92 89
}
93 90
94 91
/// Create an invalid token with the given message.
95 92
export fn invalid(offset: u32, message: *[u8]) -> Token {
149 146
            else => return,
150 147
        }
151 148
    }
152 149
}
153 150
154 -
/// Return the next assembler token. The retained pool must be valid and writable.
155 -
export unsafe fn next(s: &mut Scanner) -> Token {
151 +
/// Return the next assembler token and intern identifier text in the pool.
152 +
export fn next(s: &mut Scanner, pool: &mut strings::Pool) -> Token {
156 153
    skipWhitespace(s);
157 154
    set s.token = s.cursor;
158 155
159 156
    if isEof(s) {
160 157
        return tok(s, TokenKind::Eof);
163 160
164 161
    if char::isDigit(ch) {
165 162
        return scanNumber(s);
166 163
    }
167 164
    if char::isAlpha(ch) or ch == '_' {
168 -
        return scanIdentToken(s, TokenKind::Ident);
165 +
        return scanIdentToken(s, pool, TokenKind::Ident);
169 166
    }
170 167
171 168
    match ch {
172 169
        case '(' => return tok(s, TokenKind::LParen),
173 170
        case ')' => return tok(s, TokenKind::RParen),
179 176
            }
180 177
            return invalid(s.token, "unexpected `:`");
181 178
        }
182 179
        case '"' => return scanString(s),
183 180
        case '\'' => return scanChar(s),
184 -
        case '.' => return scanPrefixedToken(s, TokenKind::Directive, "expected directive name after `.`"),
185 -
        case '@' => return scanLabelToken(s),
186 -
        case '%' => return scanPrefixedToken(s, TokenKind::Register, "expected register after `%`"),
181 +
        case '.' => return scanPrefixedToken(s, pool, TokenKind::Directive, "expected directive name after `.`"),
182 +
        case '@' => return scanLabelToken(s, pool),
183 +
        case '%' => return scanPrefixedToken(s, pool, TokenKind::Register, "expected register after `%`"),
187 184
        case '-' => return scanSignedNumberOrToken(s, TokenKind::Minus),
188 185
        case '+' => return scanSignedNumberOrToken(s, TokenKind::Plus),
189 186
        case '/' => return tok(s, TokenKind::Slash),
190 187
        case '*' => return tok(s, TokenKind::Star),
191 188
        else => return invalid(s.token, "unexpected character"),
270 267
    }
271 268
    return invalid(s.token, "unterminated character");
272 269
}
273 270
274 271
/// Scan an identifier-shaped token of the given kind.
275 -
unsafe fn scanIdentToken(s: &mut Scanner, kind: TokenKind) -> Token {
272 +
fn scanIdentToken(s: &mut Scanner, pool: &mut strings::Pool, kind: TokenKind) -> Token {
276 273
    scanIdentifierBody(s);
277 274
    let source = &s.source[s.token..s.cursor];
278 275
279 276
    return Token {
280 277
        kind,
281 -
        source: strings::intern(s.pool, source),
278 +
        source: strings::intern(pool, source),
282 279
        offset: s.token,
283 280
    };
284 281
}
285 282
286 283
/// Scan a sigil-prefixed identifier-shaped token.
287 -
unsafe fn scanPrefixedToken(s: &mut Scanner, kind: TokenKind, message: *[u8]) -> Token {
284 +
fn scanPrefixedToken(s: &mut Scanner, pool: &mut strings::Pool, kind: TokenKind, message: *[u8]) -> Token {
288 285
    let ch = current(s) else {
289 286
        return invalid(s.token, message);
290 287
    };
291 288
    if not char::isAlpha(ch) and ch <> '_' {
292 289
        return invalid(s.token, message);
294 291
    scanIdentifierBody(s);
295 292
    let source = &s.source[s.token..s.cursor];
296 293
297 294
    return Token {
298 295
        kind,
299 -
        source: strings::intern(s.pool, source),
296 +
        source: strings::intern(pool, source),
300 297
        offset: s.token,
301 298
    };
302 299
}
303 300
304 301
/// Scan an assembler label token, accepting either `@name` or `@"quoted"` syntax.
305 -
unsafe fn scanLabelToken(s: &mut Scanner) -> Token {
302 +
fn scanLabelToken(s: &mut Scanner, pool: &mut strings::Pool) -> Token {
306 303
    let ch = current(s) else {
307 304
        return invalid(s.token, "expected label after `@`");
308 305
    };
309 306
    if ch == '"' {
310 307
        advance(s);
311 308
        if let token = scanCharsUntil(s, '"', TokenKind::QuotedLabel) {
312 309
            return token;
313 310
        }
314 311
        return invalid(s.token, "unterminated quoted label");
315 312
    }
316 -
    return scanPrefixedToken(s, TokenKind::Label, "expected label after `@`");
313 +
    return scanPrefixedToken(s, pool, TokenKind::Label, "expected label after `@`");
317 314
}
lib/std/arch/rv64/asm/scanner/tests.rad +41 -43
1 1
use std::mem;
2 2
use std::testing;
3 3
use std::lang::strings;
4 4
5 5
/// String pool used by assembler scanner tests.
6 -
unsafe static TEST_STRING_POOL: strings::Pool = strings::Pool { table: undefined, count: 0 };
6 +
static TEST_STRING_POOL: strings::Pool = strings::Pool { table: [""; 32768], count: 0 };
7 7
8 8
/// Create a scanner for test input.
9 9
fn testScanner(source: *[u8]) -> super::Scanner {
10 -
    unsafe {
11 -
        return super::scanner(super::SourceKind::String, source, &mut TEST_STRING_POOL);
12 -
    }
10 +
    return super::scanner(super::SourceKind::String, source);
13 11
}
14 12
15 13
/// Scanner recognizes assembler-specific sigils and scoped names.
16 -
@test unsafe fn testScanRegisterDirectiveAndLabelTokens() throws (testing::TestError) {
14 +
@test fn testScanRegisterDirectiveAndLabelTokens() throws (testing::TestError) {
17 15
    let mut s = testScanner(
18 16
        ".text %sp @entry name::tail 42"
19 17
    );
20 -
    let directive = super::next(&mut s);
18 +
    let directive = super::next(&mut s, &mut TEST_STRING_POOL);
21 19
    try testing::expect(directive.kind == super::TokenKind::Directive);
22 20
    try testing::expect(mem::eq(directive.source, ".text"));
23 21
24 -
    let reg = super::next(&mut s);
22 +
    let reg = super::next(&mut s, &mut TEST_STRING_POOL);
25 23
    try testing::expect(reg.kind == super::TokenKind::Register);
26 24
    try testing::expect(mem::eq(reg.source, "%sp"));
27 25
28 -
    let label = super::next(&mut s);
26 +
    let label = super::next(&mut s, &mut TEST_STRING_POOL);
29 27
    try testing::expect(label.kind == super::TokenKind::Label);
30 28
    try testing::expect(mem::eq(label.source, "@entry"));
31 29
32 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident);
33 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::ColonColon);
34 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident);
35 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Number);
36 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof);
30 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident);
31 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::ColonColon);
32 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident);
33 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Number);
34 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof);
37 35
}
38 36
39 37
/// Keyword-shaped text remains plain assembler identifiers.
40 -
@test unsafe fn testScanKeywordShapedAsmNamesRemainAsmTokens() throws (testing::TestError) {
38 +
@test fn testScanKeywordShapedAsmNamesRemainAsmTokens() throws (testing::TestError) {
41 39
    let mut s = testScanner(
42 40
        "and or not align addi .text @label"
43 41
    );
44 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident);
45 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident);
46 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident);
47 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident);
48 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Ident);
49 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Directive);
50 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Label);
51 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof);
42 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident);
43 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident);
44 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident);
45 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident);
46 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Ident);
47 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Directive);
48 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Label);
49 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof);
52 50
}
53 51
54 52
/// Quoted labels can spell symbol names that are not identifier-shaped.
55 -
@test unsafe fn testScanQuotedLabelToken() throws (testing::TestError) {
53 +
@test fn testScanQuotedLabelToken() throws (testing::TestError) {
56 54
    let mut s = testScanner(
57 55
        "@\"foo.bar.baz\""
58 56
    );
59 -
    let label = super::next(&mut s);
57 +
    let label = super::next(&mut s, &mut TEST_STRING_POOL);
60 58
    try testing::expect(label.kind == super::TokenKind::QuotedLabel);
61 59
    try testing::expect(mem::eq(label.source, "@\"foo.bar.baz\""));
62 -
    try testing::expect(super::next(&mut s).kind == super::TokenKind::Eof);
60 +
    try testing::expect(super::next(&mut s, &mut TEST_STRING_POOL).kind == super::TokenKind::Eof);
63 61
}
64 62
65 63
/// Sigil-prefixed tokens require the name to start immediately after the sigil.
66 -
@test unsafe fn testScanSigilsRequireAdjacency() throws (testing::TestError) {
64 +
@test fn testScanSigilsRequireAdjacency() throws (testing::TestError) {
67 65
    let mut regScan = testScanner("% a0");
68 -
    try testing::expect(super::next(&mut regScan).kind == super::TokenKind::Invalid);
66 +
    try testing::expect(super::next(&mut regScan, &mut TEST_STRING_POOL).kind == super::TokenKind::Invalid);
69 67
70 68
    let mut labelScan = testScanner("@ entry");
71 -
    try testing::expect(super::next(&mut labelScan).kind == super::TokenKind::Invalid);
69 +
    try testing::expect(super::next(&mut labelScan, &mut TEST_STRING_POOL).kind == super::TokenKind::Invalid);
72 70
73 71
    let mut directiveScan = testScanner(". text");
74 -
    try testing::expect(super::next(&mut directiveScan).kind == super::TokenKind::Invalid);
72 +
    try testing::expect(super::next(&mut directiveScan, &mut TEST_STRING_POOL).kind == super::TokenKind::Invalid);
75 73
}
76 74
77 75
/// Scanner reaches EOF after trailing whitespace and comments.
78 -
@test unsafe fn testScanProgramEndingWithNewline() throws (testing::TestError) {
76 +
@test fn testScanProgramEndingWithNewline() throws (testing::TestError) {
79 77
    let mut s = testScanner(
80 78
        ".text;\n@start\naddi %a0 %zero 42;\nsd %a0 8(%sp);\n// comment\nbeq %a0 %zero @done;\n@done\nret;\n"
81 79
    );
82 80
    loop {
83 -
        let tok = super::next(&mut s);
81 +
        let tok = super::next(&mut s, &mut TEST_STRING_POOL);
84 82
        if tok.kind == super::TokenKind::Eof {
85 83
            try testing::expect(tok.source.len == 0);
86 84
            return;
87 85
        }
88 86
    }
89 87
}
90 88
91 89
/// Signed numbers scan only the numeric formats supported by the assembler scanner.
92 -
@test unsafe fn testScanSignedHexAndUnsupportedNumericForms() throws (testing::TestError) {
90 +
@test fn testScanSignedHexAndUnsupportedNumericForms() throws (testing::TestError) {
93 91
    let mut s = testScanner(
94 92
        "+0x2a -0b10 45.5"
95 93
    );
96 -
    let mut tok = super::next(&mut s);
94 +
    let mut tok = super::next(&mut s, &mut TEST_STRING_POOL);
97 95
    try testing::expect(tok.kind == super::TokenKind::Number);
98 96
    try testing::expect(mem::eq(tok.source, "+0x2a"));
99 97
100 -
    set tok = super::next(&mut s);
98 +
    set tok = super::next(&mut s, &mut TEST_STRING_POOL);
101 99
    try testing::expect(tok.kind == super::TokenKind::Number);
102 100
    try testing::expect(mem::eq(tok.source, "-0"));
103 101
104 -
    set tok = super::next(&mut s);
102 +
    set tok = super::next(&mut s, &mut TEST_STRING_POOL);
105 103
    try testing::expect(tok.kind == super::TokenKind::Ident);
106 104
    try testing::expect(mem::eq(tok.source, "b10"));
107 105
108 -
    set tok = super::next(&mut s);
106 +
    set tok = super::next(&mut s, &mut TEST_STRING_POOL);
109 107
    try testing::expect(tok.kind == super::TokenKind::Number);
110 108
    try testing::expect(mem::eq(tok.source, "45"));
111 109
112 -
    set tok = super::next(&mut s);
110 +
    set tok = super::next(&mut s, &mut TEST_STRING_POOL);
113 111
    try testing::expect(tok.kind == super::TokenKind::Invalid);
114 112
    try testing::expect(mem::eq(tok.source, "expected directive name after `.`"));
115 113
116 -
    set tok = super::next(&mut s);
114 +
    set tok = super::next(&mut s, &mut TEST_STRING_POOL);
117 115
    try testing::expect(tok.kind == super::TokenKind::Number);
118 116
    try testing::expect(mem::eq(tok.source, "5"));
119 117
}
120 118
121 119
/// Unterminated string and character literals report invalid tokens.
122 -
@test unsafe fn testScanUnterminatedDelimitedLiterals() throws (testing::TestError) {
120 +
@test fn testScanUnterminatedDelimitedLiterals() throws (testing::TestError) {
123 121
    let mut stringScan = testScanner("\"unterminated");
124 -
    let stringTok = super::next(&mut stringScan);
122 +
    let stringTok = super::next(&mut stringScan, &mut TEST_STRING_POOL);
125 123
    try testing::expect(stringTok.kind == super::TokenKind::Invalid);
126 124
    try testing::expect(mem::eq(stringTok.source, "unterminated string"));
127 125
128 126
    let mut escapedStringScan = testScanner("\"unterminated\\");
129 -
    let escapedStringTok = super::next(&mut escapedStringScan);
127 +
    let escapedStringTok = super::next(&mut escapedStringScan, &mut TEST_STRING_POOL);
130 128
    try testing::expect(escapedStringTok.kind == super::TokenKind::Invalid);
131 129
    try testing::expect(mem::eq(escapedStringTok.source, "unterminated string"));
132 130
133 131
    let mut charScan = testScanner("'x");
134 -
    let charTok = super::next(&mut charScan);
132 +
    let charTok = super::next(&mut charScan, &mut TEST_STRING_POOL);
135 133
    try testing::expect(charTok.kind == super::TokenKind::Invalid);
136 134
    try testing::expect(mem::eq(charTok.source, "unterminated character"));
137 135
138 136
    let mut escapedCharScan = testScanner("'\\");
139 -
    let escapedCharTok = super::next(&mut escapedCharScan);
137 +
    let escapedCharTok = super::next(&mut escapedCharScan, &mut TEST_STRING_POOL);
140 138
    try testing::expect(escapedCharTok.kind == super::TokenKind::Invalid);
141 139
    try testing::expect(mem::eq(escapedCharTok.source, "unterminated character"));
142 140
}