Use null-pointer-optimization for slice types

5aad38982977572918d961cc25411c3e7622a22858cbd4861bf2ca9471c56de9
This reduces the size of `?*[T]` types from 24 bytes to 16 bytes.
Alexis Sellier committed ago 1 parent a8794ea2
lib/std/arch/rv64/tests/builtin.size.align.rad +2 -2
35 35
    assert(@alignOf(?*u32) == 8);
36 36
37 37
    assert(@sizeOf(?*mut u32) == 8);
38 38
    assert(@alignOf(?*mut u32) == 8);
39 39
40 -
    assert(@sizeOf(?*[u16]) == 24);
40 +
    assert(@sizeOf(?*[u16]) == 16);
41 41
    assert(@alignOf(?*[u16]) == 8);
42 42
43 -
    assert(@sizeOf(?*mut [u8]) == 24);
43 +
    assert(@sizeOf(?*mut [u8]) == 16);
44 44
    assert(@alignOf(?*mut [u8]) == 8);
45 45
46 46
    assert(@sizeOf(?Foo) == 12);
47 47
    assert(@alignOf(?Foo) == 4);
48 48
lib/std/arch/rv64/tests/opt.nil.check.rad added +59 -0
1 +
//! Test nil check for optional pointers and slices.
2 +
//! Ensure that nil checks compare the full pointer width, not just the low byte.
3 +
4 +
/// Return a pointer whose low byte is zero but is non-null.
5 +
/// This tests that nil checks use W64, not W8.
6 +
fn makeAlignedPtr() -> *u8 {
7 +
    let arr: [u8; 512] = undefined;
8 +
    // Find an address within arr whose low byte is 0x00.
9 +
    let base: u64 = &arr[0] as u64;
10 +
    let offset: u64 = 256 - (base % 256);
11 +
    return &arr[offset as u32] as *u8;
12 +
}
13 +
14 +
/// Test: optional pointer nil check must use full 64-bit comparison.
15 +
fn testOptionalPtrNilCheck() -> i32 {
16 +
    let p = makeAlignedPtr();
17 +
    let opt: ?*u8 = p;
18 +
19 +
    // The pointer is not nil, but its low byte is 0x00.
20 +
    // A W8 comparison would wrongly say it's nil.
21 +
    if opt == nil {
22 +
        return 1;
23 +
    }
24 +
    if let v = opt {
25 +
        // Good: correctly identified as non-nil.
26 +
        return 0;
27 +
    }
28 +
    return 2;
29 +
}
30 +
31 +
/// Test: optional slice nil check must use full 64-bit comparison.
32 +
fn testOptionalSliceNilCheck() -> i32 {
33 +
    let arr: [u8; 512] = undefined;
34 +
    let base: u64 = &arr[0] as u64;
35 +
    let offset: u64 = 256 - (base % 256);
36 +
    // Create a slice starting at an address whose low byte is 0.
37 +
    let s: *[u8] = &arr[offset as u32 ..];
38 +
39 +
    let opt: ?*[u8] = s;
40 +
    if opt == nil {
41 +
        return 1;
42 +
    }
43 +
    if let v = opt {
44 +
        return 0;
45 +
    }
46 +
    return 2;
47 +
}
48 +
49 +
@default fn main() -> i32 {
50 +
    let r1 = testOptionalPtrNilCheck();
51 +
    if r1 != 0 {
52 +
        return r1;
53 +
    }
54 +
    let r2 = testOptionalSliceNilCheck();
55 +
    if r2 != 0 {
56 +
        return 10 + r2;
57 +
    }
58 +
    return 0;
59 +
}
lib/std/arch/rv64/tests/opt.slice.npo.rad added +198 -0
1 +
//! Test null pointer optimization for optional slices (?*[T]).
2 +
//! Optional slices should have the same size as slices (16 bytes),
3 +
//! using a null data pointer to represent `nil`.
4 +
5 +
fn assert(cond: bool) {
6 +
    if not cond {
7 +
        panic;
8 +
    }
9 +
}
10 +
11 +
fn checkSizes() -> u8 {
12 +
    // ?*[T] should be the same size as *[T] (16 bytes, not 24).
13 +
    if @sizeOf(?*[u8]) != 16 {
14 +
        return 1;
15 +
    }
16 +
    if @alignOf(?*[u8]) != 8 {
17 +
        return 2;
18 +
    }
19 +
    if @sizeOf(?*[u16]) != 16 {
20 +
        return 3;
21 +
    }
22 +
    if @sizeOf(?*mut [u8]) != 16 {
23 +
        return 4;
24 +
    }
25 +
    return 0;
26 +
}
27 +
28 +
fn checkNil() -> u8 {
29 +
    let x: ?*[u8] = nil;
30 +
    if x != nil {
31 +
        return 10;
32 +
    }
33 +
    return 0;
34 +
}
35 +
36 +
fn checkWrap() -> u8 {
37 +
    let arr: [u8; 3] = [1, 2, 3];
38 +
    let s = &arr[..];
39 +
    let opt: ?*[u8] = s;
40 +
41 +
    if opt == nil {
42 +
        return 20;
43 +
    }
44 +
    return 0;
45 +
}
46 +
47 +
fn checkIfLet() -> u8 {
48 +
    let arr: [u8; 3] = [10, 20, 30];
49 +
    let s = &arr[..];
50 +
    let opt: ?*[u8] = s;
51 +
52 +
    if let val = opt {
53 +
        if val.len != 3 {
54 +
            return 31;
55 +
        }
56 +
        if val[0] != 10 {
57 +
            return 32;
58 +
        }
59 +
        if val[1] != 20 {
60 +
            return 33;
61 +
        }
62 +
    } else {
63 +
        return 34;
64 +
    }
65 +
66 +
    let none: ?*[u8] = nil;
67 +
    if let _ = none {
68 +
        return 35;
69 +
    }
70 +
    return 0;
71 +
}
72 +
73 +
fn checkLetElse() -> u8 {
74 +
    let arr: [u8; 3] = [10, 20, 30];
75 +
    let s = &arr[..];
76 +
    let opt: ?*[u8] = s;
77 +
78 +
    let val = opt else {
79 +
        return 40;
80 +
    };
81 +
    if val.len != 3 {
82 +
        return 41;
83 +
    }
84 +
    return 0;
85 +
}
86 +
87 +
fn returnNil() -> ?*[u8] {
88 +
    return nil;
89 +
}
90 +
91 +
fn returnSome() -> ?*[u8] {
92 +
    let arr: [u8; 2] = [42, 99];
93 +
    return &arr[..];
94 +
}
95 +
96 +
fn checkReturn() -> u8 {
97 +
    let a = returnNil();
98 +
    if a != nil {
99 +
        return 50;
100 +
    }
101 +
    let b = returnSome();
102 +
    if b == nil {
103 +
        return 51;
104 +
    }
105 +
    if let val = b {
106 +
        if val[0] != 42 {
107 +
            return 52;
108 +
        }
109 +
    } else {
110 +
        return 53;
111 +
    }
112 +
    return 0;
113 +
}
114 +
115 +
fn checkMatch() -> u8 {
116 +
    let arr: [u8; 2] = [5, 6];
117 +
    let s = &arr[..];
118 +
    let opt: ?*[u8] = s;
119 +
120 +
    match opt {
121 +
        case nil => {
122 +
            return 60;
123 +
        }
124 +
        else => {}
125 +
    }
126 +
127 +
    let none: ?*[u8] = nil;
128 +
    match none {
129 +
        case nil => {}
130 +
        else => {
131 +
            return 61;
132 +
        }
133 +
    }
134 +
    return 0;
135 +
}
136 +
137 +
fn checkEq() -> u8 {
138 +
    let a: ?*[u8] = nil;
139 +
    let b: ?*[u8] = nil;
140 +
141 +
    // nil == nil
142 +
    if a != b {
143 +
        return 70;
144 +
    }
145 +
146 +
    let arr: [u8; 2] = [1, 2];
147 +
    let s = &arr[..];
148 +
    let c: ?*[u8] = s;
149 +
150 +
    // some != nil
151 +
    if c == a {
152 +
        return 71;
153 +
    }
154 +
155 +
    // some == some (same pointer)
156 +
    let d: ?*[u8] = s;
157 +
    if c != d {
158 +
        return 72;
159 +
    }
160 +
161 +
    return 0;
162 +
}
163 +
164 +
@default fn main() -> u8 {
165 +
    let r1 = checkSizes();
166 +
    if r1 != 0 {
167 +
        return r1;
168 +
    }
169 +
    let r2 = checkNil();
170 +
    if r2 != 0 {
171 +
        return r2;
172 +
    }
173 +
    let r3 = checkWrap();
174 +
    if r3 != 0 {
175 +
        return r3;
176 +
    }
177 +
    let r4 = checkIfLet();
178 +
    if r4 != 0 {
179 +
        return r4;
180 +
    }
181 +
    let r5 = checkLetElse();
182 +
    if r5 != 0 {
183 +
        return r5;
184 +
    }
185 +
    let r6 = checkReturn();
186 +
    if r6 != 0 {
187 +
        return r6;
188 +
    }
189 +
    let r7 = checkMatch();
190 +
    if r7 != 0 {
191 +
        return r7;
192 +
    }
193 +
    let r8 = checkEq();
194 +
    if r8 != 0 {
195 +
        return r8;
196 +
    }
197 +
    return 0;
198 +
}
lib/std/lang/lower.rad +52 -18
543 543
544 544
/// Classifies a match subject by how it should be compared and destructured.
545 545
union MatchSubjectKind {
546 546
    /// Regular value: direct equality comparison.
547 547
    Regular,
548 -
    /// Optional pointer `?*T`: compared against null.
548 +
    /// Optional with null pointer optimization: `?*T`, `?*[T]`.
549 549
    OptionalPtr,
550 550
    /// Optional aggregate `?T`: tagged union with payload.
551 551
    OptionalAggregate,
552 552
    /// Union type: tag compared against variant indices.
553 553
    Union(resolver::UnionType),
554 554
}
555 555
556 556
/// Determine the kind of a match subject from its type.
557 557
fn matchSubjectKind(type: resolver::Type) -> MatchSubjectKind {
558 -
    if let case resolver::Type::Optional(inner) = type {
559 -
        if let case resolver::Type::Pointer { .. } = *inner {
560 -
            return MatchSubjectKind::OptionalPtr;
561 -
        }
558 +
    if resolver::isOptionalPointer(type) {
559 +
        return MatchSubjectKind::OptionalPtr;
560 +
    }
561 +
    if resolver::isOptionalAggregate(type) {
562 562
        return MatchSubjectKind::OptionalAggregate;
563 563
    }
564 564
    if let info = unionInfoFromType(type) {
565 565
        return MatchSubjectKind::Union(info);
566 566
    }
2132 2132
    }
2133 2133
    let isNil = pattern.value == ast::NodeValue::Nil;
2134 2134
2135 2135
    match subject.kind {
2136 2136
        case MatchSubjectKind::OptionalPtr if isNil => {
2137 -
            // Optional pointer: `nil` means `null` pointer.
2138 -
            try emitBrCmp(self, il::CmpOp::Eq, subject.ilType, subject.val, il::Val::Imm(0), matchBlock, fallthrough);
2137 +
            // Null pointer optimization: branch on the data pointer being null.
2138 +
            let nilReg = try optionalNilReg(self, subject.val, subject.type);
2139 +
            try emitBrCmp(self, il::CmpOp::Eq, il::Type::W64, il::Val::Reg(nilReg), il::Val::Imm(0), matchBlock, fallthrough);
2139 2140
        }
2140 2141
        case MatchSubjectKind::OptionalAggregate => {
2141 2142
            let base = try emitValToReg(self, subject.val);
2142 2143
2143 2144
            if isNil { // Optional aggregate: `nil` means tag is zero.
2905 2906
    emitLoadW64At(self, tagReg, base, TVAL_TAG_OFFSET);
2906 2907
    return tagReg;
2907 2908
}
2908 2909
2909 2910
/// Get the register to compare against `0` for optional `nil` checking.
2910 -
/// For pointer optionals, returns the pointer itself.
2911 -
/// For aggregate optionals, returns the tag register.
2911 +
/// For null-ptr-optimized types, loads the data pointer, or returns it
2912 +
/// directly for scalar pointers. For aggregates, returns the tag register.
2912 2913
fn optionalNilReg(self: *mut FnLowerer, val: il::Val, typ: resolver::Type) -> il::Reg throws (LowerError) {
2913 2914
    let reg = try emitValToReg(self, val);
2914 -
    if let case MatchSubjectKind::OptionalAggregate = matchSubjectKind(typ) {
2915 -
        return tvalTagReg(self, reg);
2915 +
    let case resolver::Type::Optional(inner) = typ else return reg;
2916 +
2917 +
    if let case resolver::Type::Slice { .. } = *inner {
2918 +
        let ptrReg = nextReg(self);
2919 +
        emitLoadW64At(self, ptrReg, reg, SLICE_PTR_OFFSET);
2920 +
        return ptrReg;
2916 2921
    }
2917 -
    return reg;
2922 +
    if let case resolver::Type::Pointer { .. } = *inner {
2923 +
        return reg;
2924 +
    }
2925 +
    return tvalTagReg(self, reg);
2918 2926
}
2919 2927
2920 2928
/// Lower an optional nil check (`opt == nil` or `opt != nil`).
2921 2929
fn lowerNilCheck(self: *mut FnLowerer, opt: *ast::Node, isEq: bool) -> il::Val throws (LowerError) {
2922 2930
    let optTy = resolver::typeFor(self.low.resolver, opt) else {
2931 2939
        }
2932 2940
    }
2933 2941
    let val = try lowerExpr(self, opt);
2934 2942
    let cmpReg = try optionalNilReg(self, val, optTy);
2935 2943
2944 +
    // Null-pointer-optimized types compare a 64-bit pointer against zero.
2945 +
    // Aggregate optionals compare an 8-bit tag byte against zero.
2946 +
    let mut cmpType = il::Type::W8;
2947 +
    if resolver::isOptionalPointer(optTy) {
2948 +
        cmpType = il::Type::W64;
2949 +
    }
2950 +
2936 2951
    if isEq {
2937 -
        return emitTypedBinOp(self, il::BinOp::Eq, il::Type::W8, il::Val::Reg(cmpReg), il::Val::Imm(0));
2952 +
        return emitTypedBinOp(self, il::BinOp::Eq, cmpType, il::Val::Reg(cmpReg), il::Val::Imm(0));
2938 2953
    } else {
2939 -
        return emitTypedBinOp(self, il::BinOp::Ne, il::Type::W8, il::Val::Reg(cmpReg), il::Val::Imm(0));
2954 +
        return emitTypedBinOp(self, il::BinOp::Ne, cmpType, il::Val::Reg(cmpReg), il::Val::Imm(0));
2940 2955
    }
2941 2956
}
2942 2957
2943 2958
/// Load the payload value from a tagged value aggregate at the given offset.
2944 2959
fn tvalPayloadVal(self: *mut FnLowerer, base: il::Reg, payload: resolver::Type, valOffset: i32) -> il::Val {
3611 3626
            return not resolver::isVoidUnion(typ);
3612 3627
        }
3613 3628
        case resolver::Type::Slice { .. } => return true,
3614 3629
        case resolver::Type::Array(_) => return true,
3615 3630
        case resolver::Type::Nil => return true,
3616 -
        else => return resolver::isOptionalAggregate(typ),
3631 +
        else => {
3632 +
            // Optional pointers are scalar (single word). All other
3633 +
            // optionals, including optional slices with NPO, are aggregates.
3634 +
            if let case resolver::Type::Optional(inner) = typ {
3635 +
                if let case resolver::Type::Pointer { .. } = *inner {
3636 +
                    return false;
3637 +
                }
3638 +
                return true;
3639 +
            }
3640 +
            return false;
3641 +
        }
3617 3642
    }
3618 3643
}
3619 3644
3620 3645
/// Check if a node is a void union variant literal (e.g. `Color::Red`).
3621 3646
/// If so, returns the variant's tag index. This enables optimized comparisons
3735 3760
/// with the tag set to `1`, and the value as payload.
3736 3761
fn wrapInOptional(self: *mut FnLowerer, val: il::Val, optType: resolver::Type) -> il::Val throws (LowerError) {
3737 3762
    let case resolver::Type::Optional(inner) = optType else {
3738 3763
        throw LowerError::ExpectedOptional;
3739 3764
    };
3740 -
    if let case resolver::Type::Pointer { .. } = *inner {
3765 +
    // Null-pointer-optimized (NPO) types are used as-is -- valid values are never null.
3766 +
    if resolver::isNullableType(*inner) {
3741 3767
        return val;
3742 3768
    }
3743 3769
    let layout = resolver::getTypeLayout(optType);
3744 3770
    let valOffset = resolver::getOptionalValOffset(*inner) as i32;
3745 3771
3755 3781
        throw LowerError::ExpectedOptional;
3756 3782
    };
3757 3783
    if let case resolver::Type::Pointer { .. } = *inner {
3758 3784
        return il::Val::Imm(0);
3759 3785
    }
3786 +
    if let case resolver::Type::Slice { item, mutable } = *inner {
3787 +
        return try buildSliceValue(self, item, mutable, il::Val::Imm(0), il::Val::Imm(0));
3788 +
    }
3760 3789
    let valOffset = resolver::getOptionalValOffset(*inner) as i32;
3761 3790
    return try buildTagged(self, resolver::getTypeLayout(optType), 0, nil, *inner, 1, valOffset);
3762 3791
}
3763 3792
3764 3793
/// Build a result value for throwing functions.
4182 4211
    a: il::Reg,
4183 4212
    b: il::Reg,
4184 4213
    offset: i32
4185 4214
) -> il::Val throws (LowerError) {
4186 4215
    match typ {
4187 -
        case resolver::Type::Optional(inner) =>
4188 -
            return try lowerOptionalEq(self, *inner, a, b, offset),
4216 +
        case resolver::Type::Optional(inner) => {
4217 +
            if let case resolver::Type::Slice { item, mutable } = *inner {
4218 +
                // Optional slices use null pointer optimization.
4219 +
                return try lowerSliceEq(self, item, mutable, a, b, offset);
4220 +
            }
4221 +
            return try lowerOptionalEq(self, *inner, a, b, offset);
4222 +
        }
4189 4223
        case resolver::Type::Slice { item, mutable } =>
4190 4224
            return try lowerSliceEq(self, item, mutable, a, b, offset),
4191 4225
        case resolver::Type::Array(arr) =>
4192 4226
            return try lowerArrayEq(self, arr, a, b, offset),
4193 4227
        case resolver::Type::Nominal(nom) => {
lib/std/lang/lower/tests/nil.cmp.ril +2 -2
8 8
    ret 0;
9 9
}
10 10
11 11
fn w8 $nilEqOptPtr(w64 %0) {
12 12
  @entry0
13 -
    eq w8 %1 %0 0;
13 +
    eq w64 %1 %0 0;
14 14
    ret %1;
15 15
}
16 16
17 17
fn w8 $nilNeOptPtr(w64 %0) {
18 18
  @entry0
19 -
    ne w8 %1 %0 0;
19 +
    ne w64 %1 %0 0;
20 20
    ret %1;
21 21
}
22 22
23 23
fn w8 $nilEqOptAgg(w64 %0) {
24 24
  @entry0
lib/std/lang/lower/tests/opt.slice.npo.rad added +14 -0
1 +
//! Test null pointer optimization for optional slices.
2 +
//! ?*[T] uses the same representation as *[T], with nil = null data pointer.
3 +
4 +
fn wrapSlice(s: *[u8]) -> ?*[u8] {
5 +
    return s;
6 +
}
7 +
8 +
fn nilSlice() -> ?*[u8] {
9 +
    return nil;
10 +
}
11 +
12 +
fn nilCheck(s: ?*[u8]) -> bool {
13 +
    return s == nil;
14 +
}
lib/std/lang/lower/tests/opt.slice.npo.ril added +21 -0
1 +
fn w64 $wrapSlice(w64 %0, w64 %1) {
2 +
  @entry0
3 +
    blit %0 %1 16;
4 +
    ret %0;
5 +
}
6 +
7 +
fn w64 $nilSlice(w64 %0) {
8 +
  @entry0
9 +
    reserve %1 16 8;
10 +
    store w64 0 %1 0;
11 +
    store w32 0 %1 8;
12 +
    blit %0 %1 16;
13 +
    ret %0;
14 +
}
15 +
16 +
fn w8 $nilCheck(w64 %0) {
17 +
  @entry0
18 +
    load w64 %1 %0 0;
19 +
    eq w64 %2 %1 0;
20 +
    ret %2;
21 +
}
lib/std/lang/lower/tests/optional.ptr.eq.ril +2 -2
10 10
    ret %2;
11 11
}
12 12
13 13
fn w8 $optPtrEqNil(w64 %0) {
14 14
  @entry0
15 -
    eq w8 %1 %0 0;
15 +
    eq w64 %1 %0 0;
16 16
    ret %1;
17 17
}
18 18
19 19
fn w8 $optPtrNeqNil(w64 %0) {
20 20
  @entry0
21 -
    ne w8 %1 %0 0;
21 +
    ne w64 %1 %0 0;
22 22
    ret %1;
23 23
}
lib/std/lang/resolver.rad +17 -11
1290 1290
    };
1291 1291
}
1292 1292
1293 1293
/// Get the layout of an optional type.
1294 1294
pub fn getOptionalLayout(inner: Type) -> Layout {
1295 -
    if let case Type::Pointer { .. } = inner {
1296 -
        return Layout { size: PTR_SIZE, alignment: PTR_SIZE };
1295 +
    // Nullable types use null pointer optimization -- no tag byte needed.
1296 +
    if isNullableType(inner) {
1297 +
        return getTypeLayout(inner);
1297 1298
    }
1298 1299
    let innerLayout = getTypeLayout(inner);
1299 1300
    let tagSize: u32 = 1;
1300 1301
    let valOffset = mem::alignUp(tagSize, innerLayout.alignment);
1301 1302
    let alignment = max(innerLayout.alignment, 1);
1318 1319
        case Type::Optional(_) => return true,
1319 1320
        else => return false,
1320 1321
    }
1321 1322
}
1322 1323
1323 -
/// Check if a type uses the optional pointer representation.
1324 +
/// Check if a type uses null pointer optimization.
1325 +
/// This applies to optional pointers `?*T` and optional slices `?*[T]`,
1326 +
/// where `nil` is represented as a null data pointer with no tag byte.
1324 1327
pub fn isOptionalPointer(ty: Type) -> bool {
1325 1328
    if let case Type::Optional(inner) = ty {
1326 -
        if let case Type::Pointer { .. } = *inner {
1327 -
            return true;
1328 -
        }
1329 +
        return isNullableType(*inner);
1329 1330
    }
1330 1331
    return false;
1331 1332
}
1332 1333
1333 1334
/// Check if a type uses the optional aggregate representation.
1334 1335
pub fn isOptionalAggregate(ty: Type) -> bool {
1335 1336
    if let case Type::Optional(inner) = ty {
1336 -
        if let case Type::Pointer { .. } = *inner {
1337 -
            // Optional pointers are represented as scalars for efficiency.
1338 -
            return false;
1339 -
        }
1340 -
        return true;
1337 +
        return not isNullableType(*inner);
1341 1338
    }
1342 1339
    return false;
1343 1340
}
1344 1341
1342 +
/// Check if a type can use null to represent `nil`.
1343 +
/// Pointers and slices have a data pointer that is never null when valid.
1344 +
pub fn isNullableType(ty: Type) -> bool {
1345 +
    match ty {
1346 +
        case Type::Pointer { .. }, Type::Slice { .. } => return true,
1347 +
        else => return false,
1348 +
    }
1349 +
}
1350 +
1345 1351
/// Get the layout of a nominal type.
1346 1352
pub fn getNominalLayout(info: NominalType) -> Layout {
1347 1353
    match info {
1348 1354
        case NominalType::Placeholder(_) => {
1349 1355
            panic "getNominalLayout: placeholder type";