Skip to content

Commit a5caee1

Browse files
gottesmmgottesmm
committed
[concurrency] Implement bit masking for TBI when available or in tagged pointer bits otherwise.
rdar://156525771
1 parent b472f28 commit a5caee1

File tree

8 files changed

+328
-20
lines changed

8 files changed

+328
-20
lines changed

‎lib/IRGen/IRGenFunction.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,10 @@ SILModule &IRGenFunction::getSILModule() const {
105105
return IGM.getSILModule();
106106
}
107107

108+
ASTContext &IRGenFunction::getASTContext() const {
109+
return getSILModule().getASTContext();
110+
}
111+
108112
Lowering::TypeConverter &IRGenFunction::getSILTypes() const {
109113
return IGM.getSILTypes();
110114
}

‎lib/IRGen/IRGenFunction.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ class IRGenFunction {
8484
llvm::Function *const CurFn;
8585
ModuleDecl *getSwiftModule() const;
8686
SILModule &getSILModule() const;
87+
ASTContext &getASTContext() const;
8788
Lowering::TypeConverter &getSILTypes() const;
8889
const IRGenOptions &getOptions() const;
8990

‎lib/IRGen/IRGenSIL.cpp

Lines changed: 73 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,9 @@ class LoweredValue {
312312
bool isBoxWithAddress() const {
313313
return kind == Kind::OwnedAddress;
314314
}
315-
315+
316+
bool isExplosionVector() const { return kind == Kind::ExplosionVector; }
317+
316318
const StackAddress &getStackAddress() const {
317319
return Storage.get<StackAddress>(kind);
318320
}
@@ -3373,12 +3375,44 @@ void IRGenSILFunction::visitExistentialMetatypeInst(
33733375
setLoweredExplosion(i, result);
33743376
}
33753377

3376-
static void emitApplyArgument(IRGenSILFunction &IGF,
3377-
SILValue arg,
3378-
SILType paramType,
3379-
Explosion &out,
3380-
SILInstruction *apply = nullptr,
3381-
unsigned idx = 0) {
3378+
static llvm::Value *clearImplicitIsolatedActorBits(IRGenFunction &IGF,
3379+
llvm::Value *value) {
3380+
// Helper. We conditionally use this and just use a simple helper function
3381+
// here so we do not expose the internals of clearing implicit isolated actor
3382+
// bits into the rest of the file. The work here is very straight forward.
3383+
auto getTBIClearMask = [](IRGenFunction &IGF) -> llvm::Value * {
3384+
auto *one = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, 1);
3385+
auto *three = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, 3);
3386+
auto *four = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, 4);
3387+
auto *valueToShiftLit = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, 3);
3388+
3389+
auto *sizeOfWord =
3390+
llvm::ConstantInt::get(IGF.IGM.IntPtrTy, one->getBitWidth() >> 3);
3391+
3392+
// sizeof(Word) - 1
3393+
auto *sub = IGF.Builder.CreateSub(sizeOfWord, one);
3394+
// (sizeof(Word) - 1) << 3
3395+
auto *innerShift = IGF.Builder.CreateShl(sub, three);
3396+
// ((sizeof(Word) - 1) << 3) + 4
3397+
auto *innerShiftOffset = IGF.Builder.CreateAdd(innerShift, four);
3398+
auto *negBits = llvm::ConstantInt::get(IGF.IGM.IntPtrTy, -1);
3399+
// (valueToShiftLit << innerShiftOffset) ^ -1.
3400+
return IGF.Builder.CreateXor(
3401+
IGF.Builder.CreateShl(valueToShiftLit, innerShiftOffset), negBits);
3402+
};
3403+
3404+
auto *cast = IGF.Builder.CreateBitOrPointerCast(value, IGF.IGM.IntPtrTy);
3405+
auto *bitMask = IGF.getASTContext().LangOpts.HasAArch64TBI
3406+
? getTBIClearMask(IGF)
3407+
: llvm::ConstantInt::get(IGF.IGM.IntPtrTy, -4);
3408+
auto *result = IGF.Builder.CreateAnd(cast, bitMask);
3409+
return IGF.Builder.CreateBitOrPointerCast(result, value->getType());
3410+
}
3411+
3412+
static void emitApplyArgument(IRGenSILFunction &IGF, SILValue arg,
3413+
SILType paramType, Explosion &out,
3414+
SILInstruction *apply = nullptr, unsigned idx = 0,
3415+
bool isImplicitIsolatedParameter = false) {
33823416
bool isSubstituted = (arg->getType() != paramType);
33833417

33843418
// For indirect arguments, we just need to pass a pointer.
@@ -3420,7 +3454,20 @@ static void emitApplyArgument(IRGenSILFunction &IGF,
34203454
}
34213455
canForwardLoadToIndirect = true;
34223456
}();
3423-
IGF.getLoweredExplosion(arg, out);
3457+
3458+
// If we are emitting a parameter for an implicit isolated parameter, then
3459+
// we need to clear the implicit isolated actor bits.
3460+
if (isImplicitIsolatedParameter) {
3461+
auto &loweredValue = IGF.getLoweredValue(arg);
3462+
assert(loweredValue.isExplosionVector() &&
3463+
"Should be an explosion of two pointers");
3464+
auto explosionVector = loweredValue.getKnownExplosionVector();
3465+
assert(explosionVector.size() == 2 && "We should have two values");
3466+
out.add(explosionVector[0]);
3467+
out.add(clearImplicitIsolatedActorBits(IGF, explosionVector[1]));
3468+
} else {
3469+
IGF.getLoweredExplosion(arg, out);
3470+
}
34243471
if (canForwardLoadToIndirect) {
34253472
IGF.setForwardableArgument(idx);
34263473
}
@@ -3838,6 +3885,20 @@ void IRGenSILFunction::visitFullApplySite(FullApplySite site) {
38383885
}
38393886
}
38403887

3888+
// Extract the implicit isolated parameter so that we can mask it as
3889+
// appropriate.
3890+
//
3891+
// NOTE: We cannot just drop_front since we could be between the indirect
3892+
// results and the parameters.
3893+
std::optional<unsigned> implicitIsolatedParameterIndex;
3894+
if (auto actorIsolation = site.getFunction()->getActorIsolation();
3895+
actorIsolation && actorIsolation->isCallerIsolationInheriting() &&
3896+
site.isCallerIsolationInheriting()) {
3897+
auto *iso = site.getIsolatedArgumentOperandOrNullPtr();
3898+
assert(iso);
3899+
implicitIsolatedParameterIndex = site.getAppliedArgIndex(*iso);
3900+
}
3901+
38413902
// Lower the arguments and return value in the callee's generic context.
38423903
GenericContextScope scope(IGM,
38433904
origCalleeType->getInvocationGenericSignature());
@@ -3898,8 +3959,11 @@ void IRGenSILFunction::visitFullApplySite(FullApplySite site) {
38983959
emission->setIndirectTypedErrorResultSlot(addr.getAddress());
38993960
continue;
39003961
}
3962+
39013963
emitApplyArgument(*this, args[index], emission->getParameterType(index),
3902-
llArgs, site.getInstruction(), index);
3964+
llArgs, site.getInstruction(), index,
3965+
implicitIsolatedParameterIndex &&
3966+
*implicitIsolatedParameterIndex == index);
39033967
}
39043968

39053969
// Pass the generic arguments.

‎lib/SIL/Utils/ConcurrencyUtils.cpp

Lines changed: 107 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,121 @@
1717

1818
using namespace swift;
1919

20+
/// We could use a higher bit. But by reusing the first bit, we just save a
21+
/// little bit of code.
22+
///
23+
/// $valueToShift << (((sizeof(Word) - 1) << 3) + 4)
24+
///
25+
/// Mathematicaly this is $valueToShift * 2**((sizeof(Word) - 1)*8 + 4)
26+
///
27+
/// On 64 bit this is 60. This works since we want to use the bottom two bits of
28+
/// the top nibble of the TBI bits.
29+
static SILValue getTBIBits(SILBuilder &b, SILLocation loc,
30+
unsigned valueToShift = 1) {
31+
auto &ctx = b.getASTContext();
32+
auto silWordType = SILType::getBuiltinWordType(ctx);
33+
34+
auto id = ctx.getIdentifier(getBuiltinName(BuiltinValueKind::Sizeof));
35+
auto *builtin = cast<FuncDecl>(getBuiltinValueDecl(ctx, id));
36+
auto wordType = BuiltinIntegerType::getWordType(ctx)->getCanonicalType();
37+
auto metatypeTy = SILType::getPrimitiveObjectType(CanMetatypeType::get(
38+
wordType->getCanonicalType(), MetatypeRepresentation::Thin));
39+
auto metatypeVal = b.createMetatype(loc, metatypeTy);
40+
41+
auto sizeOfWord =
42+
b.createBuiltin(loc, id, silWordType,
43+
SubstitutionMap::get(builtin->getGenericSignature(),
44+
ArrayRef<Type>{wordType},
45+
LookUpConformanceInModule()),
46+
{metatypeVal});
47+
auto one = b.createIntegerLiteral(loc, silWordType, 1);
48+
auto three = b.createIntegerLiteral(loc, silWordType, 3);
49+
auto four = b.createIntegerLiteral(loc, silWordType, 4);
50+
auto valueToShiftLit = b.createIntegerLiteral(loc, silWordType, valueToShift);
51+
52+
// sizeof(Word) - 1
53+
auto sub = b.createBuiltinBinaryFunction(loc, "sub", silWordType, silWordType,
54+
{sizeOfWord, one});
55+
// (sizeof(Word) - 1) << 3
56+
auto innerShift = b.createBuiltinBinaryFunction(loc, "shl", silWordType,
57+
silWordType, {sub, three});
58+
// ((sizeof(Word) - 1) << 3) + 4
59+
auto innerShiftOffset = b.createBuiltinBinaryFunction(
60+
loc, "add", silWordType, silWordType, {innerShift, four});
61+
auto outerShift =
62+
b.createBuiltinBinaryFunction(loc, "shl", silWordType, silWordType,
63+
{valueToShiftLit, innerShiftOffset});
64+
return outerShift;
65+
}
66+
67+
/// Construct the TBI mask in a platform independent way that works on all
68+
/// platforms.
69+
///
70+
/// We compute:
71+
///
72+
/// mask = (0x3 << (((sizeof(Word) - 1) << 3) + 4)) ^ -1
73+
static SILValue getTBIClearMask(SILBuilder &b, SILLocation loc) {
74+
auto &ctx = b.getASTContext();
75+
auto silWordType = SILType::getBuiltinWordType(ctx);
76+
auto negBits = b.createIntegerLiteral(loc, silWordType, -1);
77+
78+
return b.createBuiltinBinaryFunction(loc, "xor", silWordType, silWordType,
79+
{getTBIBits(b, loc, 3), negBits});
80+
}
81+
82+
static SILValue transformTupleElts(
83+
SILBuilder &b, SILLocation loc, SILValue mv, SILType finalType,
84+
llvm::function_ref<SILValue(ArrayRef<SILValue> destructureValues)> func) {
85+
auto &ctx = b.getASTContext();
86+
auto silWordType = SILType::getBuiltinWordType(ctx);
87+
auto tupleType =
88+
SILType::getTupleType(b.getASTContext(), {silWordType, silWordType});
89+
auto cast = b.emitUncheckedValueCast(loc, mv, tupleType);
90+
SmallVector<SILValue, 2> destructureValues;
91+
b.emitDestructureOperation(loc, cast, destructureValues);
92+
SILValue reformedValue = func(destructureValues);
93+
auto reformedPointer = b.emitUncheckedOwnershipConversion(
94+
loc, b.emitUncheckedValueCast(loc, reformedValue, finalType),
95+
OwnershipKind::Guaranteed);
96+
return b.emitMarkDependence(loc, reformedPointer, mv,
97+
MarkDependenceKind::NonEscaping);
98+
}
99+
20100
SILValue swift::clearImplicitIsolationActorBits(SILBuilder &b, SILLocation loc,
21101
SILValue value,
22102
SILType finalType) {
23103
if (!finalType)
24104
finalType =
25105
SILType::getBuiltinImplicitIsolationActorType(b.getASTContext());
26-
if (value->getType() == finalType)
27-
return value;
28-
return b.emitUncheckedValueCast(loc, value, finalType);
106+
107+
auto &ctx = b.getASTContext();
108+
109+
return transformTupleElts(
110+
b, loc, value, finalType, [&](ArrayRef<SILValue> tupleElts) {
111+
auto silWordType = SILType::getBuiltinWordType(ctx);
112+
SILValue bitMask = ctx.LangOpts.HasAArch64TBI
113+
? getTBIClearMask(b, loc)
114+
: b.createIntegerLiteral(loc, silWordType, -4);
115+
116+
auto result = b.createBuiltinBinaryFunction(
117+
loc, "and", silWordType, silWordType, {tupleElts[1], bitMask});
118+
return b.createTuple(loc, {tupleElts[0], result});
119+
});
29120
}
30121

31122
SILValue swift::setImplicitIsolationActorBits(SILBuilder &b, SILLocation loc,
32123
SILValue value) {
33-
return value;
124+
auto &ctx = b.getASTContext();
125+
126+
return transformTupleElts(
127+
b, loc, value, value->getType(), [&](ArrayRef<SILValue> tupleElts) {
128+
auto silWordType = SILType::getBuiltinWordType(ctx);
129+
SILValue bitMask = ctx.LangOpts.HasAArch64TBI
130+
? getTBIBits(b, loc)
131+
: b.createIntegerLiteral(loc, silWordType, 1);
132+
133+
auto result = b.createBuiltinBinaryFunction(
134+
loc, "or", silWordType, silWordType, {tupleElts[1], bitMask});
135+
return b.createTuple(loc, {tupleElts[0], result});
136+
});
34137
}

‎lib/SILGen/SILGenBuilder.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1255,5 +1255,5 @@ SILValue SILGenBuilder::convertToImplicitIsolationActor(SILLocation loc,
12551255
"Builtin.ImplicitIsolationActor");
12561256
if (value->getOwnershipKind() != OwnershipKind::Guaranteed)
12571257
value = SGF.emitManagedBeginBorrow(loc, value).getValue();
1258-
return createUncheckedValueCast(loc, value, type);
1258+
return emitUncheckedValueCast(loc, value, type);
12591259
}

‎lib/SILGen/SILGenConcurrency.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -425,8 +425,8 @@ emitNonOptionalActorInstanceIsolation(SILGenFunction &SGF, SILLocation loc,
425425
if (actor.getType() == anyActorTy)
426426
return actor;
427427

428-
if (actor.getType() == SILType::getPrimitiveObjectType(
429-
SGF.getASTContext().TheImplicitIsolationActorType))
428+
if (actor.getType() ==
429+
SILType::getBuiltinImplicitIsolationActorType(SGF.getASTContext()))
430430
return actor;
431431

432432
CanType anyActorType = anyActorTy.getASTType();

‎test/Concurrency/isolated_nonsending_isolation_macro_ir.swift

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// RUN: %target-swift-frontend -parse-as-library -emit-ir -disable-llvm-merge-functions-pass %s | %FileCheck --check-prefix=NO-TBI %s
2+
// RUN: %target-swift-frontend -parse-as-library -Xllvm -aarch64-use-tbi -emit-ir -disable-llvm-merge-functions-pass %s | %FileCheck --check-prefix=TBI %s
3+
4+
// This test makes sure that we can properly fold the mask for the witness table
5+
// when we have a #isolation.
6+
7+
// REQUIRES: concurrency
8+
// REQUIRES: CODEGENERATOR=AArch64
9+
// REQUIRES: PTRSIZE=64
10+
// REQUIRES: OS=macosx || OS=ios
11+
// REQUIRES: CPU=arm64
12+
13+
@inline(never)
14+
func useActor(iso: (any Actor)?) {
15+
print(iso!.unownedExecutor)
16+
}
17+
18+
@inline(never)
19+
func implicitParam(_ x: (any Actor)? = #isolation) {
20+
print(x!.unownedExecutor)
21+
}
22+
23+
// #isolation via direct usage
24+
//
25+
// TBI-LABEL: define internal swifttailcc void @"$s41isolated_nonsending_isolation_macro_ir_6446nonisolatedNonsendingUsePoundIsolationDirectlyyyYaFTY0_"(
26+
// TBI: [[MASK:%.*]] = and i64 {{%.*}}, -3458764513820540929
27+
// TBI: [[MEM_ADJUSTED:%.*]] = getelementptr inbounds <{ i64, i64 }>, ptr [[MEM:%.*]], i32 0, i32 1
28+
// TBI: store i64 [[MASK]], ptr [[MEM_ADJUSTED]]
29+
// TBI: [[MEM_ADJUSTED_2:%.*]] = getelementptr inbounds { i64, i64 }, ptr [[MEM]], i32 0, i32 1
30+
// TBI: [[MEM_LOAD:%.*]] = load i64, ptr [[MEM_ADJUSTED_2]]
31+
// TBI: call swiftcc void @"$s41isolated_nonsending_isolation_macro_ir_648useActor3isoyScA_pSg_tF"(i64 {{%.*}}, i64 [[MEM_LOAD]])
32+
33+
// NO-TBI-LABEL: define internal swifttailcc void @"$s41isolated_nonsending_isolation_macro_ir_6446nonisolatedNonsendingUsePoundIsolationDirectlyyyYaFTY0_"(
34+
// NO-TBI: [[MASK:%.*]] = and i64 {{%.*}}, -4
35+
// NO-TBI: [[MEM_ADJUSTED:%.*]] = getelementptr inbounds <{ i64, i64 }>, ptr [[MEM:%.*]], i32 0, i32 1
36+
// NO-TBI: store i64 [[MASK]], ptr [[MEM_ADJUSTED]]
37+
// NO-TBI: [[MEM_ADJUSTED_2:%.*]] = getelementptr inbounds { i64, i64 }, ptr [[MEM]], i32 0, i32 1
38+
// NO-TBI: [[MEM_LOAD:%.*]] = load i64, ptr [[MEM_ADJUSTED_2]]
39+
// NO-TBI: call swiftcc void @"$s41isolated_nonsending_isolation_macro_ir_648useActor3isoyScA_pSg_tF"(i64 {{%.*}}, i64 [[MEM_LOAD]])
40+
public nonisolated(nonsending) func nonisolatedNonsendingUsePoundIsolationDirectly() async {
41+
let iso = #isolation
42+
useActor(iso: iso)
43+
}
44+
45+
// #isolation via default arg
46+
//
47+
// TBI-LABEL: define internal swifttailcc void @"$s41isolated_nonsending_isolation_macro_ir_6445nonisolatedNonsendingPoundIsolationDefaultArgyyYaFTY0_"(
48+
// TBI: [[MASK:%.*]] = and i64 {{%.*}}, -3458764513820540929
49+
// TBI: [[MEM_ADJUSTED:%.*]] = getelementptr inbounds <{ i64, i64 }>, ptr [[MEM:%.*]], i32 0, i32 1
50+
// TBI: store i64 [[MASK]], ptr [[MEM_ADJUSTED]]
51+
// TBI: [[MEM_ADJUSTED_2:%.*]] = getelementptr inbounds { i64, i64 }, ptr [[MEM]], i32 0, i32 1
52+
// TBI: [[MEM_LOAD:%.*]] = load i64, ptr [[MEM_ADJUSTED_2]]
53+
// TBI: call swiftcc void @"$s41isolated_nonsending_isolation_macro_ir_6413implicitParamyyScA_pSgF"(i64 {{%.*}}, i64 [[MEM_LOAD]])
54+
55+
// NO-TBI-LABEL: define internal swifttailcc void @"$s41isolated_nonsending_isolation_macro_ir_6445nonisolatedNonsendingPoundIsolationDefaultArgyyYaFTY0_"(
56+
// NO-TBI: [[MASK:%.*]] = and i64 {{%.*}}, -4
57+
// NO-TBI: [[MEM_ADJUSTED:%.*]] = getelementptr inbounds <{ i64, i64 }>, ptr [[MEM:%.*]], i32 0, i32 1
58+
// NO-TBI: store i64 [[MASK]], ptr [[MEM_ADJUSTED]]
59+
// NO-TBI: [[MEM_ADJUSTED_2:%.*]] = getelementptr inbounds { i64, i64 }, ptr [[MEM]], i32 0, i32 1
60+
// NO-TBI: [[MEM_LOAD:%.*]] = load i64, ptr [[MEM_ADJUSTED_2]]
61+
// NO-TBI: call swiftcc void @"$s41isolated_nonsending_isolation_macro_ir_6413implicitParamyyScA_pSgF"(i64 {{%.*}}, i64 [[MEM_LOAD]])
62+
public nonisolated(nonsending) func nonisolatedNonsendingPoundIsolationDefaultArg() async {
63+
implicitParam()
64+
}
65+
66+
@inline(never)
67+
public nonisolated(nonsending) func calleeFunction() async {
68+
}
69+
70+
// TBI-LABEL: define internal swifttailcc void @"$s41isolated_nonsending_isolation_macro_ir_6414callerFunctionyyYaFTY0_"(ptr swiftasync %0)
71+
// TBI: [[MASKED_POINTER:%.*]] = and i64 {{%.*}}, -3458764513820540929
72+
// TBI: musttail call swifttailcc void @"$s41isolated_nonsending_isolation_macro_ir_6414calleeFunctionyyYaF"(ptr swiftasync {{%.*}}, i64 {{%.*}}, i64 [[MASKED_POINTER]])
73+
// TBI-NEXT: ret void
74+
// TBI-NEXT: }
75+
76+
// NO-TBI-LABEL: define internal swifttailcc void @"$s41isolated_nonsending_isolation_macro_ir_6414callerFunctionyyYaFTY0_"(ptr swiftasync %0)
77+
// NO-TBI: [[MASKED_POINTER:%.*]] = and i64 {{%.*}}, -4
78+
// NO-TBI: musttail call swifttailcc void @"$s41isolated_nonsending_isolation_macro_ir_6414calleeFunctionyyYaF"(ptr swiftasync {{%.*}}, i64 {{%.*}}, i64 [[MASKED_POINTER]])
79+
// NO-TBI-NEXT: ret void
80+
// NO-TBI-NEXT: }
81+
@inline(never)
82+
public nonisolated(nonsending) func callerFunction() async {
83+
await calleeFunction()
84+
}

0 commit comments

Comments
 (0)