[cxx-interop] Allow initializing std::function from Swift capturing closures

This introduces support for converting a Swift closure that captures variables from its surrounding context into an instance of `std::function`, which is useful for working with C++ APIs that use callbacks.

Each instantiation of `std::function` gets a synthesized Swift constructor that takes a Swift closure. Unlike the previous implementation, the closure is _not_ marked as `@convention(c)`. The body of the constructor is created lazily.

Under the hood, the closure is bitcast to a pair of a function pointer and a context pointer, which are then wrapped in a C++ object, `__SwiftFunctionWrapper`, that manages the lifetime of the context object via calls to `swift_retain`/`swift_release` from the copy constructor and the destructor. The `__SwiftFunctionWrapper` class is templated, and is instantiated by ClangImporter.

rdar://133777029

Author: egorzhdan

include/swift/AST/ClangModuleLoader.h

@@ -298,6 +298,24 @@ class ClangModuleLoader : public ModuleLoader {
 
   virtual FuncDecl *getDefaultArgGenerator(const clang::ParmVarDecl *param) = 0;
 
+  /// Determine whether this is a functional C++ type, e.g. std::function, for
+  /// which Swift provides a synthesized constructor that takes a Swift closure
+  /// as the single parameter.
+  virtual bool
+  needsClosureConstructor(const clang::CXXRecordDecl *recordDecl) const = 0;
+
+  /// Determine whether this is an instantiation of the __SwiftFunctionWrapper
+  /// type, which wraps around a Swift closure along with its context.
+  virtual bool isSwiftFunctionWrapper(const clang::RecordDecl *decl) const = 0;
+  virtual bool isDeconstructedSwiftClosure(const clang::Type* type) const = 0;
+
+  /// Given a functional C++ type, e.g. std::function, determine the
+  /// corresponding C++ closure type.
+  ///
+  /// \see needsClosureConstructor
+  virtual const clang::FunctionType *extractCXXFunctionType(
+      const clang::CXXRecordDecl *functionalTypeDecl) const = 0;
+
   virtual FuncDecl *
   getAvailabilityDomainPredicate(const clang::VarDecl *var) = 0;
 

include/swift/AST/Types.h

@@ -54,6 +54,10 @@ namespace llvm {
 struct fltSemantics;
 } // namespace llvm
 
+namespace clang {
+class FunctionType;
+} // namespace clang
+
 namespace swift {
 
 enum class AllocationArena;
@@ -4101,6 +4105,10 @@ class FunctionType final
     return getLifetimeDependenceFor(getNumParams());
   }
 
+  uint16_t
+  getPointerAuthDiscriminator(ModuleDecl &m,
+                              const clang::FunctionType *clangType = nullptr);
+
   void Profile(llvm::FoldingSetNodeID &ID) {
     std::optional<ExtInfo> info = std::nullopt;
     if (hasExtInfo())

include/swift/ClangImporter/ClangImporter.h

@@ -620,6 +620,15 @@ class ClangImporter final : public ClangModuleLoader {
 
   FuncDecl *getDefaultArgGenerator(const clang::ParmVarDecl *param) override;
 
+  bool needsClosureConstructor(
+      const clang::CXXRecordDecl *recordDecl) const override;
+
+  bool isSwiftFunctionWrapper(const clang::RecordDecl *decl) const override;
+  bool isDeconstructedSwiftClosure(const clang::Type *type) const override;
+
+  const clang::FunctionType *extractCXXFunctionType(
+      const clang::CXXRecordDecl *functionalTypeDecl) const override;
+
   FuncDecl *getAvailabilityDomainPredicate(const clang::VarDecl *var) override;
 
   bool isAnnotatedWith(const clang::CXXMethodDecl *method, StringRef attr);

include/swift/SIL/AbstractionPattern.h

@@ -28,6 +28,7 @@ namespace llvm {
 
 namespace clang {
   class CXXMethodDecl;
+  class CXXRecordDecl;
   class ObjCMethodDecl;
   class Type;
   class ValueDecl;
@@ -198,6 +199,10 @@ class AbstractionPattern {
     /// non-static member function. OrigType is valid and is a function type.
     /// CXXMethod is valid.
     PartialCurriedCXXMethodType,
+    /// The type of a constructor that initializes a C++ functional type, e.g.
+    /// std::function, with a Swift closure. This constructor is synthesized by
+    /// ClangImporter. ClangType is valid.
+    CXXFunctionalConstructorType,
     /// A Swift function whose parameters and results are opaque. This is
     /// like `AP::Type<T>((T) -> T)`, except that the number of parameters is
     /// unspecified.
@@ -462,6 +467,7 @@ class AbstractionPattern {
     case Kind::CFunctionAsMethodType:
     case Kind::CurriedCFunctionAsMethodType:
     case Kind::PartialCurriedCFunctionAsMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::ObjCCompletionHandlerArgumentsType:
       return true;
 
@@ -632,6 +638,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::ObjCCompletionHandlerArgumentsType:
       return true;
     case Kind::Invalid:
@@ -766,6 +773,13 @@ class AbstractionPattern {
     return pattern;
   }
 
+  /// Return an abstraction pattern for a constructor of a functional C++ type,
+  /// e.g. std::function, which takes a Swift closure as a single parameter.
+  /// This constructor was synthesized by ClangImporter.
+  static AbstractionPattern
+  getCXXFunctionalConstructor(CanType origType,
+                              const clang::CXXRecordDecl *functionalTypeDecl);
+
   /// For a C-function-as-method pattern,
   /// get the index of the C function parameter that was imported as the
   /// `self` parameter of the imported method, or None if this is a static
@@ -1048,6 +1062,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::Type:
     case Kind::Discard:
       return OrigType;
@@ -1084,6 +1099,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::Type:
     case Kind::Discard:
     case Kind::ObjCCompletionHandlerArgumentsType:
@@ -1131,6 +1147,7 @@ class AbstractionPattern {
     case Kind::CFunctionAsMethodType:
     case Kind::CurriedCFunctionAsMethodType:
     case Kind::PartialCurriedCFunctionAsMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
@@ -1148,7 +1165,8 @@ class AbstractionPattern {
   /// Return whether this abstraction pattern represents a Clang type.
   /// If so, it is legal to return getClangType().
   bool isClangType() const {
-    return (getKind() == Kind::ClangType);
+    return getKind() == Kind::ClangType ||
+           getKind() == Kind::CXXFunctionalConstructorType;
   }
 
   const clang::Type *getClangType() const {
@@ -1211,6 +1229,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::OpaqueFunction:
     case Kind::OpaqueDerivativeFunction:
     case Kind::ObjCCompletionHandlerArgumentsType:
@@ -1243,6 +1262,7 @@ class AbstractionPattern {
     case Kind::CFunctionAsMethodType:
     case Kind::CurriedCFunctionAsMethodType:
     case Kind::PartialCurriedCFunctionAsMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
@@ -1275,6 +1295,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::OpaqueFunction:
     case Kind::OpaqueDerivativeFunction:
     case Kind::ObjCCompletionHandlerArgumentsType:
@@ -1306,6 +1327,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::OpaqueFunction:
     case Kind::OpaqueDerivativeFunction:
       return false;
@@ -1334,6 +1356,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::OpaqueFunction:
     case Kind::OpaqueDerivativeFunction:
       llvm_unreachable("pattern is not a tuple");      
@@ -1414,6 +1437,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::OpaqueFunction:
     case Kind::OpaqueDerivativeFunction:
     case Kind::ObjCCompletionHandlerArgumentsType:
@@ -1441,6 +1465,7 @@ class AbstractionPattern {
     case Kind::CXXMethodType:
     case Kind::CurriedCXXMethodType:
     case Kind::PartialCurriedCXXMethodType:
+    case Kind::CXXFunctionalConstructorType:
     case Kind::OpaqueFunction:
     case Kind::OpaqueDerivativeFunction:
     case Kind::ObjCCompletionHandlerArgumentsType:

include/swift/Strings.h

@@ -37,6 +37,8 @@ constexpr static const StringLiteral SWIFT_STRING_PROCESSING_NAME = "_StringProc
 constexpr static const StringLiteral SWIFT_SHIMS_NAME = "SwiftShims";
 /// The name of the CxxShim module, which contains a cxx casting utility.
 constexpr static const StringLiteral CXX_SHIM_NAME = "CxxShim";
+/// The name of the CxxStdlibShim module, which contains utilities for the C++ stdlib overlay.
+constexpr static const StringLiteral CXX_STDLIB_SHIM_NAME = "CxxStdlibShim";
 /// The name of the Cxx module, which contains C++ interop helper protocols.
 constexpr static const StringLiteral CXX_MODULE_NAME = "Cxx";
 /// The name of the Builtin module, which contains Builtin functions.

lib/ClangImporter/ClangDerivedConformances.cpp

@@ -1180,94 +1180,6 @@ void swift::conformToCxxVectorIfNeeded(ClangImporter::Implementation &impl,
   impl.addSynthesizedProtocolAttrs(decl, {KnownProtocolKind::CxxVector});
 }
 
-void swift::conformToCxxFunctionIfNeeded(
-    ClangImporter::Implementation &impl, NominalTypeDecl *decl,
-    const clang::CXXRecordDecl *clangDecl) {
-  PrettyStackTraceDecl trace("conforming to CxxFunction", decl);
-
-  assert(decl);
-  assert(clangDecl);
-  ASTContext &ctx = decl->getASTContext();
-  clang::ASTContext &clangCtx = impl.getClangASTContext();
-  clang::Sema &clangSema = impl.getClangSema();
-
-  // Only auto-conform types from the C++ standard library. Custom user types
-  // might have a similar interface but different semantics.
-  if (!isStdDecl(clangDecl, {"function"}))
-    return;
-
-  // There is no typealias for the argument types on the C++ side, so to
-  // retrieve the argument types we look at the overload of `operator()` that
-  // got imported into Swift.
-
-  auto callAsFunctionDecl = lookupDirectSingleWithoutExtensions<FuncDecl>(
-      decl, ctx.getIdentifier("callAsFunction"));
-  if (!callAsFunctionDecl)
-    return;
-
-  auto operatorCallDecl = dyn_cast_or_null<clang::CXXMethodDecl>(
-      callAsFunctionDecl->getClangDecl());
-  if (!operatorCallDecl)
-    return;
-
-  std::vector<clang::QualType> operatorCallParamTypes;
-  llvm::transform(
-      operatorCallDecl->parameters(),
-      std::back_inserter(operatorCallParamTypes),
-      [](const clang::ParmVarDecl *paramDecl) { return paramDecl->getType(); });
-
-  auto funcPointerType = clangCtx.getPointerType(clangCtx.getFunctionType(
-      operatorCallDecl->getReturnType(), operatorCallParamTypes,
-      clang::FunctionProtoType::ExtProtoInfo())).withConst();
-
-  // Create a fake variable with a function type that matches the type of
-  // `operator()`.
-  auto fakeFuncPointerVarDecl = clang::VarDecl::Create(
-      clangCtx, /*DC*/ clangCtx.getTranslationUnitDecl(),
-      clang::SourceLocation(), clang::SourceLocation(), /*Id*/ nullptr,
-      funcPointerType, clangCtx.getTrivialTypeSourceInfo(funcPointerType),
-      clang::StorageClass::SC_None);
-  auto fakeFuncPointerRefExpr = new (clangCtx) clang::DeclRefExpr(
-      clangCtx, fakeFuncPointerVarDecl, false, funcPointerType,
-      clang::ExprValueKind::VK_LValue, clang::SourceLocation());
-
-  auto clangDeclTyInfo = clangCtx.getTrivialTypeSourceInfo(
-      clang::QualType(clangDecl->getTypeForDecl(), 0));
-  SmallVector<clang::Expr *, 1> constructExprArgs = {fakeFuncPointerRefExpr};
-
-  // Instantiate the templated constructor that would accept this fake variable.
-  auto constructExprResult = clangSema.BuildCXXTypeConstructExpr(
-      clangDeclTyInfo, clangDecl->getLocation(), constructExprArgs,
-      clangDecl->getLocation(), /*ListInitialization*/ false);
-  if (!constructExprResult.isUsable())
-    return;
-
-  auto castExpr = dyn_cast_or_null<clang::CastExpr>(constructExprResult.get());
-  if (!castExpr)
-    return;
-
-  auto bindTempExpr =
-      dyn_cast_or_null<clang::CXXBindTemporaryExpr>(castExpr->getSubExpr());
-  if (!bindTempExpr)
-    return;
-
-  auto constructExpr =
-      dyn_cast_or_null<clang::CXXConstructExpr>(bindTempExpr->getSubExpr());
-  if (!constructExpr)
-    return;
-
-  auto constructorDecl = constructExpr->getConstructor();
-
-  auto importedConstructor =
-      impl.importDecl(constructorDecl, impl.CurrentVersion);
-  if (!importedConstructor)
-    return;
-  decl->addMember(importedConstructor);
-
-  // TODO: actually conform to some form of CxxFunction protocol
-
-}
-
 void swift::conformToCxxSpanIfNeeded(ClangImporter::Implementation &impl,
                                      NominalTypeDecl *decl,
                                      const clang::CXXRecordDecl *clangDecl) {

lib/ClangImporter/ClangDerivedConformances.h

@@ -71,12 +71,6 @@ void conformToCxxVectorIfNeeded(ClangImporter::Implementation &impl,
                                 NominalTypeDecl *decl,
                                 const clang::CXXRecordDecl *clangDecl);
 
-/// If the decl is an instantiation of C++ `std::function`, synthesize a
-/// conformance to CxxFunction, which is defined in the Cxx module.
-void conformToCxxFunctionIfNeeded(ClangImporter::Implementation &impl,
-                                  NominalTypeDecl *decl,
-                                  const clang::CXXRecordDecl *clangDecl);
-                                  
 /// If the decl is an instantiation of C++ `std::span`, synthesize a
 /// conformance to CxxSpan, which is defined in the Cxx module.
 void conformToCxxSpanIfNeeded(ClangImporter::Implementation &impl,

lib/ClangImporter/ClangImporter.cpp

@@ -7929,6 +7929,65 @@ ClangImporter::getDefaultArgGenerator(const clang::ParmVarDecl *param) {
   return nullptr;
 }
 
+bool ClangImporter::needsClosureConstructor(
+    const clang::CXXRecordDecl *recordDecl) const {
+  return Impl.needsClosureConstructor(recordDecl);
+}
+
+bool ClangImporter::Implementation::needsClosureConstructor(
+    const clang::CXXRecordDecl *recordDecl) const {
+  // In the future, this should probably be configurable via a Clang attribute.
+  return recordDecl->isInStdNamespace() && recordDecl->getIdentifier() &&
+         recordDecl->getName() == "function";
+}
+
+bool ClangImporter::isSwiftFunctionWrapper(
+    const clang::RecordDecl *decl) const {
+  return Impl.isSwiftFunctionWrapper(decl);
+}
+
+bool ClangImporter::Implementation::isSwiftFunctionWrapper(
+    const clang::RecordDecl *decl) const {
+  return decl->getIdentifier() && decl->getName() == "__SwiftFunctionWrapper";
+}
+
+bool ClangImporter::isDeconstructedSwiftClosure(const clang::Type *type) const {
+  auto recordDecl = type->getAsCXXRecordDecl();
+  return recordDecl && recordDecl->getIdentifier() &&
+         recordDecl->getName() == "__swift_interop_closure";
+}
+
+const clang::FunctionType *ClangImporter::extractCXXFunctionType(
+    const clang::CXXRecordDecl *functionalTypeDecl) const {
+  auto &clangCtx = functionalTypeDecl->getASTContext();
+  auto operatorCallName = clangCtx.DeclarationNames.getCXXOperatorName(
+      clang::OverloadedOperatorKind::OO_Call);
+  auto lookupResult = functionalTypeDecl->lookup(operatorCallName);
+  const clang::CXXMethodDecl *operatorCall = nullptr;
+  // If an overload of operator() was found, assert that it's a single one.
+  if (!lookupResult.empty()) {
+    ASSERT(lookupResult.isSingleResult() &&
+           "expected single operator() in a functional type");
+    operatorCall = cast<clang::CXXMethodDecl>(lookupResult.front());
+  } else {
+    // If no overload if operator() was found, there could be a viable overload
+    // in one of the base types. For instance, std::function in Microsoft STL
+    // exposes an operator() from a base type.
+    functionalTypeDecl->forallBases([&](const clang::CXXRecordDecl *base) {
+      auto baseResult = base->lookup(operatorCallName);
+      if (!baseResult.empty()) {
+        ASSERT(!operatorCall && "expected single operator() across base types");
+        ASSERT(baseResult.isSingleResult() &&
+               "expected single operator() in a base type");
+        operatorCall = cast<clang::CXXMethodDecl>(baseResult.front());
+      }
+      return true;
+    });
+    ASSERT(operatorCall && "expected operator() in one of the bases");
+  }
+  return operatorCall->getType()->getAs<clang::FunctionType>();
+}
+
 FuncDecl *
 ClangImporter::getAvailabilityDomainPredicate(const clang::VarDecl *var) {
   auto it = Impl.availabilityDomainPredicates.find(var);