Merge pull request swiftlang#81859 from swiftlang/swiftify-inherit-imports

[MacrosOnImports][Swiftify] Copy module imports from clang node's module to its Swift macro SourceFile

Author: hnrklssn

include/swift/AST/Import.h

@@ -457,6 +457,20 @@ class ImportPath : public detail::ImportPathBase<ImportPath> {
   Access getAccessPath(ImportKind importKind) const {
     return getAccessPath(isScopedImportKind(importKind));
   }
+
+private:
+  struct UnsafePrivateConstructorTag {};
+
+  // Doesn't require a module name like the public constructor.
+  // Only used for getEmptyKey() and getTombstoneKey().
+  ImportPath(Raw raw, UnsafePrivateConstructorTag tag) : ImportPathBase(raw) {}
+public:
+  static ImportPath getEmptyKey() {
+    return swift::ImportPath(llvm::DenseMapInfo<Raw>::getEmptyKey(), UnsafePrivateConstructorTag{});
+  }
+  static ImportPath getTombstoneKey() {
+    return swift::ImportPath(llvm::DenseMapInfo<Raw>::getTombstoneKey(), UnsafePrivateConstructorTag{});
+  }
 };
 
 // MARK: - Abstractions of imports
@@ -810,6 +824,27 @@ struct DenseMapInfo<swift::AttributedImport<ModuleInfo>> {
            a.accessLevelRange == b.accessLevelRange;
   }
 };
+
+template <>
+class DenseMapInfo<swift::ImportPath> {
+  using ImportPath = swift::ImportPath;
+public:
+  static ImportPath getEmptyKey() {
+    return swift::ImportPath::getEmptyKey();
+  }
+  static ImportPath getTombstoneKey() {
+    return swift::ImportPath::getTombstoneKey();
+  }
+
+  static unsigned getHashValue(const ImportPath &val) {
+    return llvm::DenseMapInfo<ImportPath::Raw>::getHashValue(val.getRaw());
+  }
+
+  static bool isEqual(const ImportPath &lhs,
+                      const ImportPath &rhs) {
+    return lhs == rhs;
+  }
+};
 }
 
 #endif

include/swift/Basic/LangOptions.h

@@ -648,6 +648,9 @@ namespace swift {
     /// Enables dumping macro expansions.
     bool DumpMacroExpansions = false;
 
+    /// Enables dumping imports for each SourceFile.
+    bool DumpSourceFileImports = false;
+
     /// The model of concurrency to be used.
     ConcurrencyModel ActiveConcurrencyModel = ConcurrencyModel::Standard;
 

include/swift/Basic/Located.h

@@ -71,7 +71,7 @@ struct DenseMapInfo<swift::Located<T>> {
   }
 
   static unsigned getHashValue(const swift::Located<T> &LocatedVal) {
-    return combineHashValue(DenseMapInfo<T>::getHashValue(LocatedVal.Item),
+    return detail::combineHashValue(DenseMapInfo<T>::getHashValue(LocatedVal.Item),
                             DenseMapInfo<swift::SourceLoc>::getHashValue(LocatedVal.Loc));
   }
 
@@ -82,4 +82,12 @@ struct DenseMapInfo<swift::Located<T>> {
 };
 } // namespace llvm
 
+namespace swift {
+  template<typename T>
+  llvm::hash_code hash_value(const Located<T> &LocatedVal) {
+    return llvm::DenseMapInfo<Located<T>>::getHashValue(LocatedVal);
+  }
+} // namespace swift
+
+
 #endif // SWIFT_BASIC_LOCATED_H

include/swift/Option/FrontendOptions.td

@@ -433,6 +433,9 @@ def dump_macro_expansions : Flag<["-"], "dump-macro-expansions">,
 def emit_macro_expansion_files : Separate<["-"], "emit-macro-expansion-files">,
   HelpText<"Specify when to emit macro expansion file: 'none', 'debug', or 'diagnostics'">;
 
+def dump_source_file_imports : Flag<["-"], "dump-source-file-imports">,
+  HelpText<"Dumps the list of imports for each source file">;
+
 def analyze_request_evaluator : Flag<["-"], "analyze-request-evaluator">,
   Flags<[FrontendOption, HelpHidden, DoesNotAffectIncrementalBuild]>,
   HelpText<"Print out request evaluator cache statistics at the end of the compilation job">;

include/swift/Subsystems.h

@@ -126,8 +126,7 @@ namespace swift {
   /// Resolve imports for a source file generated to adapt a given
   /// Clang module.
   void performImportResolutionForClangMacroBuffer(
-    SourceFile &SF, ModuleDecl *clangModule
-  );
+      SourceFile &SF, ModuleDecl *explicitOriginModule);
 
   /// Once type-checking is complete, this instruments code with calls to an
   /// intrinsic that record the expected values of local variables so they can

lib/AST/Module.cpp

@@ -2689,6 +2689,11 @@ void
 SourceFile::setImports(ArrayRef<AttributedImport<ImportedModule>> imports) {
   assert(!Imports && "Already computed imports");
   Imports = getASTContext().AllocateCopy(imports);
+  if (getASTContext().LangOpts.DumpSourceFileImports) {
+    llvm::errs() << "imports for " << getFilename() << ":\n";
+    for (auto Import : imports)
+      llvm::errs() << "\t" << Import.module.importedModule->getName() << "\n";
+  }
 }
 
 std::optional<AttributedImport<ImportedModule>>

lib/ClangImporter/ClangImporter.cpp

@@ -2834,6 +2834,74 @@ ClangModuleUnit *ClangImporter::Implementation::getWrapperForModule(
   if (auto mainModule = SwiftContext.MainModule) {
     implicitImportInfo = mainModule->getImplicitImportInfo();
   }
+
+  if (!underlying->isSubModule()) {
+    // Make sure that synthesized Swift code in the clang module wrapper
+    // (e.g. _SwiftifyImport macro expansions) can access the same symbols
+    // as if it were actually in the clang module, by copying the imports.
+    // Because this top-level module wrapper contains all the imported decls
+    // of the clang submodules, we need to add the imports of all the
+    // transitive submodules, since we don't know at this point of the
+    // compilation which submodules will contain relevant macros.
+    // We also need to add (transitive) explicit submodules as imports,
+    // to make sure that they are marked as imported *somewhere* (clang modules
+    // including them don't count) - otherwise their decls won't be found after
+    // non-visible clang decls are filtered out.
+    llvm::SmallVector<const clang::Module *, 32> SubmoduleWorklist;
+    llvm::DenseSet<ImportPath> Imported;
+    SubmoduleWorklist.push_back(underlying);
+    ImportPath::Builder underlyingSwiftModulePath =
+        getSwiftModulePath(underlying);
+    Imported.insert(underlyingSwiftModulePath.get());
+    for (auto UI : implicitImportInfo.AdditionalUnloadedImports)
+      Imported.insert(UI.module.getImportPath());
+    assert(implicitImportInfo.AdditionalImports.empty());
+
+    auto addImplicitImport = [&implicitImportInfo, &Imported,
+                              this](const clang::Module *M,
+                                    bool guaranteedUnique) {
+      ImportPath::Builder builder = getSwiftModulePath(M);
+      if (!guaranteedUnique && Imported.count(builder.get()))
+        return;
+
+      // Don't perform this clone for modules already added to the list
+      ImportPath importedModulePath = builder.copyTo(SwiftContext);
+
+#ifndef NDEBUG
+      const bool performSanityCheck = true;
+#else
+      const bool performSanityCheck = false;
+#endif
+      if (!guaranteedUnique || performSanityCheck) {
+        bool WasInserted = Imported.insert(importedModulePath).second;
+        assert(WasInserted);
+      }
+
+      UnloadedImportedModule importedModule(importedModulePath,
+                                            ImportKind::Module);
+      implicitImportInfo.AdditionalUnloadedImports.push_back(
+          std::move(importedModule));
+    };
+
+    while (!SubmoduleWorklist.empty()) {
+      const clang::Module *CurrModule = SubmoduleWorklist.pop_back_val();
+      if (CurrModule->IsExplicit) {
+        // We don't add imports under the same TLM, and submodules form
+        // a tree, so these don't require deduplication.
+        addImplicitImport(CurrModule, /*guaranteedUnique=*/true);
+      }
+      for (auto *I : CurrModule->Imports) {
+        // `underlying` is the current TLM. Only explicit submodules need to
+        // be imported under the same TLM, which is handled above.
+        if (I->getTopLevelModule() == underlying)
+          continue;
+        addImplicitImport(I, /*guaranteedUnique=*/false);
+      }
+      for (auto *Submodule : CurrModule->submodules())
+        SubmoduleWorklist.push_back(Submodule);
+    }
+  }
+
   ClangModuleUnit *file = nullptr;
   auto wrapper = ModuleDecl::create(name, SwiftContext, implicitImportInfo,
                                     [&](ModuleDecl *wrapper, auto addFile) {
@@ -3742,7 +3810,11 @@ ImportDecl *swift::createImportDecl(ASTContext &Ctx,
   while (TmpMod) {
     // If this is a C++ stdlib module, print its name as `CxxStdlib` instead of
     // `std`. `CxxStdlib` is the only accepted spelling of the C++ stdlib module
-    // name in Swift.
+    // name in Swift. In libc++ versions 17-19 there are multiple TLMs, named
+    // std_vector, std_array etc. We don't support importing those modules, but
+    // when printing the module interface it'd be weird to print "import
+    // CxxStdlib" over and over, so those are still printed as "import
+    // std_vector". This only affects the module interface for CxxStdlib.
     Identifier moduleName = !TmpMod->isSubModule() && TmpMod->Name == "std"
                                 ? Ctx.Id_CxxStdlib
                                 : Ctx.getIdentifier(TmpMod->Name);
@@ -4597,8 +4669,8 @@ void ClangModuleUnit::getImportedModulesForLookup(
     if (owner.SwiftContext.LangOpts.EnableCXXInterop && topLevel &&
         isCxxStdModule(topLevel) && wrapper->clangModule &&
         isCxxStdModule(wrapper->clangModule)) {
-      // The CxxStdlib overlay re-exports the clang module std, which in recent
-      // libc++ versions re-exports top-level modules for different std headers
+      // The CxxStdlib overlay re-exports the clang module std, which in libc++
+      // versions 17-19 re-exports top-level modules for different std headers
       // (std_string, std_vector, etc). The overlay module for each of the std
       // modules is the CxxStdlib module itself. Make sure we return the actual
       // clang modules (std_xyz) as transitive dependencies instead of just
@@ -8663,7 +8735,7 @@ bool importer::isCxxStdModule(const clang::Module *module) {
 bool importer::isCxxStdModule(StringRef moduleName, bool IsSystem) {
   if (moduleName == "std")
     return true;
-  // In recent libc++ versions the module is split into multiple top-level
+  // In libc++ versions 17-19 the module is split into multiple top-level
   // modules (std_vector, std_utility, etc).
   if (IsSystem && moduleName.starts_with("std_")) {
     if (moduleName == "std_errno_h")
@@ -8673,6 +8745,19 @@ bool importer::isCxxStdModule(StringRef moduleName, bool IsSystem) {
   return false;
 }
 
+ImportPath::Builder ClangImporter::Implementation::getSwiftModulePath(const clang::Module *M) {
+  ImportPath::Builder builder;
+  while (M) {
+    if (!M->isSubModule() && isCxxStdModule(M))
+      builder.push_back(SwiftContext.Id_CxxStdlib);
+    else
+      builder.push_back(SwiftContext.getIdentifier(M->Name));
+    M = M->Parent;
+  }
+  std::reverse(builder.begin(), builder.end());
+  return builder;
+}
+
 std::optional<clang::QualType>
 importer::getCxxReferencePointeeTypeOrNone(const clang::Type *type) {
   if (type->isReferenceType())

lib/ClangImporter/ImporterImpl.h

@@ -1278,6 +1278,8 @@ class LLVM_LIBRARY_VISIBILITY ClangImporter::Implementation
   /// \returns The named module, or null if the module has not been imported.
   ModuleDecl *getNamedModule(StringRef name);
 
+  ImportPath::Builder getSwiftModulePath(const clang::Module *M);
+
   /// Returns the "Foundation" module, if it can be loaded.
   ///
   /// After this has been called, the Foundation module will or won't be loaded

lib/Frontend/CompilerInvocation.cpp

@@ -1736,6 +1736,9 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.DumpMacroExpansions = Args.hasArg(
       OPT_dump_macro_expansions);
 
+  Opts.DumpSourceFileImports = Args.hasArg(
+      OPT_dump_source_file_imports);
+
   if (const Arg *A = Args.getLastArg(OPT_debug_requirement_machine))
     Opts.DebugRequirementMachine = A->getValue();
 

lib/Sema/ImportResolution.cpp

@@ -163,6 +163,11 @@ class ImportResolver final : public DeclVisitor<ImportResolver> {
   /// The list of fully bound imports.
   SmallVector<AttributedImport<ImportedModule>, 16> boundImports;
 
+  /// Set of imported top-level clang modules. We normally don't expect
+  /// duplicated imports, but importing multiple submodules of the same clang
+  /// TLM would cause the same TLM to be imported once per submodule.
+  SmallPtrSet<const ModuleDecl*, 16> seenClangTLMs;
+
   /// All imported modules which should be considered when cross-importing.
   /// This is basically the transitive import graph, but with only top-level
   /// modules and without reexports from Objective-C modules.
@@ -192,6 +197,14 @@ class ImportResolver final : public DeclVisitor<ImportResolver> {
     addImplicitImports();
   }
 
+  // FIXME: Remove this ctor once namespace contents are imported to their
+  // corresponding module instead of the __ObjC module
+  ImportResolver(SourceFile &SF, ModuleDecl *explicitUnderlyingClangModule)
+      : SF(SF), ctx(SF.getASTContext()),
+        underlyingClangModule(explicitUnderlyingClangModule) {
+    addImplicitImports();
+  }
+
   void addImplicitImports();
 
   void addImplicitImport(ModuleDecl *module) {
@@ -322,26 +335,23 @@ void swift::performImportResolution(SourceFile &SF) {
 }
 
 void swift::performImportResolutionForClangMacroBuffer(
-    SourceFile &SF, ModuleDecl *clangModule
-) {
-  // If we've already performed import resolution, bail.
-  if (SF.ASTStage == SourceFile::ImportsResolved)
-    return;
-
-  ImportResolver resolver(SF);
-  resolver.addImplicitImport(clangModule);
-
-  // FIXME: This is a hack that we shouldn't need, but be sure that we can
-  // see the Swift standard library.
-  if (auto stdlib = SF.getASTContext().getStdlibModule())
-    resolver.addImplicitImport(stdlib);
+    SourceFile &SF, ModuleDecl *explicitOriginModule) {
+  assert(SF.ASTStage == SourceFile::Unprocessed);
 
+  // `getWrapperForModule` has already declared all the implicit clang module
+  // imports we need
+  ImportResolver resolver(SF, explicitOriginModule);
   SF.setImports(resolver.getFinishedImports());
   SF.setImportedUnderlyingModule(resolver.getUnderlyingClangModule());
 
   SF.ASTStage = SourceFile::ImportsResolved;
 }
 
+static bool isSubmodule(const ModuleDecl* M) {
+  auto clangMod = M->findUnderlyingClangModule();
+  return clangMod && clangMod->Parent;
+}
+
 //===----------------------------------------------------------------------===//
 // MARK: Import handling generally
 //===----------------------------------------------------------------------===//
@@ -389,14 +399,23 @@ void ImportResolver::bindImport(UnboundImport &&I) {
 
   I.validateOptions(topLevelModule, SF);
 
-  if (topLevelModule && topLevelModule != M) {
-    // If we have distinct submodule and top-level module, add both.
-    addImport(I, M);
-    addImport(I, topLevelModule.get());
-  }
-  else {
-    // Add only the import itself.
+  auto alreadyImportedTLM = [ID,this](const ModuleDecl *MD) {
+    ASSERT(!isSubmodule(MD));
+    // Scoped imports don't import all symbols from the module, so a scoped
+    // import does not count the module as imported
+    if (ID && isScopedImportKind(ID.get()->getImportKind()))
+      return false;
+    return !seenClangTLMs.insert(MD).second;
+  };
+  if (!M->isNonSwiftModule() || topLevelModule != M || !alreadyImportedTLM(M)) {
     addImport(I, M);
+    if (topLevelModule && topLevelModule != M &&
+        !alreadyImportedTLM(topLevelModule.get())) {
+      // If we have distinct submodule and top-level module, add both.
+      // Importing the submodule ensures that it gets loaded, but the decls
+      // are imported to the TLM, so import that for visibility.
+      addImport(I, topLevelModule.get());
+    }
   }
 
   crossImport(M, I);
@@ -592,7 +611,19 @@ ModuleImplicitImportsRequest::evaluate(Evaluator &evaluator,
 }
 
 void ImportResolver::addImplicitImports() {
-  auto implicitImports = SF.getParentModule()->getImplicitImports();
+  // This is a workaround for the fact that namespaces are imported into the
+  // bridging header module. To allow correct lookup, we expose a ctor with an
+  // explicit underlying clang module input, which is the clang module that this
+  // macro expansion actually originates in.
+  const ModuleDecl *moduleToInherit = nullptr;
+  if (underlyingClangModule) {
+    moduleToInherit = underlyingClangModule;
+    boundImports.push_back(
+        AttributedImport(ImportedModule(underlyingClangModule)));
+  } else {
+    moduleToInherit = SF.getParentModule();
+  }
+  auto implicitImports = moduleToInherit->getImplicitImports();
 
   // TODO: Support cross-module imports.
   for (auto &import : implicitImports.imports) {
@@ -1590,11 +1621,6 @@ void ImportResolver::findCrossImports(
   }
 }
 
-static bool isSubmodule(ModuleDecl* M) {
-  auto clangMod = M->findUnderlyingClangModule();
-  return clangMod && clangMod->Parent;
-}
-
 void ImportResolver::addCrossImportableModules(
     AttributedImport<ImportedModule> importDesc) {
   // FIXME: namelookup::getAllImports() doesn't quite do what we need (mainly