C#: Allow dependencies to be stored outside the SCRATCH_DIR.

csharp/codeql-extractor.yml

@@ -74,3 +74,8 @@ options:
       [EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
       This option only works when `--build-mode none` is also specified.
     type: array
+  buildless_dependency_dir:
+    title: The path where buildless (standalone) extraction should keep dependencies.
+    description: >
+      If set, the buildless (standalone) extractor will store dependencies in this directory.
+    type: string

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyDirectory.cs

@@ -0,0 +1,62 @@
+using System;
+using System.IO;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    /// <summary>
+    /// A directory used for storing fetched dependencies.
+    /// When a specific directory is set via the dependency directory extractor option,
+    /// we store dependencies in that directory for caching purposes.
+    /// Otherwise, we create a temporary directory that is deleted upon disposal.
+    /// </summary>
+    public sealed class DependencyDirectory : IDisposable
+    {
+        private readonly string userReportedDirectoryPurpose;
+        private readonly ILogger logger;
+        private readonly bool attemptCleanup;
+
+        public DirectoryInfo DirInfo { get; }
+
+        public DependencyDirectory(string subfolderName, string userReportedDirectoryPurpose, ILogger logger)
+        {
+            this.logger = logger;
+            this.userReportedDirectoryPurpose = userReportedDirectoryPurpose;
+
+            string path;
+            if (EnvironmentVariables.GetBuildlessDependencyDir() is string dir)
+            {
+                path = dir;
+                attemptCleanup = false;
+            }
+            else
+            {
+                path = FileUtils.GetTemporaryWorkingDirectory(out _);
+                attemptCleanup = true;
+            }
+            DirInfo = new DirectoryInfo(Path.Join(path, subfolderName));
+            DirInfo.Create();
+        }
+
+        public void Dispose()
+        {
+            if (!attemptCleanup)
+            {
+                logger.LogInfo($"Keeping {userReportedDirectoryPurpose} directory {DirInfo.FullName} for possible caching purposes.");
+                return;
+            }
+
+            try
+            {
+                DirInfo.Delete(true);
+            }
+            catch (Exception exc)
+            {
+                logger.LogInfo($"Couldn't delete {userReportedDirectoryPurpose} directory {exc.Message}");
+            }
+        }
+
+        public override string ToString() => DirInfo.FullName.ToString();
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -25,15 +25,15 @@ internal class NugetExeWrapper : IDisposable
 
         /// <summary>
         /// The computed packages directory.
-        /// This will be in the Temp location
+        /// This will be in the Cached or Temp location
         /// so as to not trample the source tree.
         /// </summary>
-        private readonly TemporaryDirectory packageDirectory;
+        private readonly DependencyDirectory packageDirectory;
 
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
-        public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
+        public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger)
         {
             this.fileProvider = fileProvider;
             this.packageDirectory = packageDirectory;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -24,12 +24,12 @@ internal sealed partial class NugetPackageRestorer : IDisposable
         private readonly IDotNet dotnet;
         private readonly DependabotProxy? dependabotProxy;
         private readonly IDiagnosticsWriter diagnosticsWriter;
-        private readonly TemporaryDirectory legacyPackageDirectory;
-        private readonly TemporaryDirectory missingPackageDirectory;
+        private readonly DependencyDirectory legacyPackageDirectory;
+        private readonly DependencyDirectory missingPackageDirectory;
         private readonly ILogger logger;
         private readonly ICompilationInfoContainer compilationInfoContainer;
 
-        public TemporaryDirectory PackageDirectory { get; }
+        public DependencyDirectory PackageDirectory { get; }
 
         public NugetPackageRestorer(
             FileProvider fileProvider,
@@ -48,9 +48,9 @@ public NugetPackageRestorer(
             this.logger = logger;
             this.compilationInfoContainer = compilationInfoContainer;
 
-            PackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("packages"), "package", logger);
-            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("legacypackages"), "legacy package", logger);
-            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath("missingpackages"), "missing package", logger);
+            PackageDirectory = new DependencyDirectory("packages", "package", logger);
+            legacyPackageDirectory = new DependencyDirectory("legacypackages", "legacy package", logger);
+            missingPackageDirectory = new DependencyDirectory("missingpackages", "missing package", logger);
         }
 
         public string? TryRestore(string package)

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -76,5 +76,15 @@ public static IEnumerable<string> GetURLs(string name)
         {
             return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_BASE_METADATA_OUT");
         }
+
+        /// <summary>
+        /// If set, returns the directory where buildless dependencies should be stored.
+        /// This is needed for caching dependencies.
+        /// </summary>
+        /// <returns></returns>
+        public static string? GetBuildlessDependencyDir()
+        {
+            return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS_DEPENDENCY_DIR");
+        }
     }
 }