CSHARP-5784: Execute dotnet restore with retries on Evergreen (#1816)

Author: sanych-sun

build.cake

@@ -25,82 +25,14 @@ var artifactsPackagesDirectory = artifactsDirectory.Combine("packages");
 var srcDirectory = solutionDirectory.Combine("src");
 var testsDirectory = solutionDirectory.Combine("tests");
 var outputDirectory = solutionDirectory.Combine("build");
-var toolsDirectory = solutionDirectory.Combine("tools");
-var toolsHugoDirectory = toolsDirectory.Combine("Hugo");
-var mongoDbDriverPackageName = "MongoDB.Driver";
 
 var solutionFile = solutionDirectory.CombineWithFilePath("CSharpDriver.sln");
 var solutionFullPath = solutionFile.FullPath;
 
 Task("Default")
     .IsDependentOn("Test");
 
-Task("Release")
-    .IsDependentOn("Build")
-    .IsDependentOn("Package");
-
-Task("Restore")
-    .Does(() =>
-    {
-        // disable parallel restore to work around apparent bugs in restore
-        var restoreSettings = new DotNetRestoreSettings
-        {
-            DisableParallel = true
-        };
-        DotNetRestore(solutionFullPath, restoreSettings);
-    });
-
-Task("Build")
-    .IsDependentOn("Restore")
-    .Does<BuildConfig>((buildConfig) =>
-    {
-       var settings = new DotNetBuildSettings
-       {
-           NoRestore = true,
-           Configuration = configuration,
-           EnvironmentVariables = new Dictionary<string, string>
-           {
-               { "Version", gitVersion.LegacySemVer },
-               { "SourceRevisionId", gitVersion.Sha }
-           }
-        };
-
-        DotNetBuild(solutionFullPath, settings);
-    });
-
-Task("BuildArtifacts")
-    .IsDependentOn("Build")
-    .Does(() =>
-    {
-        foreach (var targetFramework in new[] { "net472", "netstandard2.0", "netstandard2.1" })
-        {
-            var toDirectory = artifactsBinDirectory.Combine(targetFramework);
-            CleanDirectory(toDirectory);
-
-            var projects = new[] { "MongoDB.Bson", "MongoDB.Driver" };
-            foreach (var project in projects)
-            {
-                var fromDirectory = srcDirectory.Combine(project).Combine("bin").Combine(configuration).Combine(targetFramework);
-
-                var fileNames = new List<string>();
-                foreach (var extension in new[] { "dll", "pdb", "xml" })
-                {
-                    var fileName = $"{project}.{extension}";
-                    fileNames.Add(fileName);
-                }
-
-                foreach (var fileName in fileNames)
-                {
-                    var fromFile = fromDirectory.CombineWithFilePath(fileName);
-                    var toFile = toDirectory.CombineWithFilePath(fileName);
-                    CopyFile(fromFile, toFile);
-                }
-            }
-        }
-    });
-
 Task("Test")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/*.Tests.csproj").Where(name => !name.ToString().Contains("Atlas")),
         action: (BuildConfig buildConfig, Path testProject) =>
@@ -121,68 +53,58 @@ Task("Test")
     .DeferOnError();
 
 Task("TestAwsAuthentication")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
             RunTests(buildConfig, testProject, filter: "Category=\"AwsMechanism\""));
 
 Task("TestPlainAuthentication")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
             RunTests(buildConfig, testProject, filter: "Category=\"PlainMechanism\""));
 
 Task("TestAtlasConnectivity")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/AtlasConnectivity.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) => RunTests(buildConfig, testProject));
 
 Task("TestAtlasSearch")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
            RunTests(buildConfig, testProject, filter: "Category=\"AtlasSearch\""));
 
 Task("TestAtlasSearchIndexHelpers")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
            RunTests(buildConfig, testProject, filter: "Category=\"AtlasSearchIndexHelpers\""));
 
 Task("TestOcsp")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
             RunTests(buildConfig, testProject, filter: "Category=\"OCSP\""));
 
 Task("TestGssapi")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
            RunTests(buildConfig, testProject, filter: "Category=\"GssapiMechanism\""));
 
 Task("TestMongoDbOidc")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
             RunTests(buildConfig, testProject, filter: "Category=\"MongoDbOidc\""));
 
 Task("TestLibMongoCrypt")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Encryption.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) => RunTests(buildConfig, testProject));
 
 Task("TestLoadBalanced")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/*.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
@@ -217,91 +139,17 @@ Task("TestCsfleWithGcpKms")
             RunTests(buildConfig, testProject, filter: "Category=\"CsfleGCPKMS\""));
 
 Task("TestX509")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/MongoDB.Driver.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
             RunTests(buildConfig, testProject, filter: "Category=\"X509\""));
 
 Task("TestSocks5Proxy")
-    .IsDependentOn("Build")
     .DoesForEach(
         items: GetFiles("./**/*.Tests.csproj"),
         action: (BuildConfig buildConfig, Path testProject) =>
             RunTests(buildConfig, testProject, filter: "Category=\"Socks5Proxy\""));
 
-Task("Package")
-    .IsDependentOn("PackageNugetPackages");
-
-Task("PackageNugetPackages")
-    .IsDependentOn("Build")
-    .Does<BuildConfig>((buildConfig) =>
-    {
-        EnsureDirectoryExists(artifactsPackagesDirectory);
-        CleanDirectory(artifactsPackagesDirectory);
-
-        var projects = new[]
-        {
-            "MongoDB.Bson",
-            "MongoDB.Driver",
-            "MongoDB.Driver.Encryption"
-        };
-
-        foreach (var project in projects)
-        {
-            var projectPath = $"{srcDirectory}\\{project}\\{project}.csproj";
-            var settings = new DotNetPackSettings
-            {
-                Configuration = configuration,
-                OutputDirectory = artifactsPackagesDirectory,
-                NoBuild = true, // SetContinuousIntegrationBuild is enabled for nupkg on the Build step
-                IncludeSymbols = true,
-                MSBuildSettings = new DotNetMSBuildSettings()
-                    // configure deterministic build for better compatibility with debug symbols (used in Package/Build tasks). Affects: *.snupkg
-                    .SetContinuousIntegrationBuild(continuousIntegrationBuild: true)
-                    .WithProperty("PackageVersion", buildConfig.PackageVersion)
-            };
-            DotNetPack(projectPath, settings);
-        }
-    });
-
-Task("PushToNuGet")
-    .Does(() =>
-    {
-        var nugetApiKey = EnvironmentVariable("NUGETAPIKEY");
-        if (nugetApiKey == null)
-        {
-            throw new Exception("NUGETAPIKEY environment variable missing");
-        }
-
-        var packageFiles = new List<FilePath>();
-
-        var projects = new[]
-        {
-            "MongoDB.Bson",
-            "MongoDB.Driver"
-        };
-
-        foreach (var project in projects)
-        {
-            var packageFileName = $"{project}.{gitVersion.LegacySemVer}.nupkg";
-            var packageFile = artifactsPackagesDirectory.CombineWithFilePath(packageFileName);
-            packageFiles.Add(packageFile);
-        }
-
-        NuGetPush(packageFiles, new NuGetPushSettings
-        {
-            ApiKey = nugetApiKey,
-            Source = "https://api.nuget.org/v3/index.json"
-        });
-    });
-
-Task("DumpGitVersion")
-    .Does(() =>
-    {
-        Information(gitVersion.Dump());
-    });
-
 Task("SmokeTests")
     .DoesForEach(
         GetFiles("./**/SmokeTests/**/*.SmokeTests*.csproj"),

evergreen/build-packages.sh

@@ -12,4 +12,5 @@ fi
 echo Creating nuget package...
 
 dotnet clean ./CSharpDriver.sln
-dotnet pack ./CSharpDriver.sln -o ./artifacts/nuget -c Release -p:Version="$PACKAGE_VERSION" --include-symbols -p:SymbolPackageFormat=snupkg -p:ContinuousIntegrationBuild=true
+. ./evergreen/compile-sources.sh
+dotnet pack ./CSharpDriver.sln --no-build -o ./artifacts/nuget -c Release -p:Version="$PACKAGE_VERSION" --include-symbols -p:SymbolPackageFormat=snupkg -p:ContinuousIntegrationBuild=true

evergreen/compile-sources.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+
+if [ -z "$PACKAGE_VERSION" ]; then
+  PACKAGE_VERSION=$(bash ./evergreen/get-version.sh)
+  echo Calculated PACKAGE_VERSION value: "$PACKAGE_VERSION"
+fi
+
+RESTORE_MAX_RETRIES=5
+RESTORE_RETRY_DELAY_SECONDS_MULTIPLIER=10
+
+for (( ATTEMPT=1; ATTEMPT<=RESTORE_MAX_RETRIES; ATTEMPT++ ))
+do
+  echo "Attempt $ATTEMPT of $RESTORE_MAX_RETRIES to run dotnet restore..."
+  exit_status=0
+  dotnet restore || exit_status=$?
+  if [[ "$exit_status" -eq 0 ]]; then
+    echo "dotnet restore succeeded."
+    break
+  fi
+
+  if [[ $ATTEMPT -eq $RESTORE_MAX_RETRIES ]]; then
+    echo "Failed to restore packages after $RESTORE_MAX_RETRIES retries."
+    exit 1
+  fi
+
+  DELAY=$((ATTEMPT * RESTORE_RETRY_DELAY_SECONDS_MULTIPLIER))
+  echo "dotnet restore attempt $ATTEMPT failed. Retrying in $DELAY seconds..."
+  sleep $DELAY
+done
+
+dotnet build -c Release --no-restore -p:Version="$PACKAGE_VERSION"

evergreen/evergreen.yml

@@ -104,13 +104,14 @@ functions:
     - command: shell.exec
       type: system
       params:
+        working_dir: mongo-csharp-driver
         include_expansions_in_env:
           - "OS"
           - "DOTNET_SDK_VERSION"
           - "FRAMEWORK"
         script: |
           ${PREPARE_SHELL}
-          bash ${PROJECT_DIRECTORY}/evergreen/install-dotnet.sh
+          bash evergreen/install-dotnet.sh
 
   prepare-resources:
     - command: shell.exec
@@ -196,40 +197,6 @@ functions:
         content_type: ${content_type|text/plain}
         display_name: "orchestration.log"
 
-  upload-working-dir:
-    - command: archive.targz_pack
-      params:
-        target: "working-dir.tar.gz"
-        source_dir: ${PROJECT_DIRECTORY}/
-        include:
-          - "./**"
-    - command: s3.put
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: working-dir.tar.gz
-        remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz
-        bucket: mciuploads
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: "working-dir.tar.gz"
-    - command: archive.targz_pack
-      params:
-        target: "drivers-dir.tar.gz"
-        source_dir: ${DRIVERS_TOOLS}
-        include:
-          - "./**"
-    - command: s3.put
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: drivers-dir.tar.gz
-        remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz
-        bucket: mciuploads
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: "drivers-dir.tar.gz"
-
   upload-test-results:
     - command: attach.xunit_results
       params:
@@ -376,7 +343,7 @@ functions:
           - "FRAMEWORK"
         script: |
           ${PREPARE_SHELL}
-          bash evergreen/run-unit-tests.sh
+          bash ./evergreen/run-unit-tests.sh
 
   run-tests:
     - command: shell.exec
@@ -614,6 +581,7 @@ functions:
              echo "This platform does not support the ECS auth test, skipping..."
              exit 0
           fi
+          . ./evergreen/compile-sources.sh
           echo "Project Directory: $PROJECT_DIRECTORY"
           # SRC_DIRECTORY is workaround since EG_TOOLS expects "src" folder as a root
           SRC_DIRECTORY=$(dirname $PROJECT_DIRECTORY)/src

evergreen/run-atlas-connectivity-tests.sh

@@ -8,4 +8,5 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 # Provision the correct connection string and set up SSL if needed
 for var in TMP TEMP NUGET_PACKAGES NUGET_HTTP_CACHE_PATH APPDATA; do setx $var z:\\data\\tmp; export $var=z:\\data\\tmp; done
+. ./evergreen/compile-sources.sh
 powershell.exe .\\build.ps1 --target TestAtlasConnectivity

evergreen/run-atlas-search-index-helpers-test.sh

@@ -13,4 +13,5 @@ echo "Running Atlas Search Index Helpers driver tests"
 
 export ATLAS_SEARCH_INDEX_HELPERS_TESTS_ENABLED=true
 
-./build.sh --target=TestAtlasSearchIndexHelpers
+. ./evergreen/compile-sources.sh
+./build.sh --target=TestAtlasSearchIndexHelpers

evergreen/run-atlas-search-test.sh

@@ -13,4 +13,5 @@ echo "Running Atlas Search driver tests"
 
 export ATLAS_SEARCH_TESTS_ENABLED=true
 
-powershell.exe .\\build.ps1 --target=TestAtlasSearch
+. ./evergreen/compile-sources.sh
+powershell.exe .\\build.ps1 --target=TestAtlasSearch

evergreen/run-csfle-azure-tests.sh

@@ -24,4 +24,5 @@ export CSFLE_AZURE_KMS_TESTS_ENABLED=true
 export FRAMEWORK=net6.0
 . ./evergreen/install-dotnet.sh
 
+. ./evergreen/compile-sources.sh
 . ./build.sh --target=TestCsfleWithAzureKms

evergreen/run-csfle-gcp-tests.sh

@@ -17,4 +17,5 @@ export CSFLE_GCP_KMS_TESTS_ENABLED=true
 export FRAMEWORK=net6.0
 . ./evergreen/install-dotnet.sh
 
+. ./evergreen/compile-sources.sh
 . ./build.sh --target=TestCsfleWithGcpKms