diff --git a/README.md b/README.md index 95f0a7b714..33f9ff4965 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Option 1 is recommended if you can use the default worker. #### Drawbacks: - It takes ~10 minutes to build and pack Calamari, however you can reduce this significantly by targeting a specific runtime/framework if you don't need the rest - - eg `./build-local.sh -y --framework "net6.0" --runtime "linux-x64"` (note that consolidation tests will not run when targeting a specific runtime) + - eg `./build-local.sh -y --framework "net8.0" --runtime "linux-x64"` (note that consolidation tests will not run when targeting a specific runtime) - You need to restart Server for Calamari changes to take effect ### Bonus Variables! diff --git a/build-local.sh b/build-local.sh index 527ef87986..eae1900627 100755 --- a/build-local.sh +++ b/build-local.sh @@ -39,7 +39,7 @@ StartMessage="${Green}\ ║ This script is intended to only be run locally and not in CI. ║ ║ ║ ║ If something unexpected is happening in your build or Calamari changes you may want to run ║ -║ the full build by running ./build.ps1 and check again as something in the optimizations here ║ +║ the full build by running ./build.sh and check again as something in the optimizations here ║ ║ ║ ║ might have caused an issue. ║ ╬════════════════════════════════════════════════════════════════════════════════════════════════╬\ @@ -51,7 +51,7 @@ WarningMessage="${Yellow}\ ║ WARNING: ║ ║ Building Calamari on a non-windows machine will result ║ ║ in Calmari and Calamari.Cloud nuget packages being ║ -║ built against net6.0. This means that some ║ +║ built against net8.0. This means that some ║ ║ steps may not work as expected because they require a ║ ║ .Net Framework compatible Calamari Nuget Package. ║ ╬════════════════════════════════════════════════════════╬\ diff --git a/build.ps1 b/build.ps1 index b5950cb493..f667ed6ef8 100644 --- a/build.ps1 +++ b/build.ps1 @@ -55,17 +55,49 @@ else { } } + # ----- Octopus Deploy Modification ----- + # + # The default behaviour of the Nuke Bootstrapper (when .NET is not already preinstalled) is + # to read from the global.json, then install that exact version. It doesn't roll forward. + # This means that if our global.json says 8.0.100, and the latest version is 8.0.200, it will + # always install 8.0.100 and we will not pick up any security or bug fixes that 8.0.200 carries. + # + # This means we would need to manually update our global.json file every time there is a new + # .NET SDK available, and then all developers would need to immediately install this on their machines. + # + # In our builds, we want the same "automatic roll-forward" behaviour that we get when we use the dotnet/sdk:8.0 docker + # images -- where we always get the latest patch version of the SDK without manual intervention. + # + # We achieve this with a small tweak to the Nuke bootstrapper to tell it to install the latest version from + # the 8.0 channel, regardless of what's in the global.json. + + Remove-Variable DotNetVersion + $DotNetChannel = "8.0" + # ----- End Octopus Deploy Modification ----- + # Install by channel or version $DotNetDirectory = "$TempDirectory\dotnet-win" if (!(Test-Path variable:DotNetVersion)) { - ExecSafe { & powershell $DotNetInstallFile -InstallDir $DotNetDirectory -Channel $DotNetChannel -NoPath } + ExecSafe { & powershell $DotNetInstallFile -InstallDir $DotNetDirectory -Channel $DotNetChannel } } else { - ExecSafe { & powershell $DotNetInstallFile -InstallDir $DotNetDirectory -Version $DotNetVersion -NoPath } + ExecSafe { & powershell $DotNetInstallFile -InstallDir $DotNetDirectory -Version $DotNetVersion } } $env:DOTNET_EXE = "$DotNetDirectory\dotnet.exe" + + # ----- Octopus Deploy Modification ----- + # Update the path with the temporary dotnet exe so it can be found by anything be run out of this shell + $env:PATH = "$($env:Path);$DotNetDirectory" + # We update the global path + [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) + Write-Output "Updating Path variable to $($env:PATH)" + # ----- End Octopus Deploy Modification ----- } Write-Output "Microsoft (R) .NET Core SDK version $(& $env:DOTNET_EXE --version)" ExecSafe { & $env:DOTNET_EXE build $BuildProjectFile /nodeReuse:false /p:UseSharedCompilation=false -nologo -clp:NoSummary --verbosity quiet } ExecSafe { & $env:DOTNET_EXE run --project $BuildProjectFile --no-build -- $BuildArguments } + + + + diff --git a/build.sh b/build.sh index ad37088c2c..805bd1bf15 100755 --- a/build.sh +++ b/build.sh @@ -44,15 +44,40 @@ else unset DOTNET_VERSION fi fi + + # ----- Octopus Deploy Modification ----- + # + # The default behaviour of the Nuke Bootstrapper (when .NET is not already preinstalled) is + # to read from the global.json, then install that exact version. It doesn't roll forward. + # This means that if our global.json says 8.0.100, and the latest version is 8.0.200, it will + # always install 8.0.100 and we will not pick up any security or bug fixes that 8.0.200 carries. + # + # This means we would need to manually update our global.json file every time there is a new + # .NET SDK available, and then all developers would need to immediately install this on their machines. + # + # In our builds, we want the same "automatic roll-forward" behaviour that we get when we use the dotnet/sdk:8.0 docker + # images -- where we always get the latest patch version of the SDK without manual intervention. + # + # We achieve this with a small tweak to the Nuke bootstrapper to tell it to install the latest version from + # the 8.0 channel, regardless of what's in the global.json. + + unset DOTNET_VERSION + DOTNET_CHANNEL="8.0" + # ----- End Octopus Deploy Modification ----- # Install by channel or version DOTNET_DIRECTORY="$TEMP_DIRECTORY/dotnet-unix" if [[ -z ${DOTNET_VERSION+x} ]]; then - "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --channel "$DOTNET_CHANNEL" --no-path + "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --channel "$DOTNET_CHANNEL" else - "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --version "$DOTNET_VERSION" --no-path + "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --version "$DOTNET_VERSION" fi export DOTNET_EXE="$DOTNET_DIRECTORY/dotnet" + + # ----- Octopus Deploy Modification ----- + # Update the path with the temporary dotnet exe so it can be found by anything be run out of this shell + export PATH="$PATH:$DOTNET_DIRECTORY" + # ----- End Octopus Deploy Modification ----- fi echo "Microsoft (R) .NET Core SDK version $("$DOTNET_EXE" --version)" diff --git a/build/Build.CalamariTesting.cs b/build/Build.CalamariTesting.cs new file mode 100644 index 0000000000..d6e1fb4582 --- /dev/null +++ b/build/Build.CalamariTesting.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using JetBrains.Annotations; +using Nuke.Common; +using Nuke.Common.Tooling; +using Nuke.Common.Tools.DotNet; +using Nuke.Common.CI.TeamCity; + +namespace Calamari.Build; + +partial class Build +{ + [PublicAPI] + Target PlatformAgnosticTesting => + target => target + .Executes(async () => + { + var dotnetPath = await LocateOrInstallDotNetSdk(); + + CreateTestRun("Binaries/Calamari.Tests.dll") + .WithDotNetPath(dotnetPath) + .WithFilter("TestCategory = PlatformAgnostic") + .Execute(); + }); + + [PublicAPI] + Target LinuxSpecificTesting => + target => target + .Executes(async () => + { + var dotnetPath = await LocateOrInstallDotNetSdk(); + + CreateTestRun("Binaries/Calamari.Tests.dll") + .WithDotNetPath(dotnetPath) + .WithFilter("TestCategory != Windows & TestCategory != PlatformAgnostic & TestCategory != RunOnceOnWindowsAndLinux") + .Execute(); + }); + + [PublicAPI] + Target OncePerWindowsOrLinuxTesting => + target => target + .Executes(async () => + { + var dotnetPath = await LocateOrInstallDotNetSdk(); + + CreateTestRun("Binaries/Calamari.Tests.dll") + .WithDotNetPath(dotnetPath) + .WithFilter("(TestCategory != Windows & TestCategory != PlatformAgnostic) | TestCategory = RunOnceOnWindowsAndLinux") + .Execute(); + }); + + [PublicAPI] + Target OncePerWindowsTesting => + target => target + .Executes(async () => + { + var dotnetPath = await LocateOrInstallDotNetSdk(); + + CreateTestRun("Binaries/Calamari.Tests.dll") + .WithDotNetPath(dotnetPath) + .WithFilter("TestCategory != macOs & TestCategory != Nix & TestCategory != PlatformAgnostic & TestCategory != nixMacOS & TestCategory != RunOnceOnWindowsAndLinux & TestCategory != ModifiesSystemProxy") + .Execute(); + }); + + [PublicAPI] + Target WindowsSystemProxyTesting => + target => target + .Executes(async () => + { + var dotnetPath = await LocateOrInstallDotNetSdk(); + + CreateTestRun("Binaries/Calamari.Tests.dll") + .WithDotNetPath(dotnetPath) + .WithFilter("TestCategory = Windows & TestCategory = ModifiesSystemProxy") + .Execute(); + }); +} \ No newline at end of file diff --git a/build/Build.CreateTestRun.cs b/build/Build.CreateTestRun.cs new file mode 100644 index 0000000000..f886a5c3f2 --- /dev/null +++ b/build/Build.CreateTestRun.cs @@ -0,0 +1,13 @@ +using System.Threading.Tasks; + +namespace Calamari.Build; + +partial class Build +{ + + CalamariTestRunBuilder CreateTestRun(string projectFileOrDll) + { + var outputDir = RootDirectory / "outputs"; + return new CalamariTestRunBuilder(projectFileOrDll, outputDir); + } +} \ No newline at end of file diff --git a/build/Build.InstallDotNetSdk.cs b/build/Build.InstallDotNetSdk.cs new file mode 100644 index 0000000000..203b048d55 --- /dev/null +++ b/build/Build.InstallDotNetSdk.cs @@ -0,0 +1,263 @@ +using System; +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Calamari.Build.Utilities; +using JetBrains.Annotations; +using Nuke.Common; +using Nuke.Common.IO; +using Nuke.Common.Tooling; +using Nuke.Common.Tools.DotNet; +using Serilog; + +namespace Calamari.Build +{ + public partial class Build + { + [Parameter("Specify this if you want to install a particular version of the SDK. Otherwise the InstallDotNetSdk Target will use global.json to determine the .NET SDK Version", Name = "dotnet-version")] + public static string? dotNetVersionParameter; + + /// + /// This target only exists so you can run nuke InstallDotNetSdk outside of another Target. + /// If you have some Target that wants to install the .NET SDK, please call the InstallDotNetSdkIfRequired() + /// method directly. + /// + [PublicAPI] + public Target InstallDotNetSdk => t => t.Executes(async () => await LocateOrInstallDotNetSdk(dotNetVersionParameter)); + + /// + /// Searches for an appropriate dotnet SDK and returns the path to `dotnet.exe` (or unix equivalent). + /// Will install the SDK if it is not found. + /// Note: If an SDK is installed, it will typically go into a temporary folder and not the system-wide one. + /// This is why it's important to use returned path. If you call this method and later just shell out to "dotnet" + /// you may get the wrong thing. + /// + /// + /// This implements the "rollForward" feature that is typically specified in a global.json file. + /// If `specificVersion` is not supplied, and we resolve a global.json that says something like + /// { "version": "6.0.403", "rollForward": "latestFeature" } + /// then it will go search the internet for the latest 6.0.x release and install that instead. + /// + /// If `specificVersion` is supplied and is a 2-part number, this invokes the rollForward behaviour as well, + /// e.g. "6.0" will install the latest 6.0.x release. + /// + /// + /// If set, will find or install a particular version of the .NET SDK e.g. 6.0.417. + /// If not set, will look for an appropriate version by scanning for a global.json file + /// + async Task LocateOrInstallDotNetSdk(string? specificVersion = null) + { + var httpClient = new Lazy(() => new HttpClient(), isThreadSafe: true); + + DotNetDownloadStrategy strategy; + if (specificVersion != null) + { + Log.Information("Request to install .NET SDK using command-line parameter: {DotNetVersion}", specificVersion); + strategy = GlobalJson.DetermineDownloadStrategy(specificVersion, null); + } + else + { + var assemblyLocation = Assembly.GetExecutingAssembly().Location; + var directory = Path.GetDirectoryName(assemblyLocation) ?? throw new InvalidOperationException("can't determine directory for executing assembly"); + + var globalJsonFile = GlobalJson.Find(directory, Log.Logger); + if (globalJsonFile == null) throw new Exception("--dotnet-version parameter was not supplied, and could not find a global.json file to tell us the SDK version to install; aborting"); + + var parsed = GlobalJson.Parse(globalJsonFile); + Log.Information("Request to install .NET SDK using {GlobalJsonPath} with Version {Version} and RollForward {RollForward}", + globalJsonFile, parsed.Version, parsed.RollForward); + + strategy = GlobalJson.DetermineDownloadStrategy(parsed.Version, parsed.RollForward); + } + + string targetSdkVersion; + switch (strategy) + { + case DotNetDownloadStrategy.Exact exact: + targetSdkVersion = exact.Version; + break; + + case DotNetDownloadStrategy.LatestInChannel latest: + targetSdkVersion = await DetermineLatestVersion(httpClient, latest.Channel); + Log.Information("Using LatestInChannel strategy; found target version {Version} in channel {Channel}", targetSdkVersion, latest.Channel); + break; + + default: + throw new NotSupportedException($"Unhandled download strategy {strategy}"); + } + + if (DotNetSdkIsInstalled(targetSdkVersion)) + { + // assume if it already exists we don't need to chmod + Log.Information(".NET {DotNetVersion} is already installed", targetSdkVersion); + return DotNetTasks.DotNetPath; // DotNetTasks.DotNetPath finds the system-default dotnet in program files or equivalent + } + + var temporaryDotNetDirectory = TemporaryDirectory / $"dotnet-{targetSdkVersion}"; + if (Directory.Exists(temporaryDotNetDirectory)) + { + Log.Information(".NET {DotNetVersion} is not known to `dotnet --list-sdks` but {temporaryDotNetDirectory} exists, assuming it is there", + targetSdkVersion, temporaryDotNetDirectory); + + // as above, assume if it already exists we don't need to chmod + return temporaryDotNetDirectory / (OperatingSystem.IsWindows() ? "dotnet.exe" : "dotnet"); + } + + Log.Information("{DotNetVersion} is not installed. Downloading the .NET sdk zip file", targetSdkVersion); + + var platform = OperatingSystem.IsWindows() + ? "win" + : OperatingSystem.IsMacOS() + ? "osx" + : "linux"; // there are distro-specific packages e.g. debian, they aren't used anymore + + var temporaryArchivePath = await DownloadDotNetSdk(httpClient, targetSdkVersion, platform, ResolveDotNetArchitectureString(RuntimeInformation.OSArchitecture)); + try + { + Log.Information("Extracting {DotNetVersion} into {temporaryDotNetDirectory}", targetSdkVersion, temporaryDotNetDirectory); + if (OperatingSystem.IsWindows()) + { + Directory.CreateDirectory(temporaryDotNetDirectory); + ZipFile.ExtractToDirectory(temporaryArchivePath, temporaryDotNetDirectory, overwriteFiles: true); + return temporaryDotNetDirectory / "dotnet.exe"; + } + else + { + Directory.CreateDirectory(temporaryDotNetDirectory); + await using (var gzipStream = new GZipStream(File.OpenRead(temporaryArchivePath), CompressionMode.Decompress)) + { + await TarFile.ExtractToDirectoryAsync(gzipStream, temporaryDotNetDirectory, overwriteFiles: true); + } + + var executablePath = temporaryDotNetDirectory / "dotnet"; + // On unix we need to chmod +x the executable so later tasks can run it + executablePath.SetExecutable(); + return executablePath; + } + } + finally + { + try + { + File.Delete(temporaryArchivePath); + } + catch + { + // Deliberate empty catch-block; we can't do much if we can't delete the temp file. Not a big deal + } + } + } + + static bool DotNetSdkIsInstalled(string version) + { + // Format: + // 6.0.414 [/usr/local/share/dotnet/sdk] + var versions = DotNetTasks.DotNet("--list-sdks").Select(o => o.Text.Split(" ").First()); + return versions.Contains(version); + } + + // possible values for architecture are [amd64, x64, x86, arm64, arm] + // refer to Get-Machine-Architecture in build.ps1 + static string ResolveDotNetArchitectureString(Architecture architecture) => architecture switch + { + Architecture.Arm or Architecture.Armv6 => "arm", + Architecture.Arm64 or Architecture.LoongArch64 => "arm64", + Architecture.X86 => "x86", + Architecture.X64 => "x64", + // explicitly reference known architectures so the compiler can tell us about new unknown ones when they are added. + Architecture.Wasm or Architecture.S390x => throw new NotSupportedException($"Unsupported OS architecture {architecture}"), + _ => throw new NotSupportedException($"Unknown OS architecture {architecture}"), + }; + + static async Task DetermineLatestVersion(Lazy httpClient, string requestedFuzzyVersion) + { + return await PerformOperationWithFeedAndRetries(async feed => + { + var downloadUrl = $"{feed}/Sdk/{requestedFuzzyVersion}/latest.version"; + + Log.Information($"Attempting download of {downloadUrl}"); + var response = await httpClient.Value.GetAsync(downloadUrl); + response.EnsureSuccessStatusCode(); + + var versionString = await response.Content.ReadAsStringAsync(); + + // sanity check, we should get an exact version number such as 6.0.417 + if (versionString.Count(c => c == '.') != 2) throw new Exception($"Unexpected response {versionString} from {downloadUrl}, expecting a version number such as 8.0.100"); + + return versionString.Trim(); + }); + } + + // This function is ported from https://dot.net/v1/dotnet-install.ps1 (and .sh variant for unix) + // You're supposed to fetch and execute the script so Microsoft can keep it up to date, + // but powershell downloading and executing scripts is slow and painful, particularly on older windows like 2016 where TLS1.2 isn't enabled. + // So we rather just do it ourselves. + // NOTE: Whenever we do a major .NET migration we should review Microsoft's dotnet-install.ps1 script + // and update our code if they've changed any of the download links/etc. Last checked on the release of .NET 8 + // + // returns a path to a temporary file containing the zip or tar.gz that has been downloaded + static async Task DownloadDotNetSdk(Lazy httpClient, string requestedVersion, string platform, string architecture) + { + return await PerformOperationWithFeedAndRetries(async feed => + { + var fileExtension = platform == "win" ? "zip" : "tar.gz"; + // Note: Version must be an exact specific version like 6.0.401. + + // refer Get-Download-Link in dotnet-install.ps1 + // Note this URL works for full releases of .NET but isn't quite right for release candidates; the two copies of + // `requestedVersion` differ when fetching an RC. Next time we want to download an RC SDK we'll need to fix this + var downloadUrl = $"{feed}/Sdk/{requestedVersion}/dotnet-sdk-{requestedVersion}-{platform}-{architecture}.{fileExtension}"; + + Log.Information($"Attempting download of {downloadUrl}"); + var targetFile = Path.GetTempFileName(); + await using var fileStream = new FileStream(targetFile, FileMode.Create, FileAccess.ReadWrite); + + var response = await httpClient.Value.GetAsync(downloadUrl); + response.EnsureSuccessStatusCode(); + + await response.Content.CopyToAsync(fileStream); + + return targetFile; + }); + } + + // feeds are tried in this order + static readonly string[] Feeds = + { + // CDN's + "https://builds.dotnet.microsoft.com/dotnet", + "https://ci.dot.net/public", + + // direct + "https://dotnetcli.blob.core.windows.net/dotnet", + "https://dotnetbuilds.blob.core.windows.net/public" + }; + + static async Task PerformOperationWithFeedAndRetries(Func> performOperation) + { + ExceptionDispatchInfo? lastException = null; + foreach (var feed in Feeds.Concat(Feeds)) // get a retry on each feed with sneaky concat + { + try + { + return await performOperation(feed); + } + catch (Exception ex) + { + lastException = ExceptionDispatchInfo.Capture(ex); + Log.Warning(ex, $"Exception occurred using feed {feed}"); + // carry on, let the foreach loop roll over to the next mirror + } + } + + lastException?.Throw(); + throw new Exception("PerformOperationWithFeedAndRetries did not return a result, but caught no exception? Are there any feeds?"); // shouldn't happen; last resort + } + } +} \ No newline at end of file diff --git a/build/Build.NetCoreTesting.cs b/build/Build.NetCoreTesting.cs deleted file mode 100644 index 105d85370e..0000000000 --- a/build/Build.NetCoreTesting.cs +++ /dev/null @@ -1,30 +0,0 @@ -using JetBrains.Annotations; -using Nuke.Common; -using Nuke.Common.Tooling; -using Nuke.Common.Tools.DotNet; - -namespace Calamari.Build; - -partial class Build -{ - [PublicAPI] - Target NetCoreTesting => - target => target - .Executes(() => - { - const string testFilter = - "TestCategory != Windows & TestCategory != PlatformAgnostic & TestCategory != RunOnceOnWindowsAndLinux"; - - DotNetTasks.DotNetTest(settings => settings - .SetProjectFile("Binaries/Calamari.Tests.dll") - .SetFilter(testFilter) - .SetLoggers("trx") - .SetProcessExitHandler( - process => process.ExitCode switch - { - 0 => null, //successful - 1 => null, //some tests failed - _ => throw new ProcessException(process) - })); - }); -} \ No newline at end of file diff --git a/build/Build.TestCalamariFlavourProject.cs b/build/Build.TestCalamariFlavourProject.cs index edf8bd5e3f..15d0d17475 100644 --- a/build/Build.TestCalamariFlavourProject.cs +++ b/build/Build.TestCalamariFlavourProject.cs @@ -1,9 +1,8 @@ -using System.IO; using JetBrains.Annotations; using Nuke.Common; -using Nuke.Common.IO; using Nuke.Common.Tooling; using Nuke.Common.Tools.DotNet; +using Nuke.Common.CI.TeamCity; using Serilog; namespace Calamari.Build; @@ -14,48 +13,15 @@ partial class Build [Parameter(Name = "VSTest_TestCaseFilter")] readonly string? CalamariFlavourTestCaseFilter; [PublicAPI] - Target TestCalamariFlavourProject => target => target.Executes(async () => - { - var testProject = $"Calamari.{CalamariFlavourToTest}.Tests"; + Target TestCalamariFlavourProject => + target => target + .Executes(async () => + { + var dotnetPath = await LocateOrInstallDotNetSdk(); - var affectedProjectFile = RootDirectory / "affected.proj"; - bool isAffected; - if (affectedProjectFile.FileExists()) - { - Log.Information("Affected projects analysis found; checking to see if {TestProject} is affected", - testProject); - var contents = await File.ReadAllTextAsync(affectedProjectFile); - - isAffected = contents.Contains(testProject); - } - else - { - Log.Information("Affected projects analysis not found; assuming {TestProject} *is* affected", testProject); - isAffected = true; - } - - if (isAffected) - { - Log.Verbose("{TestProject} tests will be executed", testProject); - - DotNetTasks.DotNetTest(settings => settings - .SetProjectFile($"CalamariTests/{testProject}.dll") - .SetFilter(CalamariFlavourTestCaseFilter) - .SetLoggers("trx") - .SetProcessExitHandler( - process => process.ExitCode switch - { - 0 => null, //successful - 1 => null, //some tests failed - _ => throw new ProcessException(process) - })); - } - else - { - Log.Information("{TestProject} is not affected, so no tests will be executed", testProject); - Log.Information( - $"##teamcity[testStarted name='{testProject}-NoTests' captureStandardOutput='false']"); - Log.Information($"##teamcity[testFinished name='{testProject}-NoTests' duration='0']"); - } - }); -} + CreateTestRun($"CalamariTests/Calamari.{CalamariFlavourToTest}.Tests.dll") + .WithDotNetPath(dotnetPath) + .WithFilter(CalamariFlavourTestCaseFilter) + .Execute(); + }); +} \ No newline at end of file diff --git a/build/Build.cs b/build/Build.cs index f865dccda3..8c94bf518c 100644 --- a/build/Build.cs +++ b/build/Build.cs @@ -7,6 +7,7 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Calamari.Build.Utilities; using NuGet.Packaging; using Nuke.Common; using Nuke.Common.CI.TeamCity; @@ -86,7 +87,7 @@ public Build() // Mimic the behaviour of this attribute, but lazily so we don't pay the OctoVersion cost when it isn't needed OctoVersionInfo = new Lazy(() => { - var attribute = new OctoVersionAttribute { BranchMember = nameof(BranchName), Framework = "net6.0" }; + var attribute = new OctoVersionAttribute { BranchMember = nameof(BranchName), Framework = "net8.0" }; // the Attribute does all the work such as calling TeamCity.Instance?.SetBuildNumber for us var version = attribute.GetValue(null!, this); @@ -193,22 +194,22 @@ public Build() { Log.Warning("Building Calamari on a non-windows machine will result " + "in the {DefaultNugetPackageName} and {CloudNugetPackageName} " - + "nuget packages being built as .Net Core 6.0 packages " + + "nuget packages being built as .Net Core 8.0 packages " + "instead of as .Net Framework. " + "This can cause compatibility issues when running certain " + "deployment steps in Octopus Server", RootProjectName, $"{RootProjectName}.{FixedRuntimes.Cloud}"); - DoPublish(RootProjectName, Frameworks.Net60, nugetVersion); + DoPublish(RootProjectName, Frameworks.Net80, nugetVersion); Log.Warning($"Skipping the bundling of {RootProjectName} into the Calamari.Legacy bundle. " + "This is required for providing .Net Framework executables for legacy Target Operating Systems"); - DoPublish(RootProjectName, Frameworks.Net60, nugetVersion, FixedRuntimes.Cloud); + DoPublish(RootProjectName, Frameworks.Net80, nugetVersion, FixedRuntimes.Cloud); } foreach (var rid in GetRuntimeIdentifiers(Solution.GetProject(RootProjectName)!)) - DoPublish(RootProjectName, Frameworks.Net60, nugetVersion, rid); + DoPublish(RootProjectName, Frameworks.Net80, nugetVersion, rid); }); Target GetCalamariFlavourProjectsToPublish => @@ -234,7 +235,7 @@ public Build() CalamariProjects = calamariProjects; // All cross-platform Target Frameworks contain dots, all NetFx Target Frameworks don't - // eg: net40, net452, net48 vs netcoreapp3.1, net5.0, net6.0 + // eg: net40, net452, net48 vs netcoreapp3.1, net5.0, net8.0 bool IsCrossPlatform(string targetFramework) => targetFramework.Contains('.'); var calamariPackages = @@ -410,7 +411,7 @@ static void StageLegacyCalamariAssemblies(CalamariPackageMetadata[] packagesToPu } packagesToPublish - //We only need to bundle executable (not tests or libraries) full framework projects + //We only need to bundle executable (not tests or libraries) full framework projects .Where(d => d.Framework == Frameworks.Net462 && d.Project.GetOutputType() == "Exe") .ForEach(calamariPackageMetadata => { @@ -490,10 +491,10 @@ void CompressCalamariProject(Project project) var packageActions = new List { () => DoPackage(RootProjectName, - OperatingSystem.IsWindows() ? Frameworks.Net462 : Frameworks.Net60, + OperatingSystem.IsWindows() ? Frameworks.Net462 : Frameworks.Net80, nugetVersion), () => DoPackage(RootProjectName, - OperatingSystem.IsWindows() ? Frameworks.Net462 : Frameworks.Net60, + OperatingSystem.IsWindows() ? Frameworks.Net462 : Frameworks.Net80, nugetVersion, FixedRuntimes.Cloud), }; @@ -502,7 +503,7 @@ void CompressCalamariProject(Project project) // ReSharper disable once LoopCanBeConvertedToQuery foreach (var rid in GetRuntimeIdentifiers(Solution.GetProject(RootProjectName)!)) packageActions.Add(() => DoPackage(RootProjectName, - Frameworks.Net60, + Frameworks.Net80, nugetVersion, rid)); @@ -539,12 +540,18 @@ void CompressCalamariProject(Project project) .Executes(async () => { var nugetVersion = NugetVersion.Value; - var defaultTarget = OperatingSystem.IsWindows() ? Frameworks.Net462 : Frameworks.Net60; - AbsolutePath binFolder = SourceDirectory / "Calamari.Tests" / "bin" / Configuration / defaultTarget; - Directory.Exists(binFolder); var actions = new List { - () => binFolder.CompressTo(ArtifactsDirectory / "Binaries.zip") + () => + { + //if this is windows, publish a netfx version of the tests project + if (OperatingSystem.IsWindows()) + { + var publishedLocation = DoPublish("Calamari.Tests", Frameworks.Net462, nugetVersion); + var zipName = $"Calamari.Tests.{Frameworks.Net462}.{nugetVersion}.zip"; + publishedLocation.CompressTo(ArtifactsDirectory / zipName); + } + } }; // Create a Zip for each runtime for testing @@ -554,7 +561,7 @@ void CompressCalamariProject(Project project) //run each build in sequence as it's the same project and we get issues foreach (var rid in GetRuntimeIdentifiers(Solution.GetProject("Calamari.Tests")!)) { - var publishedLocation = DoPublish("Calamari.Tests", Frameworks.Net60, nugetVersion, rid); + var publishedLocation = DoPublish("Calamari.Tests", Frameworks.Net80, nugetVersion, rid); var zipName = $"Calamari.Tests.{rid}.{nugetVersion}.zip"; File.Copy(RootDirectory / "global.json", publishedLocation / "global.json"); publishedLocation.CompressTo(ArtifactsDirectory / zipName); @@ -698,6 +705,23 @@ void CompressCalamariProject(Project project) } }); + Target PublishNukeBuild => + d => + d.Executes(async () => + { + const string runtime = "win-x64"; + var nukeBuildOutputDirectory = BuildDirectory / "outputs" / runtime / "nukebuild"; + nukeBuildOutputDirectory.CreateOrCleanDirectory(); + + DotNetPublish(p => p + .SetProject(RootDirectory / "build" / "_build.csproj") + .SetConfiguration(Configuration) + .SetRuntime(runtime) + .EnableSelfContained()); + + await Ci.ZipFolderAndUploadArtifact(nukeBuildOutputDirectory, ArtifactsDirectory / $"nukebuild.{runtime}.zip"); + }); + Target SetTeamCityVersion => d => d.Executes(() => TeamCity.Instance?.SetBuildNumber(NugetVersion.Value)); Target BuildLocal => d => @@ -707,7 +731,8 @@ void CompressCalamariProject(Project project) Target BuildCi => d => d.DependsOn(SetTeamCityVersion) .DependsOn(Pack) - .DependsOn(PackCalamariConsolidatedNugetPackage); + .DependsOn(PackCalamariConsolidatedNugetPackage) + .DependsOn(PublishNukeBuild); public static int Main() => Execute(x => IsServerBuild ? x.BuildCi : x.BuildLocal); @@ -736,7 +761,7 @@ AbsolutePath DoPublish(string project, string framework, string version, string? .SetVerbosity(BuildVerbosity) .SetRuntime(runtimeId) .SetVersion(version) - .SetSelfContained(OperatingSystem.IsWindows()) // This is here purely to make the local build experience on non-Windows devices workable - Publish breaks on non-Windows platforms with SelfContained = true + .SetSelfContained(runtimeId != null) ); if (WillSignBinaries) diff --git a/build/Build.sbom.cs b/build/Build.sbom.cs index 68cad71235..784bdda0a1 100644 --- a/build/Build.sbom.cs +++ b/build/Build.sbom.cs @@ -54,6 +54,7 @@ partial class Build .Where(path => !path.Contains("/TestResults/")) .Where(path => !path.Contains("/.git/")) .Where(path => !path.Contains(".Test")) + .Where(path => !path.Contains(".nuke")) .Where(path => !path.Contains("/_build")) .Select(ResolveCalamariComponent); @@ -205,7 +206,7 @@ void CombineAndValidateSBOM(OctoVersionInfo octoVersionInfo, string[] inputFiles var containerName = $"calamari-sbom-validator-{octoVersionInfo.FullSemVer}"; ContainersWeHaveCreated.Add(containerName); DockerTasks.DockerRun(x => x - .SetName($"octopus-sbom-validator-{octoVersionInfo.FullSemVer}") + .SetName(containerName) .SetPlatform("linux/amd64") .SetRm(true) .SetVolume($"{ArtifactsDirectory}:/sboms") diff --git a/build/Calamari.Consolidated.targets b/build/Calamari.Consolidated.targets index 9464ce1c9c..c753d94c5e 100644 --- a/build/Calamari.Consolidated.targets +++ b/build/Calamari.Consolidated.targets @@ -4,10 +4,10 @@ - + - + diff --git a/build/CalamariTestRunBuilder.cs b/build/CalamariTestRunBuilder.cs new file mode 100644 index 0000000000..1370e63e5d --- /dev/null +++ b/build/CalamariTestRunBuilder.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Nuke.Common.CI.TeamCity; +using Nuke.Common.IO; +using Nuke.Common.Tooling; +using Nuke.Common.Tools.DotNet; + +namespace Calamari.Build; + +public class CalamariTestRunBuilder(string projectFile, AbsolutePath outputDirectory) +{ + readonly string ProjectFile = projectFile; + readonly AbsolutePath OutputDirectory = outputDirectory; + AbsolutePath DotNetPath = DotNetTasks.DotNetPath; + string? Filter; + + public CalamariTestRunBuilder WithDotNetPath(AbsolutePath value) + { + DotNetPath = value; + return this; + } + + public CalamariTestRunBuilder WithFilter(string? value) + { + Filter = value; + return this; + } + + DotNetTestSettings BuildTestSettings() + { + var runningInTeamCity = TeamCity.Instance is not null; + + var settings = new DotNetTestSettings() + .SetProjectFile(ProjectFile) + .SetProcessToolPath(DotNetPath) + .SetTestAdapterPath(OutputDirectory) + // This is so we can mute tests that fail + .SetProcessExitHandler(process => process.ExitCode switch + { + 0 => null, //successful + 1 => null, //some tests failed + _ => throw new ProcessException(process) + }) + .AddLoggers("console;verbose=normal") + .When(runningInTeamCity, x => x.EnableTeamCityTestLogger(OutputDirectory)); + + var runSettingsFilePath = TryBuildExcludedTestsSettingsFile(Filter); + if (runSettingsFilePath is not null) + { + settings = settings.SetSettingsFile(runSettingsFilePath); + } + else if (Filter is not null) + { + settings = settings.SetFilter(Filter); + } + + return settings; + } + + public void Execute() + { + DotNetTasks.DotNetTest(BuildTestSettings()); + } + + static string? TryBuildExcludedTestsSettingsFile(string? baseFilter) + { + var excludedTestsFile = Environment.GetEnvironmentVariable("TeamCityTestExclusionFilePath"); + if (!string.IsNullOrEmpty(excludedTestsFile)) + { + if (File.Exists(excludedTestsFile)) + { + var testSet = new HashSet(); + + using var filestream = File.OpenRead(excludedTestsFile); + using var streamReader = new StreamReader(filestream); + while (streamReader.ReadLine() is { } line) + if (!line.StartsWith('#')) + { + testSet.Add(line); + } + + var exclusionWhere = string.Join(" and ", + testSet.Select(test => $"test != '{test}'")); + + //normalize to 'cat' for category https://docs.nunit.org/articles/nunit/running-tests/Test-Selection-Language.html + //replace & and | with words as it's being written into XML + var normalizedBaseFilter = baseFilter?.Replace("TestCategory", "cat").Replace("&", "and").Replace("|", "or"); + + var whereClause = normalizedBaseFilter is not null + ? $"({normalizedBaseFilter}) and {exclusionWhere}" + : exclusionWhere; + + var runSettingsFile = $""" + + + {whereClause} + + + """; + + var filePath = KnownPaths.RootDirectory / "excluded.runSettings"; + File.WriteAllText(filePath, runSettingsFile); + + TeamCity.Instance.PublishArtifacts(filePath); + + return filePath; + } + } + + return null; + } +} \ No newline at end of file diff --git a/build/DotNetTestSettingsExtensionMethods.cs b/build/DotNetTestSettingsExtensionMethods.cs new file mode 100644 index 0000000000..30e1908341 --- /dev/null +++ b/build/DotNetTestSettingsExtensionMethods.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using Nuke.Common.IO; +using Nuke.Common.Tooling; +using Nuke.Common.Tools.DotNet; + +namespace Calamari.Build +{ + public static class DotNetTestSettingsExtensionMethods + { + public static DotNetTestSettings EnableTeamCityTestLogger(this DotNetTestSettings settings, AbsolutePath outputDirectory) + { + settings = settings.AddLoggers("teamcity"); + + // opt in to the teamcity test reporting via file approach as suggested in the support ticket + // https://youtrack.jetbrains.com/issue/TW-80096/Inconsistent-test-counts-when-using-dotnet-test-and-NUnit-adapter#focus=Comments-27-8728443.0-0 + var testReportsDirectory = outputDirectory / "TestReports" / Guid.NewGuid().ToString(); + testReportsDirectory.CreateOrCleanDirectory(); + settings = settings.SetProcessEnvironmentVariable("TEAMCITY_TEST_REPORT_FILES_PATH", testReportsDirectory); + Console.WriteLine($"##teamcity[importData type='streamToBuildLog' filePattern='{testReportsDirectory}/*.msg' wrapFileContentInBlock='false' quiet='false']"); + return settings; + } + } +} \ No newline at end of file diff --git a/build/Frameworks.cs b/build/Frameworks.cs index 3bc89bbc49..dfe16e8345 100644 --- a/build/Frameworks.cs +++ b/build/Frameworks.cs @@ -5,6 +5,6 @@ namespace Calamari.Build public static class Frameworks { public const string Net462 = "net462"; - public const string Net60 = "net6.0"; + public const string Net80 = "net8.0"; } } \ No newline at end of file diff --git a/build/KnownPaths.cs b/build/KnownPaths.cs new file mode 100644 index 0000000000..052df59479 --- /dev/null +++ b/build/KnownPaths.cs @@ -0,0 +1,11 @@ +using Nuke.Common; +using Nuke.Common.IO; + +namespace Calamari.Build; + +public static class KnownPaths +{ + public static AbsolutePath RootDirectory => NukeBuild.RootDirectory; + public static AbsolutePath OutputsDirectory => RootDirectory / "outputs"; + +} \ No newline at end of file diff --git a/build/Utilities/Ci.cs b/build/Utilities/Ci.cs new file mode 100644 index 0000000000..a5f2383569 --- /dev/null +++ b/build/Utilities/Ci.cs @@ -0,0 +1,40 @@ +// ReSharper disable RedundantUsingDirective + +using System; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Nuke.Common.CI.TeamCity; +using Nuke.Common.IO; +using Serilog; + +namespace Calamari.Build.Utilities +{ + class Ci + { + public static async Task ZipFolderAndUploadArtifact(AbsolutePath folderPath, AbsolutePath outputPath) + { + await Task.CompletedTask; + + if (!Directory.EnumerateFiles(folderPath).Any() + && !Directory.EnumerateDirectories(folderPath).Any()) + { + Log.Information("Could not find any file or folder in {FolderPath} to zip into {ZipFileName}", folderPath, outputPath); + return; + } + + var startTimestamp = Stopwatch.GetTimestamp(); + + Log.Information("Zipping folder {FolderPath} into {ZipFileName} - {Timestamp}", folderPath, outputPath, DateTime.Now); + ZipFile.CreateFromDirectory(folderPath, outputPath); + + var stopTimestamp = Stopwatch.GetTimestamp(); + + Log.Information("Completed zipping folder {FolderPath} into {ZipFileName} in {Elapsed}", folderPath, outputPath, Stopwatch.GetElapsedTime(startTimestamp, stopTimestamp)); + + TeamCity.Instance?.PublishArtifacts($"{outputPath}=>/"); + } + } +} diff --git a/build/Utilities/GlobalJson.cs b/build/Utilities/GlobalJson.cs new file mode 100644 index 0000000000..5bc4fe4784 --- /dev/null +++ b/build/Utilities/GlobalJson.cs @@ -0,0 +1,100 @@ +using System; +using System.IO; +using System.Text.Json; +using Serilog; + +namespace Calamari.Build.Utilities +{ + /// + /// A Global.json file typically looks like this: + /// { + /// "sdk": { + /// "version": "6.0.300", + /// "rollForward": "latestFeature" + /// } + /// } + /// + public record GlobalJsonContents(string Version, string? RollForward); + + public abstract record DotNetDownloadStrategy + { + public record LatestInChannel(string Channel) : DotNetDownloadStrategy; + + public record Exact(string Version) : DotNetDownloadStrategy; + } + + /// + /// Helper code for dealing with .NET global.json files + /// + public class GlobalJson + { + public static DotNetDownloadStrategy DetermineDownloadStrategy(string version, string? rollForwardBehavior) + { + // we never roll forward a prerelease version. This is simply because we haven't written + // the code to deal with this appropriately. If you find yourself wanting to supply a prerelease + // version here, please update it + if (version.Contains('-')) return new DotNetDownloadStrategy.Exact(version); + + var components = version.Split("."); + + return rollForwardBehavior switch + { + "disable" => new DotNetDownloadStrategy.Exact(version), // this might result in a search for a runtime that doesn't exist; garbage-in/garbage out + + "latestFeature" when components.Length == 3 => new DotNetDownloadStrategy.LatestInChannel($"{components[0]}.{components[1]}"), // "8.0" is considered a valid channel in Microsoft's distribution system, so we ask for that + + null => components.Length switch + { + 2 => new DotNetDownloadStrategy.LatestInChannel(version), + 3 => new DotNetDownloadStrategy.Exact(version), + _ => throw new ArgumentException($"Can't figure out download strategy for version {version}") + }, + + _ => throw new NotSupportedException($"Unsupported rollForwardBehavior {rollForwardBehavior}") + }; + } + + public static GlobalJsonContents Parse( string filePath) + => Parse(File.ReadAllBytes(filePath), filePath); + + public static GlobalJsonContents Parse(byte[] utf8Bytes, string filePath) + { + using var doc = JsonDocument.Parse(utf8Bytes, new JsonDocumentOptions { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }); + var root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Object) throw new FormatException($"could not parse {filePath}; root was not object"); + if (!root.TryGetProperty("sdk", out var sdkElement) || sdkElement.ValueKind != JsonValueKind.Object) throw new FormatException($"could not parse {filePath}; no 'sdk' node"); + if (!sdkElement.TryGetProperty("version", out var versionElement) || versionElement.ValueKind != JsonValueKind.String) throw new FormatException($"could not parse {filePath}; no 'sdk/version' node"); + + var version = versionElement.GetString() ?? ""; + if (sdkElement.TryGetProperty("rollForward", out var rollForwardElement) && rollForwardElement.ValueKind == JsonValueKind.String) + { + return new GlobalJsonContents(version, rollForwardElement.GetString()); + } + + return new GlobalJsonContents(version, null); + } + + public static string? Find(string startingDirectory, ILogger logger) + { + var directory = startingDirectory; + while (directory is { Length: > 0 }) + { + logger.Verbose("Looking for global.json in {Directory}", directory); + var candidate = Path.Combine(directory, "global.json"); + + if (File.Exists(candidate)) + { + Log.Information("Found {FilePath}", candidate); + return candidate; + } + + var parent = Path.GetDirectoryName(directory); + + if (parent == directory) break; + directory = parent; + } + + return null; + } + } +} \ No newline at end of file diff --git a/build/_build.csproj b/build/_build.csproj index aa008e3b2b..9c7bd99989 100644 --- a/build/_build.csproj +++ b/build/_build.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 win-x64;linux-x64 Calamari.Build CS0649;CS0169 diff --git a/global.json b/global.json index 5952be4891..f023fd12ae 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "6.0.300", + "version": "8.0.415", "rollForward": "latestFeature", "allowPrerelease": false } diff --git a/source/Calamari.Azure/Calamari.Azure.csproj b/source/Calamari.Azure/Calamari.Azure.csproj index 3e86f265a5..9e67e43391 100644 --- a/source/Calamari.Azure/Calamari.Azure.csproj +++ b/source/Calamari.Azure/Calamari.Azure.csproj @@ -7,7 +7,7 @@ Octopus Deploy Pty Ltd win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 8.0 - net462;net6.0 + net462;net8.0 true diff --git a/source/Calamari.AzureAppService.Tests/AppServiceIntegrationTest.cs b/source/Calamari.AzureAppService.Tests/AppServiceIntegrationTest.cs index da9fbfd477..54c8725421 100644 --- a/source/Calamari.AzureAppService.Tests/AppServiceIntegrationTest.cs +++ b/source/Calamari.AzureAppService.Tests/AppServiceIntegrationTest.cs @@ -14,7 +14,6 @@ using Calamari.Azure; using Calamari.Azure.AppServices; using Calamari.AzureAppService.Azure; -using Calamari.AzureAppService.Json; using Calamari.CloudAccounts; using Calamari.Testing; using Calamari.Testing.Azure; diff --git a/source/Calamari.AzureAppService.Tests/AppServiceSettingsBehaviourFixture.cs b/source/Calamari.AzureAppService.Tests/AppServiceSettingsBehaviourFixture.cs index 1d19d2e0c1..2709cd0c9e 100644 --- a/source/Calamari.AzureAppService.Tests/AppServiceSettingsBehaviourFixture.cs +++ b/source/Calamari.AzureAppService.Tests/AppServiceSettingsBehaviourFixture.cs @@ -9,7 +9,6 @@ using Calamari.Azure.AppServices; using Calamari.AzureAppService.Azure; using Calamari.AzureAppService.Behaviors; -using Calamari.AzureAppService.Json; using Calamari.Common.Commands; using Calamari.Common.Plumbing.Variables; using Calamari.Testing.Helpers; diff --git a/source/Calamari.AzureAppService.Tests/Calamari.AzureAppService.Tests.csproj b/source/Calamari.AzureAppService.Tests/Calamari.AzureAppService.Tests.csproj index 936f694428..3369377f7e 100644 --- a/source/Calamari.AzureAppService.Tests/Calamari.AzureAppService.Tests.csproj +++ b/source/Calamari.AzureAppService.Tests/Calamari.AzureAppService.Tests.csproj @@ -6,7 +6,7 @@ false win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 8.0 - net462;net6.0 + net462;net8.0 true @@ -15,11 +15,12 @@ - - - + + + - + + diff --git a/source/Calamari.AzureAppService/Calamari.AzureAppService.csproj b/source/Calamari.AzureAppService/Calamari.AzureAppService.csproj index a35f9f8cea..e1c0d185a4 100644 --- a/source/Calamari.AzureAppService/Calamari.AzureAppService.csproj +++ b/source/Calamari.AzureAppService/Calamari.AzureAppService.csproj @@ -9,7 +9,7 @@ win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 8.0 NU5104 - net462;net6.0 + net462;net8.0 true @@ -24,7 +24,7 @@ - + @@ -40,10 +40,11 @@ - - - - - - + + + + + + + diff --git a/source/Calamari.AzureAppService/Json/SlotSettingsNames.cs b/source/Calamari.AzureAppService/Json/SlotSettingsNames.cs deleted file mode 100644 index 970721a80f..0000000000 --- a/source/Calamari.AzureAppService/Json/SlotSettingsNames.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Newtonsoft.Json; - -/* - * JSON format - * "properties":{ - * "appSettingNames":[ - * "string1", - * "string2" - * ] - *} - * - */ - -namespace Calamari.AzureAppService.Json -{ - public class appSettingNamesRoot - { - public string name { get; set; } - - public string type => "Microsoft.Web/sites"; - - public properties properties { get; set; } - - } - - public class properties - { - public IEnumerable appSettingNames { get; set; } - public IEnumerable connectionStringNames { get; set; } - } -} diff --git a/source/Calamari.AzureResourceGroup.Tests/Calamari.AzureResourceGroup.Tests.csproj b/source/Calamari.AzureResourceGroup.Tests/Calamari.AzureResourceGroup.Tests.csproj index 4b65a971d3..15226288c6 100644 --- a/source/Calamari.AzureResourceGroup.Tests/Calamari.AzureResourceGroup.Tests.csproj +++ b/source/Calamari.AzureResourceGroup.Tests/Calamari.AzureResourceGroup.Tests.csproj @@ -4,15 +4,15 @@ Calamari.AzureResourceGroup.Tests false win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 - net462;net6.0 + net462;net8.0 true - - - + + + - + diff --git a/source/Calamari.AzureResourceGroup/Calamari.AzureResourceGroup.csproj b/source/Calamari.AzureResourceGroup/Calamari.AzureResourceGroup.csproj index d402ad8e68..d637a549e1 100644 --- a/source/Calamari.AzureResourceGroup/Calamari.AzureResourceGroup.csproj +++ b/source/Calamari.AzureResourceGroup/Calamari.AzureResourceGroup.csproj @@ -8,7 +8,7 @@ false false win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 - net462;net6.0 + net462;net8.0 @@ -34,10 +34,11 @@ - - - - - - - \ No newline at end of file + + + + + + + + diff --git a/source/Calamari.AzureScripting.Tests/Calamari.AzureScripting.Tests.csproj b/source/Calamari.AzureScripting.Tests/Calamari.AzureScripting.Tests.csproj index ba5f0b2704..cc004f8bfd 100644 --- a/source/Calamari.AzureScripting.Tests/Calamari.AzureScripting.Tests.csproj +++ b/source/Calamari.AzureScripting.Tests/Calamari.AzureScripting.Tests.csproj @@ -7,15 +7,15 @@ enable win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 false - net462;net6.0 + net462;net8.0 true - - - - + + + + @@ -50,7 +50,7 @@ - + diff --git a/source/Calamari.AzureScripting/Calamari.AzureScripting.csproj b/source/Calamari.AzureScripting/Calamari.AzureScripting.csproj index 4e688ab36d..1079789115 100644 --- a/source/Calamari.AzureScripting/Calamari.AzureScripting.csproj +++ b/source/Calamari.AzureScripting/Calamari.AzureScripting.csproj @@ -9,7 +9,7 @@ win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 true false - net462;net6.0 + net462;net8.0 true @@ -17,7 +17,7 @@ - + @@ -38,10 +38,11 @@ - - - - - - + + + + + + + diff --git a/source/Calamari.AzureServiceFabric.Tests/Calamari.AzureServiceFabric.Tests.csproj b/source/Calamari.AzureServiceFabric.Tests/Calamari.AzureServiceFabric.Tests.csproj index 1591766149..16eb7cbbe0 100644 --- a/source/Calamari.AzureServiceFabric.Tests/Calamari.AzureServiceFabric.Tests.csproj +++ b/source/Calamari.AzureServiceFabric.Tests/Calamari.AzureServiceFabric.Tests.csproj @@ -4,17 +4,17 @@ Calamari.AzureServiceFabric.Tests Calamari.AzureServiceFabric.Tests false - net462;net6.0-windows + net462;net8.0-windows win-x64 8.0 true - - - - + + + + diff --git a/source/Calamari.AzureServiceFabric/Calamari.AzureServiceFabric.csproj b/source/Calamari.AzureServiceFabric/Calamari.AzureServiceFabric.csproj index cae6d5e717..7fd1cb589f 100644 --- a/source/Calamari.AzureServiceFabric/Calamari.AzureServiceFabric.csproj +++ b/source/Calamari.AzureServiceFabric/Calamari.AzureServiceFabric.csproj @@ -5,7 +5,7 @@ true false Exe - net462;net6.0-windows + net462;net8.0-windows win-x64 8.0 true diff --git a/source/Calamari.AzureWebApp.Tests/Calamari.AzureWebApp.Tests.csproj b/source/Calamari.AzureWebApp.Tests/Calamari.AzureWebApp.Tests.csproj index 88ec2e1f52..08873b7c92 100644 --- a/source/Calamari.AzureWebApp.Tests/Calamari.AzureWebApp.Tests.csproj +++ b/source/Calamari.AzureWebApp.Tests/Calamari.AzureWebApp.Tests.csproj @@ -2,17 +2,17 @@ Calamari.AzureWebApp.Tests Calamari.AzureWebApp.Tests - net462;net6.0 + net462;net8.0 win-x64 8.0 false true - - - - + + + + @@ -32,7 +32,7 @@ - + diff --git a/source/Calamari.AzureWebApp/Calamari.AzureWebApp.csproj b/source/Calamari.AzureWebApp/Calamari.AzureWebApp.csproj index 1d980ac58a..91a6c6ad47 100644 --- a/source/Calamari.AzureWebApp/Calamari.AzureWebApp.csproj +++ b/source/Calamari.AzureWebApp/Calamari.AzureWebApp.csproj @@ -5,8 +5,8 @@ true false Exe - net462;net6.0 - win-x64 + net462;net8.0 + win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 8.0 true @@ -41,11 +41,11 @@ PreserveNewest - + - + @@ -60,7 +60,7 @@ - + @@ -73,10 +73,11 @@ - - - - - - + + + + + + + diff --git a/source/Calamari.Common/Features/Processes/CommandLineRunner.cs b/source/Calamari.Common/Features/Processes/CommandLineRunner.cs index 64fb20ee23..46eec48982 100644 --- a/source/Calamari.Common/Features/Processes/CommandLineRunner.cs +++ b/source/Calamari.Common/Features/Processes/CommandLineRunner.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Runtime.InteropServices; using Calamari.Common.Plumbing.Commands; using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.ServiceMessages; @@ -44,7 +45,12 @@ public CommandResult Execute(CommandLineInvocation invocation) catch (Exception ex) { if (ex.InnerException is Win32Exception) + { commandOutput.WriteError(ConstructWin32ExceptionMessage(invocation.Executable)); + + //todo: @robert.erez - Remove this check if/when we can confirm that the issue is fixed. + LogOpenFileStats(invocation, ex, commandOutput); + } commandOutput.WriteError(ex.ToString()); commandOutput.WriteError("The command that caused the exception was: " + invocation); @@ -57,6 +63,33 @@ public CommandResult Execute(CommandLineInvocation invocation) } } + // Variable used for temporarily evaluating a potential bug with file handles being left open. + static void LogOpenFileStats(CommandLineInvocation invocation, Exception ex, SplitCommandInvocationOutputSink commandOutput) + { + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEAMCITY_VERSION"))) + return; // Only log in our CI environment. + + if (ex.InnerException == null || !ex.InnerException.Message.Contains("Text file busy")) + return; // "Text file busy" is the error that indicates a file is open. + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return; // lsof is not available on Windows. + + try + { + SilentProcessRunner.ExecuteCommand( + "lsof", + "", + invocation.WorkingDirectory, + commandOutput.WriteError, + commandOutput.WriteError); + } + catch (Exception e) + { + commandOutput.WriteInfo("Something really wrong happened when trying to log open file handles: " + e.Message); + } + } + protected virtual List GetCommandOutputs(CommandLineInvocation invocation) { var outputs = new List diff --git a/source/Calamari.Common/Features/Scripting/WindowsPowerShell/PowerShellScriptExecutor.cs b/source/Calamari.Common/Features/Scripting/WindowsPowerShell/PowerShellScriptExecutor.cs index f26048b9dc..3d51405afd 100644 --- a/source/Calamari.Common/Features/Scripting/WindowsPowerShell/PowerShellScriptExecutor.cs +++ b/source/Calamari.Common/Features/Scripting/WindowsPowerShell/PowerShellScriptExecutor.cs @@ -31,9 +31,24 @@ protected override IEnumerable PrepareExecution(Script script, var executable = powerShellBootstrapper.PathToPowerShellExecutable(variables); var arguments = powerShellBootstrapper.FormatCommandArguments(bootstrapFile, debuggingBootstrapFile, variables); + var effectiveEnvironmentVars = environmentVars ?? new Dictionary(); + + // Set unique cache path (XDG_CACHE_HOME) to prevent corruption in parallel executions + // XDG_CACHE_HOME is only applicable on non-Windows platforms + if (variables.GetFlag(PowerShellVariables.UniqueCachePath) + && !CalamariEnvironment.IsRunningOnWindows) + { + var workingDirectory = Path.GetDirectoryName(script.File) ?? Environment.CurrentDirectory; + effectiveEnvironmentVars["XDG_CACHE_HOME"] = Path.Combine( + workingDirectory, + "CalamariPowerShellCache", + Guid.NewGuid().ToString("N") + ); + } + var invocation = new CommandLineInvocation(executable, arguments) { - EnvironmentVars = environmentVars, + EnvironmentVars = effectiveEnvironmentVars, WorkingDirectory = Path.GetDirectoryName(script.File), UserName = powerShellBootstrapper.AllowImpersonation() ? variables.Get(PowerShellVariables.UserName) : null, Password = powerShellBootstrapper.AllowImpersonation() ? ToSecureString(variables.Get(PowerShellVariables.Password)) : null diff --git a/source/Calamari.Common/Plumbing/Variables/PowerShellVariables.cs b/source/Calamari.Common/Plumbing/Variables/PowerShellVariables.cs index 6452b116ac..d4d89d53fa 100644 --- a/source/Calamari.Common/Plumbing/Variables/PowerShellVariables.cs +++ b/source/Calamari.Common/Plumbing/Variables/PowerShellVariables.cs @@ -10,6 +10,7 @@ public class PowerShellVariables public static readonly string UserName = "Octopus.Action.PowerShell.UserName"; public static readonly string Password = "Octopus.Action.PowerShell.Password"; public static readonly string Edition = "Octopus.Action.PowerShell.Edition"; + public static readonly string UniqueCachePath = "Octopus.Action.PowerShell.UniqueCachePath"; public static class PSDebug { diff --git a/source/Calamari.ConsolidateCalamariPackages.Api/Calamari.ConsolidateCalamariPackages.Api.csproj b/source/Calamari.ConsolidateCalamariPackages.Api/Calamari.ConsolidateCalamariPackages.Api.csproj index 86198e9f76..3bea38be8a 100644 --- a/source/Calamari.ConsolidateCalamariPackages.Api/Calamari.ConsolidateCalamariPackages.Api.csproj +++ b/source/Calamari.ConsolidateCalamariPackages.Api/Calamari.ConsolidateCalamariPackages.Api.csproj @@ -3,7 +3,7 @@ Octopus.Calamari.ConsolidatedPackage.Api Octopus.Calamari.ConsolidatedPackage.Api - net6.0 + net8.0 enable enable default diff --git a/source/Calamari.ConsolidateCalamariPackages.Tests/Calamari.ConsolidateCalamariPackages.Tests.csproj b/source/Calamari.ConsolidateCalamariPackages.Tests/Calamari.ConsolidateCalamariPackages.Tests.csproj index 388cad84ac..3b0e227376 100644 --- a/source/Calamari.ConsolidateCalamariPackages.Tests/Calamari.ConsolidateCalamariPackages.Tests.csproj +++ b/source/Calamari.ConsolidateCalamariPackages.Tests/Calamari.ConsolidateCalamariPackages.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false true @@ -11,9 +11,9 @@ - - - + + + diff --git a/source/Calamari.ConsolidateCalamariPackages.Tests/ConsolidationVerificationTests.cs b/source/Calamari.ConsolidateCalamariPackages.Tests/ConsolidationVerificationTests.cs index 66eceb9c78..79cd5ce933 100644 --- a/source/Calamari.ConsolidateCalamariPackages.Tests/ConsolidationVerificationTests.cs +++ b/source/Calamari.ConsolidateCalamariPackages.Tests/ConsolidationVerificationTests.cs @@ -56,7 +56,7 @@ static Dictionary PackagesWithDetails(bool isWi { "Calamari.AzureResourceGroup", new PackagePropertiesToTest(isWindows ? AllArchitectures : NetCoreArchitectures, false) }, { "Calamari.GoogleCloudScripting", new PackagePropertiesToTest(isWindows ? AllArchitectures : NetCoreArchitectures, false) }, { "Calamari.AzureScripting", new PackagePropertiesToTest(isWindows ? AllArchitectures : NetCoreArchitectures, false) }, - { "Calamari.AzureWebApp", new PackagePropertiesToTest(isWindows ? new[] { "netfx", "win-x64" } : new[] {"win-x64"}, false) }, + { "Calamari.AzureWebApp", new PackagePropertiesToTest(isWindows ? AllArchitectures : NetCoreArchitectures, false) }, { "Calamari.Terraform", new PackagePropertiesToTest(isWindows ? AllArchitectures : NetCoreArchitectures, false) } }; } @@ -75,9 +75,9 @@ static bool PackageSupported(string packageId, bool isWindows) static IEnumerable ExpectedPackages() { var isWindowsEnvValue = Environment.GetEnvironmentVariable("IS_WINDOWS"); - + var isWindows = isWindowsEnvValue == null ? RuntimeInformation.IsOSPlatform(OSPlatform.Windows) : bool.Parse(isWindowsEnvValue); - + return PackagesWithDetails(isWindows) .Where(kvp => PackageSupported(kvp.Key, isWindows)) .Select(kvp => kvp.Key); @@ -86,9 +86,9 @@ static IEnumerable ExpectedPackages() static IEnumerable ExpectedPackageArchitectureMappings() { var isWindowsEnvValue = Environment.GetEnvironmentVariable("IS_WINDOWS"); - + var isWindows = isWindowsEnvValue == null ? RuntimeInformation.IsOSPlatform(OSPlatform.Windows) : bool.Parse(isWindowsEnvValue); - + return PackagesWithDetails(isWindows) .Where(kvp => PackageSupported(kvp.Key, isWindows)) .Select(kvp => new TestCaseData(kvp.Key, kvp.Value.Architectures).SetName($"Package_{kvp.Key}_HasExpectedArchitectures")); @@ -97,21 +97,21 @@ static IEnumerable ExpectedPackageArchitectureMappings() static IEnumerable ExpectedPackageNugetStatus() { var isWindowsEnvValue = Environment.GetEnvironmentVariable("IS_WINDOWS"); - + var isWindows = isWindowsEnvValue == null ? RuntimeInformation.IsOSPlatform(OSPlatform.Windows) : bool.Parse(isWindowsEnvValue); - + return PackagesWithDetails(isWindows) .Where(kvp => PackageSupported(kvp.Key, isWindows)) .Select(kvp => new TestCaseData(kvp.Key, kvp.Value.IsNupkg).SetName($"Package {kvp.Key} Has Expected Nuget PackageFlag")); - + } - + [SetUp] public void SetUp() { consolidatedFilePath = Environment.GetEnvironmentVariable("CONSOLIDATED_ZIP") ?? ""; expectedVersion = Environment.GetEnvironmentVariable("EXPECTED_VERSION") ?? ""; - + var indexLoader = new ConsolidatedPackageIndexLoader(); using (var fileStream = File.OpenRead(consolidatedFilePath)) { @@ -132,7 +132,7 @@ public void ConsolidatedPackageIndex_PackagesHaveCorrectVersion(string packageNa var package = consolidatedPackageIndex.GetPackage(packageName); package.Version.Should().Be(expectedVersion); } - + [TestCaseSource(nameof(ExpectedPackageNugetStatus))] public void ConsolidatedPackageIndex_FlagsNugetPackagesCorrectly(string packageName, bool isNugetPackage) { @@ -140,4 +140,4 @@ public void ConsolidatedPackageIndex_FlagsNugetPackagesCorrectly(string packageN package.IsNupkg.Should().Be(isNugetPackage); } } -} \ No newline at end of file +} diff --git a/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj b/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj index 8acc6b19cd..cec3a1dc67 100644 --- a/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj +++ b/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj @@ -3,7 +3,7 @@ Octopus.Calamari.ConsolidatedPackage Octopus.Calamari.ConsolidatedPackage - net6.0 + net8.0 default true true diff --git a/source/Calamari.GoogleCloudScripting.Tests/Calamari.GoogleCloudScripting.Tests.csproj b/source/Calamari.GoogleCloudScripting.Tests/Calamari.GoogleCloudScripting.Tests.csproj index 3c9d3f1438..f1e08445dc 100644 --- a/source/Calamari.GoogleCloudScripting.Tests/Calamari.GoogleCloudScripting.Tests.csproj +++ b/source/Calamari.GoogleCloudScripting.Tests/Calamari.GoogleCloudScripting.Tests.csproj @@ -6,15 +6,15 @@ 8 false enable - net462;net6.0 + net462;net8.0 true - - - + + + - + diff --git a/source/Calamari.GoogleCloudScripting/Calamari.GoogleCloudScripting.csproj b/source/Calamari.GoogleCloudScripting/Calamari.GoogleCloudScripting.csproj index 49f6d3d805..3a2216edae 100644 --- a/source/Calamari.GoogleCloudScripting/Calamari.GoogleCloudScripting.csproj +++ b/source/Calamari.GoogleCloudScripting/Calamari.GoogleCloudScripting.csproj @@ -7,7 +7,7 @@ 8 false false - net462;net6.0 + net462;net8.0 CS8632 true diff --git a/source/Calamari.Scripting.Tests/Calamari.Scripting.Tests.csproj b/source/Calamari.Scripting.Tests/Calamari.Scripting.Tests.csproj index 0f77ce70c2..e6954b932c 100644 --- a/source/Calamari.Scripting.Tests/Calamari.Scripting.Tests.csproj +++ b/source/Calamari.Scripting.Tests/Calamari.Scripting.Tests.csproj @@ -2,7 +2,7 @@ Calamari.Scripting.Tests Calamari.Scripting.Tests - net6.0 + net8.0 win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 false default @@ -10,12 +10,12 @@ - + - - + + - + diff --git a/source/Calamari.Scripting/Calamari.Scripting.csproj b/source/Calamari.Scripting/Calamari.Scripting.csproj index c5cadfc8bc..1ce4bfecdc 100644 --- a/source/Calamari.Scripting/Calamari.Scripting.csproj +++ b/source/Calamari.Scripting/Calamari.Scripting.csproj @@ -8,7 +8,7 @@ win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 true 9 - net462;net6.0 + net462;net8.0 true @@ -23,7 +23,7 @@ CS8600;CS8601;CS8602;CS8603;CS8604 - + $(DefineConstants);HAS_NULLABLE_REF_TYPES diff --git a/source/Calamari.Scripting/DotnetScript/dotnet-script.1.4.0.zip b/source/Calamari.Scripting/DotnetScript/dotnet-script.1.4.0.zip deleted file mode 100644 index cfcafc0d32..0000000000 Binary files a/source/Calamari.Scripting/DotnetScript/dotnet-script.1.4.0.zip and /dev/null differ diff --git a/source/Calamari.Scripting/DotnetScript/dotnet-script.1.6.0.zip b/source/Calamari.Scripting/DotnetScript/dotnet-script.1.6.0.zip new file mode 100644 index 0000000000..88cda61773 Binary files /dev/null and b/source/Calamari.Scripting/DotnetScript/dotnet-script.1.6.0.zip differ diff --git a/source/Calamari.Terraform.Tests/Calamari.Terraform.Tests.csproj b/source/Calamari.Terraform.Tests/Calamari.Terraform.Tests.csproj index 59bf847e79..033202e388 100644 --- a/source/Calamari.Terraform.Tests/Calamari.Terraform.Tests.csproj +++ b/source/Calamari.Terraform.Tests/Calamari.Terraform.Tests.csproj @@ -6,17 +6,17 @@ win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 false 9 - net462;net6.0 + net462;net8.0 CS8632 true - - - - + + + + diff --git a/source/Calamari.Terraform/Calamari.Terraform.csproj b/source/Calamari.Terraform/Calamari.Terraform.csproj index 78809089b8..f657b522ce 100644 --- a/source/Calamari.Terraform/Calamari.Terraform.csproj +++ b/source/Calamari.Terraform/Calamari.Terraform.csproj @@ -7,7 +7,7 @@ Exe 9 win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 - net462;net6.0 + net462;net8.0 true diff --git a/source/Calamari.Testing/Calamari.Testing.csproj b/source/Calamari.Testing/Calamari.Testing.csproj index cef0270885..00711f041a 100644 --- a/source/Calamari.Testing/Calamari.Testing.csproj +++ b/source/Calamari.Testing/Calamari.Testing.csproj @@ -17,11 +17,12 @@ - + + diff --git a/source/Calamari.Tests/Calamari.Tests.csproj b/source/Calamari.Tests/Calamari.Tests.csproj index c82d4a2977..ea1518752f 100644 --- a/source/Calamari.Tests/Calamari.Tests.csproj +++ b/source/Calamari.Tests/Calamari.Tests.csproj @@ -1,288 +1,305 @@  - - true - Calamari.Tests - Calamari.Tests - true - Library - false - NU1603 - - - win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 - net462;net6.0 - 8 - true - - - $(DefineConstants);NETCORE;AZURE_CORE;JAVA_SUPPORT - - - $(DefineConstants);NETFX;IIS_SUPPORT;USE_NUGET_V2_LIBS;USE_OCTODIFF_EXE; - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - - - - - PreserveNewest - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - Always - - - PreserveNewest - - - - - PreserveNewest - - - - - - - - - PreserveNewest - - - - PreserveNewest - - - - PreserveNewest - - - - PreserveNewest - - - - - - - - - - - - - - - @(NuGetCommandLineRef->'%(ResolvedPath)')/tools/*.* + true + Calamari.Tests + Calamari.Tests + true + Library + false + NU1603 + + + win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 + net462;net8.0 + 8 + true + true + + $(DefineConstants);NETCORE;AZURE_CORE;JAVA_SUPPORT + + + $(DefineConstants);NETFX;IIS_SUPPORT;USE_NUGET_V2_LIBS;USE_OCTODIFF_EXE; + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + + PreserveNewest + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + Always + + + PreserveNewest + + + + + PreserveNewest + + + + + + + + + PreserveNewest + + + + PreserveNewest + + + + PreserveNewest + + + + PreserveNewest + + - - - + + + + - - - - - + + + + + + + + + @(NuGetCommandLineRef->'%(ResolvedPath)')/tools/*.* + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/source/Calamari.Tests/Fixtures/Commands/RunTestScriptCommand.cs b/source/Calamari.Tests/Fixtures/Commands/RunTestScriptCommand.cs index 8062c576cc..80c74b3f44 100644 --- a/source/Calamari.Tests/Fixtures/Commands/RunTestScriptCommand.cs +++ b/source/Calamari.Tests/Fixtures/Commands/RunTestScriptCommand.cs @@ -13,7 +13,7 @@ namespace Calamari.Tests.Fixtures.Commands /// A cut down command that runs a script without any journaling, variable substitution or /// other optional steps. /// - [Command("run-test-script", Description = "Invokes a PowerShell, dotnet-script or scriptcs script")] + [Command("run-test-script", Description = "Invokes a PowerShell, dotnet-script script")] public class RunTestScriptCommand : Command { private string scriptFile; diff --git a/source/Calamari.Tests/Fixtures/Deployment/DeployWindowsServiceAbstractFixture.cs b/source/Calamari.Tests/Fixtures/Deployment/DeployWindowsServiceAbstractFixture.cs index 8721b905a9..e77438ed04 100644 --- a/source/Calamari.Tests/Fixtures/Deployment/DeployWindowsServiceAbstractFixture.cs +++ b/source/Calamari.Tests/Fixtures/Deployment/DeployWindowsServiceAbstractFixture.cs @@ -6,6 +6,7 @@ using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Variables; using Calamari.Tests.Fixtures.Deployment.Packages; +using FluentAssertions; using NUnit.Framework; namespace Calamari.Tests.Fixtures.Deployment @@ -43,20 +44,38 @@ private void DeleteExistingService() } } + protected void RunDeploymentAndAssertRunningState(string startMode, string desiredStatus, ServiceControllerStatus serviceStatus) + { + SetupVariables(startMode, desiredStatus); + DeployAndAssert(serviceStatus, null); + } + protected void RunDeployment(Action extraAsserts = null) + { + SetupVariables("auto", null); + DeployAndAssert(ServiceControllerStatus.Running, extraAsserts); + } + + void SetupVariables(string startMode, string desiredStatus) { if (string.IsNullOrEmpty(Variables[KnownVariables.Package.EnabledFeatures])) Variables[KnownVariables.Package.EnabledFeatures] = "Octopus.Features.WindowsService"; Variables["Octopus.Action.WindowsService.CreateOrUpdateService"] = "True"; Variables["Octopus.Action.WindowsService.ServiceAccount"] = "_CUSTOM"; - Variables["Octopus.Action.WindowsService.StartMode"] = "auto"; + Variables["Octopus.Action.WindowsService.StartMode"] = startMode; + if (desiredStatus != null) + Variables["Octopus.Action.WindowsService.DesiredStatus"] = desiredStatus; + Variables["Octopus.Action.WindowsService.ServiceName"] = ServiceName; if (Variables["Octopus.Action.WindowsService.DisplayName"] == null) { Variables["Octopus.Action.WindowsService.DisplayName"] = ServiceName; } Variables["Octopus.Action.WindowsService.ExecutablePath"] = $"{PackageName}.exe"; + } + void DeployAndAssert(ServiceControllerStatus serviceStatus, Action extraAsserts) + { using (var file = new TemporaryFile(PackageBuilder.BuildSamplePackage(PackageName, "1.0.0"))) { var result = DeployPackage(file.FilePath); @@ -68,8 +87,8 @@ protected void RunDeployment(Action extraAsserts = null) using (var installedService = GetInstalledService()) { - Assert.NotNull(installedService, "Service is installed"); - Assert.AreEqual(ServiceControllerStatus.Running, installedService.Status); + installedService.Should().NotBeNull("Service {0} should be installed", ServiceName); + installedService.Status.Should().Be(serviceStatus); } extraAsserts?.Invoke(); diff --git a/source/Calamari.Tests/Fixtures/Deployment/DeployWindowsServiceRunningStateFixture.cs b/source/Calamari.Tests/Fixtures/Deployment/DeployWindowsServiceRunningStateFixture.cs new file mode 100644 index 0000000000..efdd0c0962 --- /dev/null +++ b/source/Calamari.Tests/Fixtures/Deployment/DeployWindowsServiceRunningStateFixture.cs @@ -0,0 +1,72 @@ +using System; +using System.IO; +using System.ServiceProcess; +using Calamari.Common.Plumbing.Variables; +using Calamari.Deployment; +using Calamari.Testing.Helpers; +using Calamari.Tests.Fixtures.Util; +using NUnit.Framework; + +namespace Calamari.Tests.Fixtures.Deployment +{ + [TestFixture] + [Category(TestCategory.CompatibleOS.OnlyWindows)] + public class DeployWindowsServiceRunningStateFixture : DeployWindowsServiceAbstractFixture + { + protected override string ServiceName => "RunningStateFixture"; + + [Test] + public void ShouldBeStoppedWhenStartModeIsUnchanged() + { + RunDeploymentAndAssertRunningState("unchanged", null, ServiceControllerStatus.Stopped); + } + + [Test] + public void ShouldBeRunningWhenStartModeIsAutoAndNoDesiredStatus() + { + RunDeploymentAndAssertRunningState("auto", null, ServiceControllerStatus.Running); + } + + [Test] + public void ShouldBeStoppedWhenStartModeIsDemandAndNoDesiredStatus() + { + RunDeploymentAndAssertRunningState("demand", null, ServiceControllerStatus.Stopped); + } + + [Test] + public void ShouldBeRunningWhenStartModeIsDemandAndDesiredStatusIsStarted() + { + //Setup service in stopped state + RunDeploymentAndAssertRunningState("demand", null, ServiceControllerStatus.Stopped); + + RunDeploymentAndAssertRunningState("demand", "Started", ServiceControllerStatus.Running); + } + + [Test] + public void ShouldBeStoppedWhenStartModeIsDemandAndDesiredStatusIsStopped() + { + //Setup service in running state + RunDeploymentAndAssertRunningState("demand", "Started", ServiceControllerStatus.Running); + + RunDeploymentAndAssertRunningState("demand", "Stopped", ServiceControllerStatus.Stopped); + } + + [Test] + public void ShouldBeStoppedWhenStartModeIsDemandAndDesiredStatusIsUnchangedAndServiceAlreadyStopped() + { + //Setup service in stopped state + RunDeploymentAndAssertRunningState("demand", "Stopped", ServiceControllerStatus.Stopped); + + RunDeploymentAndAssertRunningState("demand", "Unchanged", ServiceControllerStatus.Stopped); + } + + [Test] + public void ShouldBeRunningWhenStartModeIsDemandAndDesiredStatusIsUnchangedAndServiceAlreadyRunning() + { + //Setup service in stopped state + RunDeploymentAndAssertRunningState("demand", "Started", ServiceControllerStatus.Running); + + RunDeploymentAndAssertRunningState("demand", "Unchanged", ServiceControllerStatus.Running); + } + } +} \ No newline at end of file diff --git a/source/Calamari.Tests/Fixtures/DotnetScript/DotnetScriptFixture.cs b/source/Calamari.Tests/Fixtures/DotnetScript/DotnetScriptFixture.cs index e51c821579..d4eaee3db2 100644 --- a/source/Calamari.Tests/Fixtures/DotnetScript/DotnetScriptFixture.cs +++ b/source/Calamari.Tests/Fixtures/DotnetScript/DotnetScriptFixture.cs @@ -124,13 +124,13 @@ public void UsingIsolatedAssemblyLoadContext(bool enableIsolatedLoadContext) if (enableIsolatedLoadContext) { output.AssertSuccess(); - output.AssertOutput("NuGet.Commands version: 6.10.0."); + output.AssertOutput("NuGet.Commands version: 6.10.1.5"); output.AssertOutput("Parameters Parameter0Parameter1"); } else { output.AssertFailure(); - output.AssertErrorOutput("Could not load file or assembly 'NuGet.Protocol, Version=6.10.0."); + output.AssertErrorOutput("Could not load file or assembly 'NuGet.Protocol, Version=6.10.1.5"); } } diff --git a/source/Calamari.Tests/Fixtures/DotnetScript/Scripts/IsolatedLoadContext.csx b/source/Calamari.Tests/Fixtures/DotnetScript/Scripts/IsolatedLoadContext.csx index 5dd2c97dc0..016bf3011b 100644 --- a/source/Calamari.Tests/Fixtures/DotnetScript/Scripts/IsolatedLoadContext.csx +++ b/source/Calamari.Tests/Fixtures/DotnetScript/Scripts/IsolatedLoadContext.csx @@ -1,4 +1,4 @@ -#r "nuget: NuGet.Commands, 6.10.0" +#r "nuget: NuGet.Commands, 6.10.0.107" using NuGet.Configuration; using NuGet.Protocol.Core.Types; diff --git a/source/Calamari.Tests/Fixtures/Integration/Packages/DockerImagePackageDownloaderFixture.cs b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerImagePackageDownloaderFixture.cs index 17206c5cde..cd2e4c8e2b 100644 --- a/source/Calamari.Tests/Fixtures/Integration/Packages/DockerImagePackageDownloaderFixture.cs +++ b/source/Calamari.Tests/Fixtures/Integration/Packages/DockerImagePackageDownloaderFixture.cs @@ -165,7 +165,7 @@ public void CachedNonDockerHubPackage_DoesNotGenerateImageNotCachedMessage() var downloader = GetDownloader(log); PreCacheImage(image, tag, authFeedUri, feedUsername, feedPassword); - + downloader.DownloadPackage(image, new SemanticVersion(tag), "docker-feed", diff --git a/source/Calamari.Tests/Helpers/CodeGenerator.cs b/source/Calamari.Tests/Helpers/CodeGenerator.cs index 9d73ffd501..ad275b31e5 100644 --- a/source/Calamari.Tests/Helpers/CodeGenerator.cs +++ b/source/Calamari.Tests/Helpers/CodeGenerator.cs @@ -26,11 +26,11 @@ CommandLineInvocation CreateCommandLineInvocation(string executable, string argu File.WriteAllText(Path.Combine(projectPath.FullName, "global.json"), @"{ ""sdk"": { - ""version"": ""6.0.10"", + ""version"": ""8.0.10"", ""rollForward"": ""latestFeature"" } }"); - var result = clr.Execute(CreateCommandLineInvocation("dotnet", "new console -f net6.0")); + var result = clr.Execute(CreateCommandLineInvocation("dotnet", "new console -f net8.0")); result.VerifySuccess(); var programCS = Path.Combine(projectPath.FullName, "Program.cs"); var newProgram = $@"using System; diff --git a/source/Calamari.Tests/KubernetesFixtures/Authentication/SetupKubectlAuthenticationFixture.cs b/source/Calamari.Tests/KubernetesFixtures/Authentication/SetupKubectlAuthenticationFixture.cs index 14bbbb3308..db63c12a72 100644 --- a/source/Calamari.Tests/KubernetesFixtures/Authentication/SetupKubectlAuthenticationFixture.cs +++ b/source/Calamari.Tests/KubernetesFixtures/Authentication/SetupKubectlAuthenticationFixture.cs @@ -483,7 +483,7 @@ string ToBase64(string input) } } - // This extension method only exists in .net6.0 + // This extension method only exists in .net8.0 #if NETFX public static class MiscExtensions { diff --git a/source/Calamari.Tests/KubernetesFixtures/ResourceStatus/ResourceRetrieverTests.cs b/source/Calamari.Tests/KubernetesFixtures/ResourceStatus/ResourceRetrieverTests.cs index d8671c98c3..3ac3622e78 100644 --- a/source/Calamari.Tests/KubernetesFixtures/ResourceStatus/ResourceRetrieverTests.cs +++ b/source/Calamari.Tests/KubernetesFixtures/ResourceStatus/ResourceRetrieverTests.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using Amazon.IdentityManagement.Model; using Calamari.Common.Plumbing.Extensions; using Calamari.Common.Plumbing.Logging; using Calamari.Kubernetes.Integration; @@ -216,13 +215,13 @@ public void IgnoresIrrelevantResources() } }); } - + [Test] public void HandlesInvalidJson() { var kubectlGet = new MockKubectlGet(); var resourceRetriever = new ResourceRetriever(kubectlGet, Substitute.For()); - + kubectlGet.SetResource("rs", "invalid json"); var results = resourceRetriever.GetAllOwnedResources( new List @@ -235,13 +234,13 @@ public void HandlesInvalidJson() result.IsSuccess.Should().BeFalse(); result.ErrorMessage.Should().Contain("Failed to parse JSON"); } - + [Test] public void HandlesGetErrors() { var kubectlGet = new MockKubectlGet(); var resourceRetriever = new ResourceRetriever(kubectlGet, Substitute.For()); - + Message[] messages = { new Message(Level.Error, "Error getting resource") }; kubectlGet.SetResource("rs", messages); var results = resourceRetriever.GetAllOwnedResources( @@ -249,7 +248,7 @@ public void HandlesGetErrors() { new ResourceIdentifier(SupportedResourceGroupVersionKinds.ReplicaSetV1, "rs", "octopus"), }, - null, + null, new Options() { PrintVerboseKubectlOutputOnError = true @@ -259,14 +258,14 @@ public void HandlesGetErrors() result.IsSuccess.Should().BeFalse(); result.ErrorMessage.Should().Contain("Error getting resource"); } - - + + [Test] public void HandlesEmptyResponse() { var kubectlGet = new MockKubectlGet(); var resourceRetriever = new ResourceRetriever(kubectlGet, Substitute.For()); - + kubectlGet.SetResource("rs", Array.Empty()); var results = resourceRetriever.GetAllOwnedResources( new List @@ -279,7 +278,7 @@ public void HandlesEmptyResponse() result.IsSuccess.Should().BeFalse(); result.ErrorMessage.Should().Contain("Failed to get resource"); } - + [Test] public void HandleChildFailure() { @@ -294,17 +293,17 @@ public void HandleChildFailure() var kubectlGet = new MockKubectlGet(); var log = new InMemoryLog(); var resourceRetriever = new ResourceRetriever(kubectlGet, log); - + kubectlGet.SetResource("rs", replicaSet); Message[] messages = { new Message(Level.Error, "Error getting resource") }; kubectlGet.SetAllResources("Pod", messages); - + var results = resourceRetriever.GetAllOwnedResources( new List { new ResourceIdentifier(SupportedResourceGroupVersionKinds.ReplicaSetV1, "rs", "octopus"), }, - null, + null, new Options() { PrintVerboseKubectlOutputOnError = true @@ -325,8 +324,37 @@ public void HandleChildFailure() .Should() .Contain(r => r.Contains("Error getting resource")); } + + [Test] + public void HandlesKubectlFailureWithExitCode() + { + var kubectlGet = new MockKubectlGet(); + kubectlGet.SetResource("deploy", new[] { + new Message(Level.Error, "Error from server (Forbidden): deployments.apps \"deploy\" is forbidden") + }); + + var log = new InMemoryLog(); + var resourceRetriever = new ResourceRetriever(kubectlGet, log); + + var results = resourceRetriever.GetAllOwnedResources( + new List + { + new ResourceIdentifier(SupportedResourceGroupVersionKinds.DeploymentV1, "deploy", "default") + }, + null, + new Options { PrintVerboseKubectlOutputOnError = true } + ).ToList(); + + log.MessagesVerboseFormatted + .Should() + .Contain(msg => msg.Contains("kubectl failed with exit code: 1")); + + log.MessagesVerboseFormatted + .Should() + .Contain(msg => msg.Contains("Error from server (Forbidden)")); + } } - + public class MockKubectlGet : IKubectlGet { @@ -359,16 +387,18 @@ public KubectlGetResult Resource(IResourceIdentity resourceIdentity, IKubectl ku { var resourceJson = resourceEntries[resourceIdentity.Name].Select(m => m.Text).Join(string.Empty); var rawOutput = resourceEntries[resourceIdentity.Name].Select(m => $"{m.Level}: {m.Text}").ToList(); - - return new KubectlGetResult(resourceJson, rawOutput); + var exitCode = resourceEntries[resourceIdentity.Name].Any(m => m.Level == Level.Error) ? 1 : 0; + + return new KubectlGetResult(resourceJson, rawOutput, exitCode); } public KubectlGetResult AllResources(ResourceGroupVersionKind groupVersionKind, string @namespace, IKubectl kubectl) { var resourceJson = resourcesByKind[groupVersionKind.Kind].Select(m => m.Text).Join(string.Empty); var rawOutput = resourcesByKind[groupVersionKind.Kind].Select(m => $"{m.Level}: {m.Text}").ToList(); - - return new KubectlGetResult(resourceJson, rawOutput); + var exitCode = resourcesByKind[groupVersionKind.Kind].Any(m => m.Level == Level.Error) ? 1 : 0; + + return new KubectlGetResult(resourceJson, rawOutput, exitCode); } } @@ -394,13 +424,13 @@ public class ResourceResponseBuilder string name = ""; string uid = Guid.NewGuid().ToString(); string ownerUid = Guid.NewGuid().ToString(); - + public ResourceResponseBuilder WithApiVersion(string apiVersion) { this.apiVersion = apiVersion; return this; } - + public ResourceResponseBuilder WithKind(string kind) { this.kind = kind; @@ -435,4 +465,4 @@ public string Build() => .ReplaceLineEndings() .Replace(Environment.NewLine, string.Empty); } -} \ No newline at end of file +} diff --git a/source/Calamari.Tests/KubernetesFixtures/Terraform/EC2/test.sh b/source/Calamari.Tests/KubernetesFixtures/Terraform/EC2/test.sh index 88b7807c0a..9588c592b6 100644 --- a/source/Calamari.Tests/KubernetesFixtures/Terraform/EC2/test.sh +++ b/source/Calamari.Tests/KubernetesFixtures/Terraform/EC2/test.sh @@ -1,9 +1,11 @@ -wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb +wget https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb -O packages-microsoft-prod.deb sudo dpkg -i packages-microsoft-prod.deb +rm packages-microsoft-prod.deb + sudo apt-get update sudo apt-get install -y apt-transport-https zip sudo apt-get update -sudo apt-get install -y dotnet-sdk-6.0 +sudo apt-get install -y dotnet-sdk-8.0 export AWS_CLUSTER_URL=${endpoint} export AWS_CLUSTER_NAME=${cluster_name} diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs index 322b06ec20..03b6dec457 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs @@ -2,7 +2,6 @@ #nullable enable using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Threading; @@ -10,7 +9,6 @@ using Calamari.ArgoCD.Dtos; using Calamari.ArgoCD.Git; using Calamari.ArgoCD.Git.GitVendorApiAdapters; -using Calamari.ArgoCD.GitHub; using Calamari.ArgoCD.Helm; using Calamari.ArgoCD.Models; using Calamari.Common.Commands; @@ -19,7 +17,7 @@ using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; using Calamari.Deployment.Conventions; -using Microsoft.IdentityModel.Tokens; +using Octopus.CoreUtilities.Extensions; namespace Calamari.ArgoCD.Conventions { diff --git a/source/Calamari/ArgoCD/Domain/ApplicationStatus.cs b/source/Calamari/ArgoCD/Domain/ApplicationStatus.cs index ae514006e9..1fed4c0583 100644 --- a/source/Calamari/ArgoCD/Domain/ApplicationStatus.cs +++ b/source/Calamari/ArgoCD/Domain/ApplicationStatus.cs @@ -41,11 +41,11 @@ public class StatusSummary public List Images { get; set; } = new List(); } -// Note: We only support these types currently. Argo offers Kustomize and Plugin as possible types though. public enum SourceType { Directory, Helm, - Kustomize + Kustomize, + Plugin } } diff --git a/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs b/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs index 067cf8b117..84cac7b0b5 100644 --- a/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs +++ b/source/Calamari/ArgoCD/Git/RepositoryWrapper.cs @@ -10,7 +10,7 @@ using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; using LibGit2Sharp; -using Microsoft.IdentityModel.Tokens; +using Octopus.CoreUtilities.Extensions; namespace Calamari.ArgoCD.Git { diff --git a/source/Calamari/Calamari.csproj b/source/Calamari/Calamari.csproj index 7a9c722f93..ad3156338e 100644 --- a/source/Calamari/Calamari.csproj +++ b/source/Calamari/Calamari.csproj @@ -18,7 +18,7 @@ Calamari win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 Calamari.exe.manifest - net462;net6.0 + net462;net8.0 8 CS8632 @@ -46,7 +46,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -58,7 +58,7 @@ - + @@ -89,11 +89,9 @@ PreserveNewest - - - - - + + +