From ee61492f59529d7dfbfe762afce1ebf81c6f6f83 Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Mon, 23 Mar 2026 09:02:24 +0100 Subject: [PATCH 01/13] Introduce Master Workflow Supports MTP testing Supports both Skyline.DataMiner.Sdk & NuGet projects Redirect existing NuGet & App Package workflows towards the Master Workflow Fix security concerns --- ...DataMiner App Packages Master Workflow.yml | 739 +---------- ...nternal NuGet Solution Master Workflow.yml | 443 +------ .github/workflows/Master Workflow.yml | 1155 +++++++++++++++++ .../NuGet Solution Master Workflow.yml | 452 +------ 4 files changed, 1276 insertions(+), 1513 deletions(-) create mode 100644 .github/workflows/Master Workflow.yml diff --git a/.github/workflows/DataMiner App Packages Master Workflow.yml b/.github/workflows/DataMiner App Packages Master Workflow.yml index 0bb171d..2811379 100644 --- a/.github/workflows/DataMiner App Packages Master Workflow.yml +++ b/.github/workflows/DataMiner App Packages Master Workflow.yml @@ -66,720 +66,63 @@ on: overrideCatalogDownloadToken: required: false -env: - VERSION_APPPACKAGEINSTALLER: '4.0.0' - VERSION_SDK: '2.4.6' - jobs: check_deprecated_item: name: Check deprecated items runs-on: ubuntu-latest steps: - name: Check if obsolete inputs are still used - run: | - input_names=("referenceName" "runNumber" "referenceType" "repository" "owner") - - for input_name in "${input_names[@]}"; do - value="${{ inputs.referenceName }}" # placeholder, see note below - case $input_name in - referenceName) value="${{ inputs.referenceName }}" ;; - runNumber) value="${{ inputs.runNumber }}" ;; - referenceType) value="${{ inputs.referenceType }}" ;; - repository) value="${{ inputs.repository }}" ;; - owner) value="${{ inputs.owner }}" ;; - esac + env: + INPUT_REFERENCENAME: ${{ inputs.referenceName }} + INPUT_RUNNUMBER: ${{ inputs.runNumber }} + INPUT_REFERENCETYPE: ${{ inputs.referenceType }} + INPUT_REPOSITORY: ${{ inputs.repository }} + INPUT_OWNER: ${{ inputs.owner }} + run: | + declare -A input_values=( + [referenceName]="$INPUT_REFERENCENAME" + [runNumber]="$INPUT_RUNNUMBER" + [referenceType]="$INPUT_REFERENCETYPE" + [repository]="$INPUT_REPOSITORY" + [owner]="$INPUT_OWNER" + ) - if [ -n "$value" ]; then + for input_name in "${!input_values[@]}"; do + if [ -n "${input_values[$input_name]}" ]; then echo "::warning::The input '$input_name' can be safely removed as this is not required anymore." fi done - name: Check if obsolete secrets are still used if: github.repository_owner == 'SkylineCommunications' + env: + SECRET_AZURETOKEN: ${{ secrets.azureToken }} + SECRET_SONARCLOUDTOKEN: ${{ secrets.sonarCloudToken }} run: | - secret_names=("azureToken" "sonarCloudToken") - - for secret_name in "${secret_names[@]}"; do - value="${{ secrets.sonarCloudToken }}" # placeholder, see note below - case $secret_name in - sonarCloudToken) value="${{ secrets.sonarCloudToken }}" ;; - azureToken) value="${{ secrets.azureToken }}" ;; - esac + declare -A secret_values=( + [azureToken]="$SECRET_AZURETOKEN" + [sonarCloudToken]="$SECRET_SONARCLOUDTOKEN" + ) - if [ -n "$value" ]; then + for secret_name in "${!secret_values[@]}"; do + if [ -n "${secret_values[$secret_name]}" ]; then echo "::warning::The secret '$secret_name' can be safely removed as this is not required anymore." fi done - check_oidc: - name: Check OIDC - runs-on: ubuntu-latest - outputs: - client-id: ${{ steps.set_oidc.outputs.client-id }} - tenant-id: ${{ steps.set_oidc.outputs.tenant-id }} - subscription-id: ${{ steps.set_oidc.outputs.subscription-id }} - use-oidc: ${{ steps.set_oidc.outputs.use-oidc }} - steps: - - name: Set Azure OIDC parameters - id: set_oidc - run: | - echo "Determining Azure OIDC parameters..." - - if [[ -n "${{ inputs.oidc-client-id }}" ]]; then - echo "Using provided OIDC parameters" - { - echo "client-id=${{ inputs.oidc-client-id }}" - echo "tenant-id=${{ inputs.oidc-tenant-id }}" - echo "subscription-id=${{ inputs.oidc-subscription-id }}" - echo "use-oidc=true" - } >> "$GITHUB_OUTPUT" - elif [[ "${{ github.repository_owner }}" == "SkylineCommunications" ]]; then - echo "Using SkylineCommunications default OIDC parameters" - { - echo "client-id=c50da9cc-ba14-4138-8595-a62d97ab0e53" - echo "tenant-id=5f175691-8d1c-4932-b7c8-ce990839ac40" - echo "subscription-id=d6cbb8df-56eb-451d-9db7-67f49cba3220" - echo "use-oidc=true" - } >> "$GITHUB_OUTPUT" - else - echo "No OIDC parameters provided and owner does not match SkylineCommunications" - echo "use-oidc=false" >> "$GITHUB_OUTPUT" - fi - - skyline_ci: - name: Skyline Quality Gate - runs-on: ubuntu-latest - needs: check_oidc - steps: - - name: Azure Login - uses: azure/login@v2 - if: needs.check_oidc.outputs.use-oidc == 'true' - with: - client-id: ${{ needs.check_oidc.outputs.client-id }} - tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} - subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} - - - name: Retrieve needed secrets from Azure Key Vault - if: needs.check_oidc.outputs.use-oidc == 'true' - shell: bash - run: | - echo "Fetching secrets from Azure Key Vault..." - - # List of secret names needed for this job - secret_names=("dataminer-token" "azure-token" "sonar-token") - - for secret_name in "${secret_names[@]}"; do - # Convert to uppercase and replace hyphens with underscores - env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') - - # Retrieve the secret value - secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) - - # Mask the secret value - echo "::add-mask::$secret_value" - - # Export as environment variable - echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" - done - - - name: Overwrite default secrets with user-defined secrets - shell: bash - run: | - if [[ -n "${{ secrets.azureToken }}" ]]; then - echo "Using provided azureToken secret" - echo "AZURE_TOKEN=${{ secrets.azureToken }}" >> "$GITHUB_ENV" - fi - - if [[ -n "${{ secrets.sonarCloudToken }}" ]]; then - echo "Using provided sonarCloudToken secret" - echo "SONAR_TOKEN=${{ secrets.sonarCloudToken }}" >> "$GITHUB_ENV" - fi - - if [[ -n "${{ secrets.dataminerToken }}" ]]; then - echo "Using provided dataminerToken secret" - echo "DATAMINER_TOKEN=${{ secrets.dataminerToken }}" >> "$GITHUB_ENV" - fi - - - name: Enable long paths - run: git config --global core.longpaths true - - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Setup .NET Core - uses: actions/setup-dotnet@v5 - with: - dotnet-version: '8.0.x' - - - name: Cache and Install Mono - uses: awalsh128/cache-apt-pkgs-action@v1.6.0 - with: - packages: mono-complete - - - name: Validate SonarCloud Project Name - id: validate-sonar-name - run: | - if [[ -z "${{ inputs.sonarCloudProjectName }}" ]]; then - echo "Error: sonarCloudProjectName is not set." - echo "Please create a SonarCloud project by visiting: https://sonarcloud.io/projects/create and copy the id of the project as mentioned in the sonarcloud project url." - repo_url="https://github.com/${{ github.repository }}/settings/variables/actions" - echo "Then set a SONAR_NAME variable in your repository settings: $repo_url" - echo "Alternatively, if you do not wish to use the Skyline Quality Gate but intend to publish your results to the catalog, you may create your workflow to include the equivalent of a dotnet publish step as shown below (remove the \\):" - echo " - name: Publish" - echo " env:" - echo " api-key: $\{{ secrets.DATAMINER_TOKEN }}" - echo " run: dotnet publish -p:Version=\"0.0.$\{{ github.run_number }}\" -p:VersionComment=\"Iterative Development\" -p:CatalogPublishKeyName=api-key" - exit 1 - fi - - - name: Validate SonarCloud Secret Token - id: validate-sonar-token - run: | - if [[ -z "${{ env.SONAR_TOKEN }}" ]]; then - echo "Error: sonarCloudToken is not set." - echo "Please create a SonarCloud token by visiting: https://sonarcloud.io/account/security and copy the value of the created token." - repo_url="https://github.com/${{ github.repository }}/settings/secrets/actions" - echo "Then set a SONAR_TOKEN secret in your repository settings: $repo_url" - echo "Alternatively, if you do not wish to use the Skyline Quality Gate but intend to publish your results to the catalog, you may create your workflow to include the equivalent of a dotnet publish step as shown below (remove the \\):" - echo " - name: Publish" - echo " env:" - echo " api-key: $\{{ secrets.DATAMINER_TOKEN }}" - echo " run: dotnet publish -p:Version=\"0.0.$\{{ github.run_number }}\" -p:VersionComment=\"Iterative Development\" -p:CatalogPublishKeyName=api-key" - exit 1 - fi - - - name: Validate DataMiner Secret Token - id: validate-dataminer-token - if: github.ref_type == 'tag' - run: | - if [[ -z "${{ env.DATAMINER_TOKEN }}" ]]; then - echo "Error: dataminerToken is not set. Release not possible!" - echo "Please create or re-use an admin.dataminer.services token by visiting: https://admin.dataminer.services/." - echo "Navigate to the right organization, then go to Keys and create or find a key with the permissions Register catalog items, Download catalog versions, and Read catalog items." - echo "Copy the value of the token." - repo_url="https://github.com/${{ github.repository }}/settings/secrets/actions" - echo "Then set a DATAMINER_TOKEN secret in your repository settings: $repo_url" - exit 1 - fi - - - name: Find solution file - id: findSlnFile - run: | - if [[ -z "${{ inputs.solutionFilterName }}" ]]; then - echo solutionFilePath=$(find . -type f -name '*.sln' -o -name '*.slnx') >> $GITHUB_OUTPUT - else - echo solutionFilePath=$(find . -type f -name '${{ inputs.solutionFilterName }}') >> $GITHUB_OUTPUT - fi - shell: bash - - - name: Enable Skyline NuGet Registries - if: github.repository_owner == 'SkylineCommunications' - run: | - $sources = @( - @{ Name = "PrivateGitHubNugets"; URL = "https://nuget.pkg.github.com/SkylineCommunications/index.json"; Username = "USERNAME"; Password = "${{ secrets.GITHUB_TOKEN }}" }, - @{ Name = "CloudNuGets"; URL = "https://pkgs.dev.azure.com/skyline-cloud/Cloud_NuGets/_packaging/CloudNuGet/nuget/v3/index.json"; Username = "az"; Password = "${{ env.AZURE_TOKEN }}" }, - @{ Name = "PrivateAzureNuGets"; URL = "https://pkgs.dev.azure.com/skyline-cloud/_packaging/skyline-private-nugets/nuget/v3/index.json"; Username = "az"; Password = "${{ env.AZURE_TOKEN }}" } - ) - - foreach ($source in $sources) { - if ($source.Password -ne "") { - Write-Host "Checking source $($source.Name)..." - - if (dotnet nuget list source | Select-String -Pattern $source.Name) { - Write-Host "Updating existing source $($source.Name)." - dotnet nuget update source $source.Name --source $source.URL --username $source.Username --password $source.Password --store-password-in-clear-text - } else { - Write-Host "Adding new source $($source.Name)." - dotnet nuget add source $source.URL --name $source.Name --username $source.Username --password $source.Password --store-password-in-clear-text - } - } else { - Write-Host "Skipping $($source.Name) because the password is not set." - } - } - shell: pwsh - - - name: Install Tools - run: | - dotnet tool install dotnet-sonarscanner --global - dotnet tool install Skyline.DataMiner.CICD.Tools.Sbom --global --version 1.* - dotnet tool install Skyline.DataMiner.CICD.Tools.NuGetChangeVersion --global --version 2.* - - - name: Update Skyline.DataMiner.Core.AppPackages - run: NuGetChangeVersion --name Skyline.DataMiner.Core.AppPackageInstaller --workspace "${{ github.workspace }}" --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" --nugetVersion $VERSION_APPPACKAGEINSTALLER - - - name: Update Skyline.DataMiner.Sdk version in global.json (if present) - shell: pwsh - run: | - $globalJsonPath = Join-Path $env:GITHUB_WORKSPACE "global.json" - - if (-Not (Test-Path $globalJsonPath)) { - Write-Host "No global.json found. Skipping update." - return - } - - $jsonContent = Get-Content $globalJsonPath -Raw | ConvertFrom-Json - - if (-not $jsonContent.'msbuild-sdks') { - $jsonContent | Add-Member -MemberType NoteProperty -Name 'msbuild-sdks' -Value @{} - } - - $jsonContent.'msbuild-sdks'.'Skyline.DataMiner.Sdk' = $env:VERSION_SDK - - $updatedJson = $jsonContent | ConvertTo-Json -Depth 10 - - $updatedJson | Set-Content $globalJsonPath -Encoding UTF8 - - Write-Host "Updated global.json:" - Write-Host $updatedJson - - - name: Update Catalog Identifiers - if: ${{ inputs.overrideCatalogIdentifiers != '' }} - run: | - $rawMappings = @" - ${{ inputs.overrideCatalogIdentifiers }} - "@ - $mappings = $rawMappings -split '[\r\n]+' | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne '' } - - foreach ($mapping in $mappings) { - $splitIndex = $mapping.IndexOf('=') - if ($splitIndex -lt 1 -or $splitIndex -eq ($mapping.Length - 1)) { - Write-Error "Invalid entry '$mapping'. Expected format: =." - exit 1 - } - - $manifestPath = $mapping.Substring(0, $splitIndex).Trim() - $identifier = $mapping.Substring($splitIndex + 1).Trim() - - if ($identifier -notmatch '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$') { - Write-Error "Identifier '$identifier' is not a valid GUID." - exit 1 - } - - if ([System.IO.Path]::GetFileName($manifestPath) -ne 'manifest.yml') { - Write-Error "Manifest path '$manifestPath' must point to 'manifest.yml'." - exit 1 - } - - if (-not (Test-Path -Path $manifestPath -PathType Leaf)) { - Write-Error "Manifest file '$manifestPath' does not exist." - exit 1 - } - - $content = Get-Content -Path $manifestPath -Raw - $hasActiveIdLine = [regex]::IsMatch($content, '(?m)^(?!\s*#)\s*id:\s*.*$') - - if (-not $hasActiveIdLine) { - Write-Error "No active id line found in '$manifestPath'." - exit 1 - } - - $updatedContent = [regex]::Replace($content, '(?m)^(?!\s*#)\s*id:\s*.*$', "id: $identifier", 1) - - if ($updatedContent -eq $content) { - Write-Host "'$manifestPath' already has id: $identifier" - } else { - Set-Content -Path $manifestPath -Value $updatedContent -NoNewline - Write-Host "Updated '$manifestPath' to id: $identifier" - } - } - shell: pwsh - - - name: Prepare SonarCloud Variables - id: prepSonarCloudVar - run: | - import os - env_file = os.getenv('GITHUB_ENV') - with open(env_file, "a") as myfile: - myfile.write("lowerCaseOwner=" + str.lower("${{ github.repository_owner }}")) - shell: python - - - name: Get SonarCloud Status - id: get-sonarcloud-status - run: | - sonarCloudProjectStatus=$(curl -s -u "${{ env.SONAR_TOKEN }}:" "https://sonarcloud.io/api/qualitygates/project_status?projectKey=${{ inputs.sonarCloudProjectName }}&branch=${{ github.ref_name }}") - - # Check if the response is empty or not valid JSON - if [ -z "$sonarCloudProjectStatus" ] || ! echo "$sonarCloudProjectStatus" | jq . > /dev/null 2>&1; then - echo "Error: The SONAR_TOKEN is invalid, expired, or the response is empty. Please check: https://sonarcloud.io/account/security and update your token: https://github.com/${{ github.repository }}/settings/secrets/actions" >&2 - echo "Returned response: $sonarCloudProjectStatus" >&2 - exit 1 - fi - - # Check if the response contains errors - if echo "$sonarCloudProjectStatus" | jq -e '.errors' > /dev/null 2>&1; then - echo "Error: SonarCloud API returned errors. Initial analysis needed." >&2 - echo "Returned response: $sonarCloudProjectStatus" >&2 - echo "needsInitialAnalysis=true" >> $GITHUB_OUTPUT - exit 0 - fi - - # Check if project status is NONE (needs initial analysis) - projectStatus=$(echo "$sonarCloudProjectStatus" | jq -r '.projectStatus.status // empty') - if [ "$projectStatus" = "NONE" ]; then - echo "Project status is NONE. Initial analysis needed." - echo "needsInitialAnalysis=true" >> $GITHUB_OUTPUT - else - echo "needsInitialAnalysis=false" >> $GITHUB_OUTPUT - fi - - # Output the JSON response if valid - echo "Returned response: $sonarCloudProjectStatus" - shell: bash - - - name: Apply SourceCode Url To Manifest - run: | - $manifestFiles = Get-ChildItem -Recurse -Filter 'manifest.yml' | - Where-Object { $_.FullName -match '[\\/]CatalogInformation[\\/]' } - - foreach ($file in $manifestFiles) { - $lines = Get-Content -Path $file.FullName - $updated = $false - - for ($i = 0; $i -lt $lines.Count; $i++) { - if ($lines[$i] -match '^\s*source_code_url:\s*$') { - $indent = ($lines[$i] -match '^(\s*)source_code_url:')[1] - $lines[$i] = "$indent" + "source_code_url: 'https://github.com/${{ github.repository }}'" - $updated = $true - break - } - } - - if ($updated) { - Write-Host "Updating: $($file.FullName) with 'source_code_url: https://github.com/${{ github.repository }}'" - Set-Content -Path $file.FullName -Value $lines -Encoding UTF8 - } - } - shell: pwsh - - - name: Trigger Initial Analysis - if: ${{ steps.get-sonarcloud-status.outputs.needsInitialAnalysis == 'true' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner begin ` - /k:"${{ inputs.sonarCloudProjectName }}" ` - /o:"${{ env.lowerCaseOwner }}" ` - /d:sonar.token="${{ env.SONAR_TOKEN }}" ` - /d:sonar.host.url="https://sonarcloud.io" ` - /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" ` - /d:sonar.cs.vstest.reportsPaths="**/TestResults/**.trx" ` - /d:sonar.exclusions="**/*.yml,**/*.xml" ` - /d:sonar.branch.name="${{ github.ref_name }}" - dotnet build "${{ steps.findSlnFile.outputs.solutionFilePath }}" ` - -p:GenerateDataMinerPackage=false ` - --configuration ${{ inputs.configuration }} ` - -nodeReuse:false - dotnet sonarscanner end /d:sonar.token="${{ env.SONAR_TOKEN }}" - continue-on-error: true - shell: pwsh - - - name: Start Analysis - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner begin ` - /k:"${{ inputs.sonarCloudProjectName }}" ` - /o:"${{ env.lowerCaseOwner }}" ` - /d:sonar.token="${{ env.SONAR_TOKEN }}" ` - /d:sonar.host.url="https://sonarcloud.io" ` - /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" ` - /d:sonar.cs.vstest.reportsPaths="**/TestResults/**.trx" ` - /d:sonar.exclusions="**/*.yml,**/*.xml" ` - /d:sonar.branch.name="${{ github.ref_name }}" - continue-on-error: true - shell: pwsh - - - name: Build for pre-release - if: github.ref_type == 'branch' - env: - OVERRIDE_CATALOG_DOWNLOAD_TOKEN: ${{ secrets.overrideCatalogDownloadToken }} - run: | - dotnet build "${{ steps.findSlnFile.outputs.solutionFilePath }}" ` - -p:Version="0.0.${{ github.run_number }}" ` - --configuration ${{ inputs.configuration }} ` - -p:CatalogPublishKeyName="DATAMINER_TOKEN" ` - -p:CatalogDefaultDownloadKeyName="OVERRIDE_CATALOG_DOWNLOAD_TOKEN" ` - -p:SkylineDataMinerSdkDebug="${{ inputs.debug }}" ` - -nodeReuse:false - shell: pwsh - - - name: Build for release - if: github.ref_type == 'tag' - env: - OVERRIDE_CATALOG_DOWNLOAD_TOKEN: ${{ secrets.overrideCatalogDownloadToken }} - run: | - dotnet build "${{ steps.findSlnFile.outputs.solutionFilePath }}" ` - -p:Version="${{ github.ref_name }}" ` - --configuration ${{ inputs.configuration }} ` - -p:CatalogPublishKeyName="DATAMINER_TOKEN" ` - -p:CatalogDefaultDownloadKeyName="OVERRIDE_CATALOG_DOWNLOAD_TOKEN" ` - -p:SkylineDataMinerSdkDebug="${{ inputs.debug }}" ` - -nodeReuse:false - shell: pwsh - - - name: Unit Tests - # when not using MSTest you'll need to install coverlet.collector nuget in your test solutions - id: unit-tests - run: dotnet test "${{ steps.findSlnFile.outputs.solutionFilePath }}" --no-build --configuration ${{ inputs.configuration }} --filter TestCategory!=IntegrationTest --logger "trx;logfilename=unitTestResults.trx" --collect "XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura,opencover - - - name: Stop Analysis - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner end /d:sonar.token="${{ env.SONAR_TOKEN }}" - continue-on-error: true - - - name: SonarCloud Quality Gate check - id: sonarcloud-quality-gate-check - uses: sonarsource/sonarqube-quality-gate-action@master - with: - scanMetadataReportFile: .sonarqube/out/.sonar/report-task.txt - continue-on-error: true - timeout-minutes: 5 - - - name: Upload Sonar report-task.txt artifact - continue-on-error: true - uses: actions/upload-artifact@v6 - with: - name: sonar-report-task - path: .sonarqube/out/.sonar/report-task.txt - if-no-files-found: ignore - retention-days: 14 - - - name: SonarCloud Quality Gate - id: quality-step - run: | - if "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" == "FAILED": - print("Code Analysis Quality gate failed.") - exit(1) - if "${{ steps.sonarcloud-quality-gate-check.outcome }}" == "failure" and "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" != "FAILED": - print("- Could not retrieve SonarCloud quality gate status, potentially due to reaching License max LoC. Ignoring SonarCloud quality gate status in this case.") - shell: python - - - name: Create package name - id: packageName - run: | - $tempName = "${{ github.repository }}" - $safeName = $tempName -replace '[\"\/\\<>|:*?]', '_' - echo "name=$safeName" >> $env:GITHUB_OUTPUT - shell: pwsh - - - name: Generate SBOM file - run: | - find . -type f \( -name "*.dmapp" -o -name "*.dmtest" \) -print0 | while IFS= read -r -d '' file; do - echo "Generating SBOM for $file" - dataminer-sbom generate-and-add \ - --solution-path "${{ steps.findSlnFile.outputs.solutionFilePath }}" \ - --package-file "$file" \ - --package-name "${{ steps.packageName.outputs.name }}" \ - --package-version "${{ github.ref_name }}" \ - --package-supplier "Skyline Communications" \ - --debug "${{ inputs.debug }}" - done - - - uses: actions/upload-artifact@v6 - with: - name: DataMiner Installation Packages (${{ inputs.configuration }} ${{ inputs.solutionFilterName }}) unsigned - path: | - **/bin/${{ inputs.configuration }}/*.dmapp - **/bin/${{ inputs.configuration }}/*.dmtest - **/bin/${{ inputs.configuration }}/*.zip - **/bin/${{ inputs.configuration }}/**/*.dmapp - **/bin/${{ inputs.configuration }}/**/*.dmtest - **/bin/${{ inputs.configuration }}/**/*.zip - continue-on-error: true - - skyline_cd: - name: Skyline Catalog Release - runs-on: windows-latest - if: github.ref_type == 'tag' - needs: [skyline_ci, check_oidc] - steps: - - name: Azure Login - uses: azure/login@v2 - if: needs.check_oidc.outputs.use-oidc == 'true' - with: - client-id: ${{ needs.check_oidc.outputs.client-id }} - tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} - subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} - - - name: Retrieve needed secrets from Azure Key Vault - if: needs.check_oidc.outputs.use-oidc == 'true' - shell: bash - run: | - echo "Fetching secrets from Azure Key Vault..." - - # List of secret names needed for this job - secret_names=("dataminer-token" "signing-client-id" "signing-client-secret" "signing-tenant-id" "signing-key-vault-certificate" "signing-key-vault-url") - - for secret_name in "${secret_names[@]}"; do - # Convert to uppercase and replace hyphens with underscores - env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') - - # Retrieve the secret value - secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) - - # Mask the secret value - echo "::add-mask::$secret_value" - - # Export as environment variable - echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" - done - - - name: Overwrite default secrets with user-defined secrets - shell: bash - run: | - if [[ -n "${{ secrets.dataminerToken }}" ]]; then - echo "Using provided dataminerToken secret" - echo "DATAMINER_TOKEN=${{ secrets.dataminerToken }}" >> "$GITHUB_ENV" - fi - - - name: Enable long paths - run: git config --global core.longpaths true - - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Find solution file - id: findSlnFile - run: | - if [[ -z "${{ inputs.solutionFilterName }}" ]]; then - echo solutionFilePath=$(find . -type f -name '*.sln' -o -name '*.slnx') >> $GITHUB_OUTPUT - else - echo solutionFilePath=$(find . -type f -name '${{ inputs.solutionFilterName }}') >> $GITHUB_OUTPUT - fi - shell: bash - -# Alternative option here, is to perform the dotnet build again, but that needs all the nuget access again, etc. - - name: Download artifact from CI - uses: actions/download-artifact@v7 - with: - name: DataMiner Installation Packages (${{ inputs.configuration }} ${{ inputs.solutionFilterName }}) unsigned - path: downloaded_artifacts - - - name: Extract artifact contents putting them back in the bin folders. - run: | - Write-Output "Extracting all zip, dmapp and dmtest files and restoring original structure..." - - $downloadRoot = Resolve-Path "downloaded_artifacts" - $workspaceRoot = Resolve-Path "." - - Get-ChildItem -Path $downloadRoot -Recurse -File | Where-Object { $_.Extension -in '.zip', '.dmapp', '.dmtest' } | ForEach-Object { - $archiveFile = $_.FullName - $relativePath = $archiveFile.Substring($downloadRoot.Path.Length + 1) -replace '\\', '/' - $targetPath = Join-Path $workspaceRoot $relativePath - - # Ensure target directory exists - $targetFolder = Split-Path $targetPath - if (-not (Test-Path $targetFolder)) { - New-Item -ItemType Directory -Path $targetFolder -Force | Out-Null - } - - # Move the file - Move-Item -Path $archiveFile -Destination $targetPath -Force - Write-Output "Moved $relativePath to workspace root" - } - - # Remove the downloaded_artifacts directory after migration - Write-Output "Removing downloaded_artifacts folder..." - Remove-Item -Path $downloadRoot -Recurse -Force - Write-Output "Cleanup complete." - shell: pwsh - - - name: Install Tools - if: needs.check_oidc.outputs.use-oidc == 'true' - run: | - dotnet tool install Skyline.DataMiner.CICD.Tools.PackageSign --global --version 2.* - - - name: Sign generated dmapp/dmtest packages - if: needs.check_oidc.outputs.use-oidc == 'true' - shell: pwsh - env: - AZURE_TENANT_ID: ${{ env.SIGNING_TENANT_ID }} - AZURE_CLIENT_ID: ${{ env.SIGNING_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ env.SIGNING_CLIENT_SECRET }} - AZURE_KEY_VAULT_URL: ${{ env.SIGNING_KEY_VAULT_URL }} - AZURE_KEY_VAULT_CERTIFICATE: ${{ env.SIGNING_KEY_VAULT_CERTIFICATE }} - run: | - dataminer-package-signature sign dmapp ` - --package-location "${{ github.workspace }}" ` - --debug ${{ inputs.debug }} - - - name: Authenticate with GitHub CLI - shell: pwsh - run: | - "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token - - - name: Find Version Comment - id: findVersionComment - run: | - echo "Checking for release notes associated with the reference: '${{ github.ref_name }}'" - - # Retrieve the release information (both body and name) - RELEASE_INFO=$(gh release view "${{ github.ref_name }}" --json body,name 2>/dev/null || echo "{}") - RELEASE_NOTE=$(echo "$RELEASE_INFO" | jq -r '.body // ""' 2>/dev/null || echo "") - RELEASE_TITLE=$(echo "$RELEASE_INFO" | jq -r '.name // ""' 2>/dev/null || echo "") - - escape_special_chars() { - echo "$1" | sed -e 's/,/%2c/g' -e 's/"/%22/g' -e 's/;/%3b/g' - } - - if [[ -n "$RELEASE_NOTE" ]]; then - ESCAPED_RELEASE_NOTE=$(escape_special_chars "$RELEASE_NOTE") - echo "Release description found for '${{ github.ref_name }}': $ESCAPED_RELEASE_NOTE" - # Escape multiline string for GITHUB_OUTPUT - echo "versionComment<> $GITHUB_OUTPUT - echo "$ESCAPED_RELEASE_NOTE" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - elif [[ -n "$RELEASE_TITLE" ]]; then - ESCAPED_RELEASE_TITLE=$(escape_special_chars "$RELEASE_TITLE") - echo "Release title found for '${{ github.ref_name }}': $ESCAPED_RELEASE_TITLE" - echo "versionComment=$ESCAPED_RELEASE_TITLE" >> $GITHUB_OUTPUT - else - echo "No release description or title found for '${{ github.ref_name }}'. Falling back to tag or commit message." - VERSION_COMMENT=$(git describe --tags --exact-match 2>/dev/null || git log -1 --pretty=format:%s) - ESCAPED_VERSION_COMMENT=$(escape_special_chars "$VERSION_COMMENT") - echo "Fallback version comment: $ESCAPED_VERSION_COMMENT" - echo "versionComment=$ESCAPED_VERSION_COMMENT" >> $GITHUB_OUTPUT - fi - shell: bash - - - name: Update Skyline.DataMiner.Sdk version in global.json (if present) - shell: pwsh - run: | - $globalJsonPath = Join-Path $env:GITHUB_WORKSPACE "global.json" - - if (-Not (Test-Path $globalJsonPath)) { - Write-Host "No global.json found. Skipping update." - return - } - - $jsonContent = Get-Content $globalJsonPath -Raw | ConvertFrom-Json - - if (-not $jsonContent.'msbuild-sdks') { - $jsonContent | Add-Member -MemberType NoteProperty -Name 'msbuild-sdks' -Value @{} - } - - $jsonContent.'msbuild-sdks'.'Skyline.DataMiner.Sdk' = $env:VERSION_SDK - - $updatedJson = $jsonContent | ConvertTo-Json -Depth 10 - - $updatedJson | Set-Content $globalJsonPath -Encoding UTF8 - - Write-Host "Updated global.json:" - Write-Host $updatedJson - - - name: Publish To Catalog - shell: pwsh - run: | - dotnet publish ` - "${{ steps.findSlnFile.outputs.solutionFilePath }}" ` - --no-build ` - -p:Version="${{ github.ref_name }}" ` - -p:VersionComment="${{ steps.findVersionComment.outputs.versionComment }}" ` - -p:CatalogPublishKeyName="DATAMINER_TOKEN" ` - --configuration ${{ inputs.configuration }} ` - -p:SkylineDataMinerSdkDebug="${{ inputs.debug }}" ` - -p:IsPublishable=false + master_workflow: + uses: ./.github/workflows/Master Workflow.yml + with: + oidc-client-id: ${{ inputs.oidc-client-id }} + oidc-tenant-id: ${{ inputs.oidc-tenant-id }} + oidc-subscription-id: ${{ inputs.oidc-subscription-id }} + sonarcloud-project-name: ${{ inputs.sonarCloudProjectName }} + configuration: ${{ inputs.configuration }} + solution-filter-name: ${{ inputs.solutionFilterName }} + override-catalog-identifiers: ${{ inputs.overrideCatalogIdentifiers }} + debug: ${{ inputs.debug }} + secrets: + SONAR_TOKEN: ${{ secrets.sonarCloudToken }} + AZURE_TOKEN: ${{ secrets.azureToken }} + DATAMINER_TOKEN: ${{ secrets.dataminerToken }} + OVERRIDE_CATALOG_DOWNLOAD_TOKEN: ${{ secrets.overrideCatalogDownloadToken }} \ No newline at end of file diff --git a/.github/workflows/Internal NuGet Solution Master Workflow.yml b/.github/workflows/Internal NuGet Solution Master Workflow.yml index 008a678..5f3db4b 100644 --- a/.github/workflows/Internal NuGet Solution Master Workflow.yml +++ b/.github/workflows/Internal NuGet Solution Master Workflow.yml @@ -5,11 +5,6 @@ permissions: write-all on: workflow_call: - outputs: - quality_gate: - description: "Results from Skyline Quality Gate." - value: ${{ jobs.validate_skyline_quality_gate.outputs.quality }} - inputs: # No need to specify these as the workflow can access github.* (https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/accessing-contextual-information-about-workflow-runs#github-context) # When a reusable workflow is triggered by a caller workflow, the github context is always associated with the caller workflow. @@ -61,417 +56,53 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if obsolete inputs are still used - run: | - input_names=("referenceName" "runNumber" "referenceType" "repository" "owner") - - for input_name in "${input_names[@]}"; do - value="${{ inputs.referenceName }}" # placeholder, see note below - case $input_name in - referenceName) value="${{ inputs.referenceName }}" ;; - runNumber) value="${{ inputs.runNumber }}" ;; - referenceType) value="${{ inputs.referenceType }}" ;; - repository) value="${{ inputs.repository }}" ;; - owner) value="${{ inputs.owner }}" ;; - esac + env: + INPUT_REFERENCENAME: ${{ inputs.referenceName }} + INPUT_RUNNUMBER: ${{ inputs.runNumber }} + INPUT_REFERENCETYPE: ${{ inputs.referenceType }} + INPUT_REPOSITORY: ${{ inputs.repository }} + INPUT_OWNER: ${{ inputs.owner }} + run: | + declare -A input_values=( + [referenceName]="$INPUT_REFERENCENAME" + [runNumber]="$INPUT_RUNNUMBER" + [referenceType]="$INPUT_REFERENCETYPE" + [repository]="$INPUT_REPOSITORY" + [owner]="$INPUT_OWNER" + ) - if [ -n "$value" ]; then + for input_name in "${!input_values[@]}"; do + if [ -n "${input_values[$input_name]}" ]; then echo "::warning::The input '$input_name' can be safely removed as this is not required anymore." fi done - name: Check if obsolete secrets are still used if: github.repository_owner == 'SkylineCommunications' + env: + SECRET_AZURETOKEN: ${{ secrets.azureToken }} + SECRET_SONARCLOUDTOKEN: ${{ secrets.sonarCloudToken }} run: | - secret_names=("azureToken" "sonarCloudToken") - - for secret_name in "${secret_names[@]}"; do - value="${{ secrets.sonarCloudToken }}" # placeholder, see note below - case $secret_name in - sonarCloudToken) value="${{ secrets.sonarCloudToken }}" ;; - azureToken) value="${{ secrets.azureToken }}" ;; - esac + declare -A secret_values=( + [azureToken]="$SECRET_AZURETOKEN" + [sonarCloudToken]="$SECRET_SONARCLOUDTOKEN" + ) - if [ -n "$value" ]; then + for secret_name in "${!secret_values[@]}"; do + if [ -n "${secret_values[$secret_name]}" ]; then echo "::warning::The secret '$secret_name' can be safely removed as this is not required anymore." fi done - check_oidc: - name: Check OIDC - runs-on: ubuntu-latest - outputs: - client-id: ${{ steps.set_oidc.outputs.client-id }} - tenant-id: ${{ steps.set_oidc.outputs.tenant-id }} - subscription-id: ${{ steps.set_oidc.outputs.subscription-id }} - use-oidc: ${{ steps.set_oidc.outputs.use-oidc }} - steps: - - name: Set Azure OIDC parameters - id: set_oidc - run: | - echo "Determining Azure OIDC parameters..." - - if [[ -n "${{ inputs.oidc-client-id }}" ]]; then - echo "Using provided OIDC parameters" - { - echo "client-id=${{ inputs.oidc-client-id }}" - echo "tenant-id=${{ inputs.oidc-tenant-id }}" - echo "subscription-id=${{ inputs.oidc-subscription-id }}" - echo "use-oidc=true" - } >> "$GITHUB_OUTPUT" - elif [[ "${{ github.repository_owner }}" == "SkylineCommunications" ]]; then - echo "Using SkylineCommunications default OIDC parameters" - { - echo "client-id=c50da9cc-ba14-4138-8595-a62d97ab0e53" - echo "tenant-id=5f175691-8d1c-4932-b7c8-ce990839ac40" - echo "subscription-id=d6cbb8df-56eb-451d-9db7-67f49cba3220" - echo "use-oidc=true" - } >> "$GITHUB_OUTPUT" - else - echo "No OIDC parameters provided and owner does not match SkylineCommunications" - echo "use-oidc=false" >> "$GITHUB_OUTPUT" - fi - - validate_skyline_quality_gate: - name: Skyline Quality Gate - runs-on: windows-latest - needs: check_oidc - outputs: - quality: ${{ steps.quality-step.outputs.results }} - steps: - - name: Azure Login - uses: azure/login@v2 - if: needs.check_oidc.outputs.use-oidc == 'true' - with: - client-id: ${{ needs.check_oidc.outputs.client-id }} - tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} - subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} - - - name: Retrieve needed secrets from Azure Key Vault - if: needs.check_oidc.outputs.use-oidc == 'true' - shell: bash - run: | - echo "Fetching secrets from Azure Key Vault..." - - # List of secret names needed for this job - secret_names=("azure-token" "sonar-token") - - for secret_name in "${secret_names[@]}"; do - # Convert to uppercase and replace hyphens with underscores - env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') - - # Retrieve the secret value - secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) - - # Mask the secret value - echo "::add-mask::$secret_value" - - # Export as environment variable - echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" - done - - - name: Overwrite default secrets with user-defined secrets - shell: bash - run: | - if [[ -n "${{ secrets.azureToken }}" ]]; then - echo "Using provided azureToken secret" - echo "AZURE_TOKEN=${{ secrets.azureToken }}" >> "$GITHUB_ENV" - fi - - if [[ -n "${{ secrets.sonarCloudToken }}" ]]; then - echo "Using provided sonarCloudToken secret" - echo "SONAR_TOKEN=${{ secrets.sonarCloudToken }}" >> "$GITHUB_ENV" - fi - - - name: Enable long paths - run: git config --global core.longpaths true - - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - java-version: 17 - distribution: 'zulu' - - - name: Install .NET Tools - run: | - dotnet tool install -g dotnet-sonarscanner - dotnet tool install -g Skyline.DataMiner.CICD.Tools.NuGetToggleOnBuild - dotnet tool install -g Skyline.DataMiner.CICD.Tools.NuGetPreBuildApplyBranchOrTag - dotnet tool install -g Skyline.DataMiner.CICD.Tools.NuGetValidateSkylineSpecifications - - - name: Find solution file - id: findSlnFile - run: | - if [[ -z "${{ inputs.solutionFilterName }}" ]]; then - echo solutionFilePath=$(find . -type f -name '*.sln' -o -name '*.slnx' | sort -r | head -n 1) >> $GITHUB_OUTPUT - else - echo solutionFilePath=$(find . -type f -name '${{ inputs.solutionFilterName }}') >> $GITHUB_OUTPUT - fi - shell: bash - - - name: Validate NuGet Metadata - run: NuGetValidateSkylineSpecifications --workspace ${{ github.workspace }} --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" - - - name: Apply Branch and output path for pre-release NuGet - if: github.ref_type == 'branch' - run: NuGetPreBuildApplyBranchOrTag --workspace ${{ github.workspace }} --tag " " --branch "${{ github.ref_name }}" --build ${{ github.run_number }} --nugetResultFolder "${{ github.workspace }}/_NuGetResults" --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" - - - name: Apply Tag and output path for Release NuGet - if: github.ref_type == 'tag' - run: NuGetPreBuildApplyBranchOrTag --workspace ${{ github.workspace }} --tag "${{ github.ref_name }}" --branch " " --build ${{ github.run_number }} --nugetResultFolder "${{ github.workspace }}/_NuGetResults" --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" - - - name: Enable Skyline NuGet Registries - if: github.repository_owner == 'SkylineCommunications' - run: | - $sources = @( - @{ Name = "PrivateGitHubNugets"; URL = "https://nuget.pkg.github.com/SkylineCommunications/index.json"; Username = "USERNAME"; Password = "${{ secrets.GITHUB_TOKEN }}" }, - @{ Name = "CloudNuGets"; URL = "https://pkgs.dev.azure.com/skyline-cloud/Cloud_NuGets/_packaging/CloudNuGet/nuget/v3/index.json"; Username = "az"; Password = "${{ env.AZURE_TOKEN }}" }, - @{ Name = "PrivateAzureNuGets"; URL = "https://pkgs.dev.azure.com/skyline-cloud/_packaging/skyline-private-nugets/nuget/v3/index.json"; Username = "az"; Password = "${{ env.AZURE_TOKEN }}" } - ) - - foreach ($source in $sources) { - if ($source.Password -ne "") { - Write-Host "Checking source $($source.Name)..." - - if (dotnet nuget list source | Select-String -Pattern $source.Name) { - Write-Host "Updating existing source $($source.Name)." - dotnet nuget update source $source.Name --source $source.URL --username $source.Username --password $source.Password --store-password-in-clear-text - } else { - Write-Host "Adding new source $($source.Name)." - dotnet nuget add source $source.URL --name $source.Name --username $source.Username --password $source.Password --store-password-in-clear-text - } - } else { - Write-Host "Skipping $($source.Name) because the password is not set." - } - } - shell: pwsh - - - name: Prepare SonarCloud Variables - id: prepSonarCloudVar - run: | - import os - env_file = os.getenv('GITHUB_ENV') - with open(env_file, "a") as myfile: - myfile.write("lowerCaseOwner=" + str.lower("${{ github.repository_owner }}")) - shell: python - - - name: Get SonarCloud Status - id: get-sonarcloud-status - run: | - sonarCloudProjectStatus=$(curl -s -u "${{ env.SONAR_TOKEN }}:" "https://sonarcloud.io/api/qualitygates/project_status?projectKey=${{ inputs.sonarCloudProjectName }}&branch=${{ github.ref_name }}") - - # Check if the response is empty or not valid JSON - if [ -z "$sonarCloudProjectStatus" ] || ! echo "$sonarCloudProjectStatus" | jq . > /dev/null 2>&1; then - echo "Error: The SONAR_TOKEN is invalid, expired, or the response is empty. Please check: https://sonarcloud.io/account/security and update your token: https://github.com/${{ github.repository }}/settings/secrets/actions" >&2 - echo "Returned response: $sonarCloudProjectStatus" >&2 - exit 1 - fi - - # Check if the response contains errors - if echo "$sonarCloudProjectStatus" | jq -e '.errors' > /dev/null 2>&1; then - echo "Error: SonarCloud API returned errors. Initial analysis needed." >&2 - echo "Returned response: $sonarCloudProjectStatus" >&2 - echo "needsInitialAnalysis=true" >> $GITHUB_OUTPUT - exit 0 - fi - - # Check if project status is NONE (needs initial analysis) - projectStatus=$(echo "$sonarCloudProjectStatus" | jq -r '.projectStatus.status // empty') - if [ "$projectStatus" = "NONE" ]; then - echo "Project status is NONE. Initial analysis needed." - echo "needsInitialAnalysis=true" >> $GITHUB_OUTPUT - else - echo "needsInitialAnalysis=false" >> $GITHUB_OUTPUT - fi - - # Output the JSON response if valid - echo "Returned response: $sonarCloudProjectStatus" - shell: bash - - - name: Trigger Initial Analysis - if: steps.get-sonarcloud-status.outputs.needsInitialAnalysis == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner begin ` - /k:"${{ inputs.sonarCloudProjectName }}" ` - /o:"${{ env.lowerCaseOwner }}" ` - /d:sonar.token="${{ env.SONAR_TOKEN }}" ` - /d:sonar.host.url="https://sonarcloud.io" ` - /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" ` - /d:sonar.cs.vstest.reportsPaths="**/TestResults/**.trx" ` - /d:sonar.exclusions="**/*.yml,**/*.xml" ` - /d:sonar.branch.name="${{ github.ref_name }}" - dotnet build "${{ steps.findSlnFile.outputs.solutionFilePath }}" -p:DefineConstants="DCFv1%3BDBInfo%3BALARM_SQUASHING" --configuration Release -nodeReuse:false - dotnet sonarscanner end /d:sonar.token="${{ env.SONAR_TOKEN }}" - continue-on-error: true - - - name: Start Analysis - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner begin ` - /k:"${{ inputs.sonarCloudProjectName }}" ` - /o:"${{ env.lowerCaseOwner }}" ` - /d:sonar.token="${{ env.SONAR_TOKEN }}" ` - /d:sonar.host.url="https://sonarcloud.io" ` - /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" ` - /d:sonar.cs.vstest.reportsPaths="**/TestResults/**.trx" ` - /d:sonar.exclusions="**/*.yml,**/*.xml" ` - /d:sonar.branch.name="${{ github.ref_name }}" - continue-on-error: true - - - name: Building - run: dotnet build "${{ steps.findSlnFile.outputs.solutionFilePath }}" -p:DefineConstants="DCFv1%3BDBInfo%3BALARM_SQUASHING" --configuration Release -nodeReuse:false - - - uses: actions/upload-artifact@v6 - with: - name: NugetPackages - path: "${{ github.workspace }}/_NuGetResults" - - - name: Disable NuGet Creation on Subsequent Builds - run: NuGetToggleOnBuild --setToActive false --workspace ${{ github.workspace }} --onlyOnConfiguredNuGet false --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" - - - name: Unit Tests - # when not using MSTest you'll need to install coverlet.collector nuget in your test solutions - id: unit-tests - run: dotnet test "${{ steps.findSlnFile.outputs.solutionFilePath }}" --filter TestCategory!=IntegrationTest --logger "trx;logfilename=unitTestResults.trx" --collect "XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura,opencover - continue-on-error: true - - - name: Stop Analysis - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner end /d:sonar.token="${{ env.SONAR_TOKEN }}" - continue-on-error: true - - - name: SonarCloud Quality Gate check - id: sonarcloud-quality-gate-check - uses: sonarsource/sonarqube-quality-gate-action@master - with: - scanMetadataReportFile: .sonarqube/out/.sonar/report-task.txt - continue-on-error: true - # Force to fail step after specific time. - timeout-minutes: 5 - - - name: Upload Sonar report-task.txt artifact - continue-on-error: true - uses: actions/upload-artifact@v6 - with: - name: sonar-report-task - path: .sonarqube/out/.sonar/report-task.txt - if-no-files-found: ignore - retention-days: 14 - - - name: Quality Gate - id: quality-step - run: | - if "${{ steps.unit-tests.outcome }}" == "failure" or "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" == "FAILED": - print("Quality gate failed due to:") - if "${{ steps.unit-tests.outcome }}" == "failure": - print("- Test failures") - if "${{ steps.sonarcloud-quality-gate-check.outcome }}" == "failure" and "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" != "FAILED": - print("- Could not retrieve SonarCloud quality gate status, potentially due to reaching License max LoC. Ignoring SonarCloud quality gate status in this case.") - if "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" == "FAILED": - print("- Code analysis quality gate failed") - if "${{ steps.unit-tests.outcome }}" == "failure" or "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" == "FAILED": - exit(1) - shell: python - - # Signing cannot be done from linux environment (https://github.com/dotnet/runtime/issues/48794) - sign: - # Don't run the signing when dependabot branch/pull request - if: ${{ github.actor != 'dependabot[bot]' }} - runs-on: windows-latest - needs: [validate_skyline_quality_gate, check_oidc] - steps: - - name: Azure Login - uses: azure/login@v2 - if: needs.check_oidc.outputs.use-oidc == 'true' - with: - client-id: ${{ needs.check_oidc.outputs.client-id }} - tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} - subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} - - - name: Retrieve needed secrets from Azure Key Vault - if: needs.check_oidc.outputs.use-oidc == 'true' - shell: bash - run: | - echo "Fetching secrets from Azure Key Vault..." - - # List of secret names needed for this job - secret_names=("signing-client-id" "signing-client-secret" "signing-tenant-id" "signing-key-vault-certificate" "signing-key-vault-url") - - for secret_name in "${secret_names[@]}"; do - # Convert to uppercase and replace hyphens with underscores - env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') - - # Retrieve the secret value - secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) - - # Mask the secret value - echo "::add-mask::$secret_value" - - # Export as environment variable - echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" - done - - - name: Download Unsigned NuGet - id: downloadUnsignedNuget - uses: actions/download-artifact@v7 - with: - name: NugetPackages - path: _NuGetResults - - - name: Install dotnet sign - run: dotnet tool install sign --global --prerelease - - - name: Sign NuGet Package - env: - AZURE_TENANT_ID: ${{ env.SIGNING_TENANT_ID }} - AZURE_CLIENT_ID: ${{ env.SIGNING_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ env.SIGNING_CLIENT_SECRET }} - run: | - IFS=$'\n' - sign code azure-key-vault "_NuGetResults/**/*.nupkg" --publisher-name "Skyline Communications" --description "Skyline Signing" --description-url "https://www.skyline.be/" --azure-key-vault-certificate "${{ env.SIGNING_KEY_VAULT_CERTIFICATE }}" --azure-key-vault-url "${{ env.SIGNING_KEY_VAULT_URL }}" --output "_SignedNuGetResults" - unset IFS - shell: bash - - - uses: actions/upload-artifact@v6 - with: - name: SignedNugetPackages - path: "${{ github.workspace }}/_SignedNuGetResults" - - push: - if: github.ref_type == 'tag' - name: push - runs-on: ubuntu-latest - needs: sign - steps: - - name: Download Signed NuGet - id: downloadSignedNuget - uses: actions/download-artifact@v7 - with: - name: SignedNugetPackages - path: _SignedNuGetResults - - - name: Find Nuget - id: findcreatednuget - run: | - IFS=$'\n' - echo nugetPackageName=$(find _SignedNuGetResults -type f -name '*.nupkg') >> $GITHUB_OUTPUT - unset IFS - shell: bash - - - name: Push to GitHub NuGet repository - run: | - IFS=$'\n' - for i in ${{ steps.findcreatednuget.outputs.nugetPackageName }}; - do - dotnet nuget push "$i" --api-key ${{ secrets.nugetApiKey }} --source https://nuget.pkg.github.com/SkylineCommunications/index.json - done - unset IFS - shell: bash + master_workflow: + uses: ./.github/workflows/Master Workflow.yml + with: + oidc-client-id: ${{ inputs.oidc-client-id }} + oidc-tenant-id: ${{ inputs.oidc-tenant-id }} + oidc-subscription-id: ${{ inputs.oidc-subscription-id }} + sonarcloud-project-name: ${{ inputs.sonarCloudProjectName }} + solution-filter-name: ${{ inputs.solutionFilterName }} + secrets: + NUGET_API_KEY: ${{ secrets.nugetApiKey }} + SONAR_TOKEN: ${{ secrets.sonarCloudToken }} + AZURE_TOKEN: ${{ secrets.azureToken }} \ No newline at end of file diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml new file mode 100644 index 0000000..45c3df1 --- /dev/null +++ b/.github/workflows/Master Workflow.yml @@ -0,0 +1,1155 @@ +name: Main workflow + +permissions: + contents: read + id-token: write + packages: write + actions: read + +on: + workflow_call: + inputs: + oidc-client-id: + required: false + type: string + oidc-tenant-id: + required: false + type: string + oidc-subscription-id: + required: false + type: string + + sonarcloud-project-name: + required: true + type: string + + configuration: + required: false + type: string + default: 'Release' + + solution-filter-name: + required: false + type: string + + debug: + required: false + type: boolean + default: false + + # One or more lines with =. + # Example: + # PackageA.Install/CatalogInformation/manifest.yml=11111111-1111-1111-1111-111111111111 + # PackageB.Install/CatalogInformation/manifest.yml=22222222-2222-2222-2222-222222222222 + override-catalog-identifiers: + required: false + type: string + + # NuGet push destination URL. Defaults to the GitHub Packages registry of the calling repository's owner. + nuget-push-source: + required: false + type: string + + secrets: + SONAR_TOKEN: + required: false + DATAMINER_TOKEN: + required: false + AZURE_TOKEN: + required: false + OVERRIDE_CATALOG_DOWNLOAD_TOKEN: + required: false + NUGET_API_KEY: + required: false + +env: + VERSION_SDK: '2.4.7' + VERSION_APPPACKAGEINSTALLER: '4.0.0' + +jobs: + + check_oidc: + name: Check OIDC + runs-on: ubuntu-latest + outputs: + client-id: ${{ steps.set_oidc.outputs.client-id }} + tenant-id: ${{ steps.set_oidc.outputs.tenant-id }} + subscription-id: ${{ steps.set_oidc.outputs.subscription-id }} + use-oidc: ${{ steps.set_oidc.outputs.use-oidc }} + steps: + - name: Set Azure OIDC parameters + id: set_oidc + env: + OIDC_CLIENT_ID: ${{ inputs.oidc-client-id }} + OIDC_TENANT_ID: ${{ inputs.oidc-tenant-id }} + OIDC_SUBSCRIPTION_ID: ${{ inputs.oidc-subscription-id }} + REPO_OWNER: ${{ github.repository_owner }} + run: | + echo "Determining Azure OIDC parameters..." + + if [[ -n "$OIDC_CLIENT_ID" ]]; then + echo "Using provided OIDC parameters" + { + echo "client-id=$OIDC_CLIENT_ID" + echo "tenant-id=$OIDC_TENANT_ID" + echo "subscription-id=$OIDC_SUBSCRIPTION_ID" + echo "use-oidc=true" + } >> "$GITHUB_OUTPUT" + elif [[ "$REPO_OWNER" == "SkylineCommunications" ]]; then + echo "Using SkylineCommunications default OIDC parameters" + { + echo "client-id=c50da9cc-ba14-4138-8595-a62d97ab0e53" + echo "tenant-id=5f175691-8d1c-4932-b7c8-ce990839ac40" + echo "subscription-id=d6cbb8df-56eb-451d-9db7-67f49cba3220" + echo "use-oidc=true" + } >> "$GITHUB_OUTPUT" + else + echo "No OIDC parameters provided and owner does not match SkylineCommunications" + echo "use-oidc=false" >> "$GITHUB_OUTPUT" + fi + + discover_projects: + name: Discover Project Types + runs-on: ubuntu-latest + outputs: + has-dataminer-projects: ${{ steps.detect-projects.outputs.has-dataminer-projects }} + solution-path: ${{ steps.find-solution-file.outputs.path }} + steps: + - uses: actions/checkout@v6 + + - name: Find solution file + id: find-solution-file + env: + SOLUTION_FILTER_NAME: ${{ inputs.solution-filter-name }} + run: | + if [[ -z "$SOLUTION_FILTER_NAME" ]]; then + results=$(find . -type f \( -name '*.sln' -o -name '*.slnx' \)) + else + results=$(find . -type f -name "$SOLUTION_FILTER_NAME") + fi + + count=$(echo "$results" | grep -c .) + if [ "$count" -eq 0 ]; then + echo "Error: No solution file found." + exit 1 + elif [ "$count" -gt 1 ]; then + echo "Error: Multiple solution files found:" + echo "$results" + echo "Please specify a solution-filter-name input to disambiguate." + exit 1 + fi + + echo "path=$results" >> $GITHUB_OUTPUT + shell: bash + + - name: Detect projects with specific properties + id: detect-projects + run: | + $projectFiles = Get-ChildItem -Path . -Recurse -Filter "*.csproj" + $hasDataminerProjects = $false + + foreach ($project in $projectFiles) { + [xml]$content = Get-Content $project.FullName + + # Check for Skyline.DataMiner.Sdk projects (based on DataMinerType property) + if ($content.SelectNodes("//DataMinerType").Count -gt 0) { + echo "Found at least one DataMiner project" + $hasDataminerProjects = $true + break + } + } + + echo "has-dataminer-projects=$($hasDataminerProjects.ToString().ToLower())" >> $env:GITHUB_OUTPUT + shell: pwsh + + ci: + name: CI + needs: [check_oidc, discover_projects] + runs-on: ubuntu-latest + outputs: + has-nuget-packages: ${{ steps.detect-nuget-packages.outputs.has-nuget-packages }} + has-dataminer-packages: ${{ steps.detect-dataminer-packages.outputs.has-dataminer-packages }} + steps: + - name: Azure Login + uses: azure/login@v3 + if: needs.check_oidc.outputs.use-oidc == 'true' + with: + client-id: ${{ needs.check_oidc.outputs.client-id }} + tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} + subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} + + - name: Retrieve needed secrets from Azure Key Vault + if: needs.check_oidc.outputs.use-oidc == 'true' + shell: bash + env: + HAS_DM_PROJECTS: ${{ needs.discover_projects.outputs.has-dataminer-projects }} + run: | + echo "Fetching secrets from Azure Key Vault..." + + # List of secret names needed for this job + secret_names=("azure-token" "sonar-token") + + if [[ "$HAS_DM_PROJECTS" == "true" ]]; then + secret_names+=("dataminer-token") + fi + + for secret_name in "${secret_names[@]}"; do + # Convert to uppercase and replace hyphens with underscores + env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + + # Retrieve the secret value + secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) + + # Mask the secret value + echo "::add-mask::$secret_value" + + # Export as environment variable + echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" + done + + - name: Overwrite default secrets with user-defined secrets + shell: bash + env: + USER_AZURE_TOKEN: ${{ secrets.AZURE_TOKEN }} + USER_SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + USER_DATAMINER_TOKEN: ${{ secrets.DATAMINER_TOKEN }} + run: | + if [[ -n "$USER_AZURE_TOKEN" ]]; then + echo "Using provided AZURE_TOKEN secret" + echo "AZURE_TOKEN=$USER_AZURE_TOKEN" >> "$GITHUB_ENV" + fi + + if [[ -n "$USER_SONAR_TOKEN" ]]; then + echo "Using provided SONAR_TOKEN secret" + echo "SONAR_TOKEN=$USER_SONAR_TOKEN" >> "$GITHUB_ENV" + fi + + if [[ -n "$USER_DATAMINER_TOKEN" ]]; then + echo "Using provided DATAMINER_TOKEN secret" + echo "DATAMINER_TOKEN=$USER_DATAMINER_TOKEN" >> "$GITHUB_ENV" + fi + + - name: Validate SonarCloud Project Name + id: validate-sonar-name + if: github.actor != 'dependabot[bot]' + env: + SONAR_PROJECT_NAME: ${{ inputs.sonarcloud-project-name }} + HAS_DM_PROJECTS: ${{ needs.discover_projects.outputs.has-dataminer-projects }} + REPO: ${{ github.repository }} + run: | + if [[ -z "$SONAR_PROJECT_NAME" ]]; then + echo "Error: sonarcloud-project-name is not set." + echo "Please create a SonarCloud project by visiting: https://sonarcloud.io/projects/create and copy the id of the project as mentioned in the sonarcloud project url." + repo_url="https://github.com/$REPO/settings/variables/actions" + echo "Then set a SONAR_NAME variable in your repository settings: $repo_url" + if [[ "$HAS_DM_PROJECTS" == "true" ]]; then + echo "Alternatively, if you do not wish to use the Skyline Quality Gate but intend to publish your results to the catalog, you may create your workflow to include the equivalent of a dotnet publish step as shown below (remove the \\):" + echo " - name: Publish" + echo " env:" + echo " api-key: $\{{ secrets.DATAMINER_TOKEN }}" + echo " run: dotnet publish -p:Version=\"0.0.$\{{ github.run_number }}\" -p:VersionComment=\"Iterative Development\" -p:CatalogPublishKeyName=api-key" + fi + exit 1 + fi + + - name: Validate SonarCloud Secret Token + id: validate-sonar-token + if: github.actor != 'dependabot[bot]' + env: + HAS_DM_PROJECTS: ${{ needs.discover_projects.outputs.has-dataminer-projects }} + REPO: ${{ github.repository }} + run: | + if [[ -z "$SONAR_TOKEN" ]]; then + echo "Error: sonarCloudToken is not set." + echo "Please create a SonarCloud token by visiting: https://sonarcloud.io/account/security and copy the value of the created token." + repo_url="https://github.com/$REPO/settings/secrets/actions" + echo "Then set a SONAR_TOKEN secret in your repository settings: $repo_url" + if [[ "$HAS_DM_PROJECTS" == "true" ]]; then + echo "Alternatively, if you do not wish to use the Skyline Quality Gate but intend to publish your results to the catalog, you may create your workflow to include the equivalent of a dotnet publish step as shown below (remove the \\):" + echo " - name: Publish" + echo " env:" + echo " api-key: $\{{ secrets.DATAMINER_TOKEN }}" + echo " run: dotnet publish -p:Version=\"0.0.$\{{ github.run_number }}\" -p:VersionComment=\"Iterative Development\" -p:CatalogPublishKeyName=api-key" + fi + exit 1 + fi + + - name: Validate DataMiner Secret Token + id: validate-dataminer-token + if: github.ref_type == 'tag' && needs.discover_projects.outputs.has-dataminer-projects == 'true' + env: + REPO: ${{ github.repository }} + run: | + if [[ -z "$DATAMINER_TOKEN" ]]; then + echo "Error: dataminerToken is not set. Release not possible!" + echo "Please create or re-use an admin.dataminer.services token by visiting: https://admin.dataminer.services/." + echo "Navigate to the right organization, then go to Keys and create or find a key with the permissions Register catalog items, Download catalog versions, and Read catalog items." + echo "Copy the value of the token." + repo_url="https://github.com/$REPO/settings/secrets/actions" + echo "Then set a DATAMINER_TOKEN secret in your repository settings: $repo_url" + exit 1 + fi + + - name: Enable long paths + run: git config --global core.longpaths true + + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Cache and Install Mono + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + with: + packages: mono-complete + + - name: Enable GitHub NuGet Registry + env: + REPO_OWNER: ${{ github.repository_owner }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + $source = @{ Name = "PrivateGitHubNugets"; URL = "https://nuget.pkg.github.com/$env:REPO_OWNER/index.json"; Username = "USERNAME"; Password = "$env:GH_TOKEN" } + + Write-Host "Checking source $($source.Name)..." + + if (dotnet nuget list source | Select-String -Pattern $source.Name) { + Write-Host "Updating existing source $($source.Name)." + dotnet nuget update source $source.Name --source $source.URL --username $source.Username --password $source.Password --store-password-in-clear-text + } else { + Write-Host "Adding new source $($source.Name)." + dotnet nuget add source $source.URL --name $source.Name --username $source.Username --password $source.Password --store-password-in-clear-text + } + shell: pwsh + + - name: Enable Skyline NuGet Registries + if: github.repository_owner == 'SkylineCommunications' + run: | + $sources = @( + @{ Name = "CloudNuGets"; URL = "https://pkgs.dev.azure.com/skyline-cloud/Cloud_NuGets/_packaging/CloudNuGet/nuget/v3/index.json"; Username = "az"; Password = "$env:AZURE_TOKEN" }, + @{ Name = "PrivateAzureNuGets"; URL = "https://pkgs.dev.azure.com/skyline-cloud/_packaging/skyline-private-nugets/nuget/v3/index.json"; Username = "az"; Password = "$env:AZURE_TOKEN" } + ) + + foreach ($source in $sources) { + if ($source.Password -ne "") { + Write-Host "Checking source $($source.Name)..." + + if (dotnet nuget list source | Select-String -Pattern $source.Name) { + Write-Host "Updating existing source $($source.Name)." + dotnet nuget update source $source.Name --source $source.URL --username $source.Username --password $source.Password --store-password-in-clear-text + } else { + Write-Host "Adding new source $($source.Name)." + dotnet nuget add source $source.URL --name $source.Name --username $source.Username --password $source.Password --store-password-in-clear-text + } + } else { + Write-Host "Skipping $($source.Name) because the password is not set." + } + } + shell: pwsh + + - name: Install .NET tools + env: + GH_ACTOR: ${{ github.actor }} + HAS_DM_PROJECTS: ${{ needs.discover_projects.outputs.has-dataminer-projects }} + run: | + if [[ "$GH_ACTOR" != "dependabot[bot]" ]]; then + dotnet tool install dotnet-sonarscanner --global + fi + + # dotnet tool install -g Skyline.DataMiner.CICD.Tools.NuGetValidateSkylineSpecifications --version 3.* + dotnet tool install -g Skyline.DataMiner.CICD.Tools.NuGetValidateSkylineSpecifications --prerelease + + if [[ "$HAS_DM_PROJECTS" == "true" ]]; then + if [[ "$GH_ACTOR" != "dependabot[bot]" ]]; then + dotnet tool install Skyline.DataMiner.CICD.Tools.Sbom --global --version 1.* + fi + + # dotnet tool install Skyline.DataMiner.CICD.Tools.NuGetChangeVersion --global --version 3.* + dotnet tool install Skyline.DataMiner.CICD.Tools.NuGetChangeVersion --global --prerelease + fi + shell: bash + + - name: Validate NuGet projects + if: github.actor != 'dependabot[bot]' + env: + SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + REPO_OWNER: ${{ github.repository_owner }} + run: | + $minimal = "$env:REPO_OWNER" -ne "SkylineCommunications" + NuGetValidateSkylineSpecifications --solution-filepath "$env:SOLUTION_PATH" --minimal $minimal + shell: pwsh + + # ────────────────────────────────────────────────────────────── + # A. DataMiner-specific setup + # ────────────────────────────────────────────────────────────── + + - name: Update Skyline.DataMiner.Core.AppPackageInstaller + if: needs.discover_projects.outputs.has-dataminer-projects == 'true' + env: + SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + run: NuGetChangeVersion --solution-filepath "$SOLUTION_PATH" --nuget-name Skyline.DataMiner.Core.AppPackageInstaller --nuget-version $VERSION_APPPACKAGEINSTALLER + + - name: Update Skyline.DataMiner.Sdk version in global.json (if present) + if: needs.discover_projects.outputs.has-dataminer-projects == 'true' + shell: pwsh + run: | + $globalJsonPath = Join-Path $env:GITHUB_WORKSPACE "global.json" + + if (-Not (Test-Path $globalJsonPath)) { + Write-Host "No global.json found. Skipping update." + return + } + + $jsonContent = Get-Content $globalJsonPath -Raw | ConvertFrom-Json + + if (-not $jsonContent.'msbuild-sdks') { + $jsonContent | Add-Member -MemberType NoteProperty -Name 'msbuild-sdks' -Value @{} + } + + $jsonContent.'msbuild-sdks'.'Skyline.DataMiner.Sdk' = $env:VERSION_SDK + + $updatedJson = $jsonContent | ConvertTo-Json -Depth 10 + + $updatedJson | Set-Content $globalJsonPath -Encoding UTF8 + + Write-Host "Updated global.json:" + Write-Host $updatedJson + + - name: Update Catalog Identifiers + if: inputs.override-catalog-identifiers != '' + env: + CATALOG_IDENTIFIERS: ${{ inputs.override-catalog-identifiers }} + run: | + $rawMappings = $env:CATALOG_IDENTIFIERS + $mappings = $rawMappings -split '[\r\n]+' | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne '' } + + foreach ($mapping in $mappings) { + $splitIndex = $mapping.IndexOf('=') + if ($splitIndex -lt 1 -or $splitIndex -eq ($mapping.Length - 1)) { + Write-Error "Invalid entry '$mapping'. Expected format: =." + exit 1 + } + + $manifestPath = $mapping.Substring(0, $splitIndex).Trim() + $identifier = $mapping.Substring($splitIndex + 1).Trim() + + if ($identifier -notmatch '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$') { + Write-Error "Identifier '$identifier' is not a valid GUID." + exit 1 + } + + if ([System.IO.Path]::GetFileName($manifestPath) -ne 'manifest.yml') { + Write-Error "Manifest path '$manifestPath' must point to 'manifest.yml'." + exit 1 + } + + if (-not (Test-Path -Path $manifestPath -PathType Leaf)) { + Write-Error "Manifest file '$manifestPath' does not exist." + exit 1 + } + + $content = Get-Content -Path $manifestPath -Raw + $hasActiveIdLine = [regex]::IsMatch($content, '(?m)^(?!\s*#)\s*id:\s*.*$') + + if (-not $hasActiveIdLine) { + Write-Error "No active id line found in '$manifestPath'." + exit 1 + } + + $updatedContent = [regex]::Replace($content, '(?m)^(?!\s*#)\s*id:\s*.*$', "id: $identifier", 1) + + if ($updatedContent -eq $content) { + Write-Host "'$manifestPath' already has id: $identifier" + } else { + Set-Content -Path $manifestPath -Value $updatedContent -NoNewline + Write-Host "Updated '$manifestPath' to id: $identifier" + } + } + shell: pwsh + + - name: Apply SourceCode Url To Manifest + if: needs.discover_projects.outputs.has-dataminer-projects == 'true' + env: + REPO: ${{ github.repository }} + run: | + $manifestFiles = Get-ChildItem -Recurse -Filter 'manifest.yml' | + Where-Object { $_.FullName -match '[\\/]CatalogInformation[\\/]' } + + foreach ($file in $manifestFiles) { + $lines = Get-Content -Path $file.FullName + $updated = $false + + for ($i = 0; $i -lt $lines.Count; $i++) { + if ($lines[$i] -match '^\s*source_code_url:\s*$') { + $indent = ($lines[$i] -match '^(\s*)source_code_url:')[1] + $lines[$i] = "$indent" + "source_code_url: 'https://github.com/$env:REPO'" + $updated = $true + break + } + } + + if ($updated) { + Write-Host "Updating: $($file.FullName) with 'source_code_url: https://github.com/$env:REPO'" + Set-Content -Path $file.FullName -Value $lines -Encoding UTF8 + } + } + shell: pwsh + + - name: Detect test runner mode + id: detect-test-mode + shell: pwsh + run: | + $globalJsonPath = Join-Path $env:GITHUB_WORKSPACE "global.json" + + if (Test-Path $globalJsonPath) { + $jsonContent = Get-Content $globalJsonPath -Raw | ConvertFrom-Json + $runner = $jsonContent.test.runner + + if ($runner -eq "Microsoft.Testing.Platform") { + Write-Host "Detected Microsoft.Testing.Platform (MTP) test runner in global.json" + echo "test-runner-mode=mtp" >> $env:GITHUB_OUTPUT + return + } + } + + Write-Host "Using default VSTest test runner" + echo "test-runner-mode=vstest" >> $env:GITHUB_OUTPUT + + # ────────────────────────────────────────────────────────────── + # B. SonarCloud analysis (skipped for Dependabot) + # ────────────────────────────────────────────────────────────── + + - name: Prepare SonarCloud Variables + if: github.actor != 'dependabot[bot]' + id: prepSonarCloudVar + env: + OWNER: ${{ github.repository_owner }} + run: echo "lowerCaseOwner=${OWNER,,}" >> "$GITHUB_ENV" + + - name: Get SonarCloud Status + if: github.actor != 'dependabot[bot]' + id: get-sonarcloud-status + env: + SONAR_PROJECT_NAME: ${{ inputs.sonarcloud-project-name }} + BRANCH_NAME: ${{ github.ref_name }} + REPO: ${{ github.repository }} + run: | + sonarCloudProjectStatus=$(curl -s -u "$SONAR_TOKEN:" "https://sonarcloud.io/api/qualitygates/project_status?projectKey=$SONAR_PROJECT_NAME&branch=$BRANCH_NAME") + + # Check if the response is empty or not valid JSON + if [ -z "$sonarCloudProjectStatus" ] || ! echo "$sonarCloudProjectStatus" | jq . > /dev/null 2>&1; then + echo "Error: The SONAR_TOKEN is invalid, expired, or the response is empty. Please check: https://sonarcloud.io/account/security and update your token: https://github.com/$REPO/settings/secrets/actions" >&2 + echo "Returned response: $sonarCloudProjectStatus" >&2 + exit 1 + fi + + # Check if the response contains errors + if echo "$sonarCloudProjectStatus" | jq -e '.errors' > /dev/null 2>&1; then + echo "Error: SonarCloud API returned errors. Initial analysis needed." >&2 + echo "Returned response: $sonarCloudProjectStatus" >&2 + echo "needsInitialAnalysis=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # Check if project status is NONE (needs initial analysis) + projectStatus=$(echo "$sonarCloudProjectStatus" | jq -r '.projectStatus.status // empty') + if [ "$projectStatus" = "NONE" ]; then + echo "Project status is NONE. Initial analysis needed." + echo "needsInitialAnalysis=true" >> $GITHUB_OUTPUT + else + echo "needsInitialAnalysis=false" >> $GITHUB_OUTPUT + fi + + # Output the JSON response if valid + echo "Returned response: $sonarCloudProjectStatus" + shell: bash + + - name: Trigger Initial Analysis + if: github.actor != 'dependabot[bot]' && steps.get-sonarcloud-status.outputs.needsInitialAnalysis == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_PROJECT_NAME: ${{ inputs.sonarcloud-project-name }} + BRANCH_NAME: ${{ github.ref_name }} + SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + CONFIGURATION: ${{ inputs.configuration }} + run: | + dotnet sonarscanner begin \ + /k:"$SONAR_PROJECT_NAME" \ + /o:"$lowerCaseOwner" \ + /d:sonar.token="$SONAR_TOKEN" \ + /d:sonar.host.url="https://sonarcloud.io" \ + /d:sonar.exclusions="**/*.yml,**/*.xml" \ + /d:sonar.branch.name="$BRANCH_NAME" + dotnet build "$SOLUTION_PATH" \ + -p:GenerateDataMinerPackage=false \ + -p:GeneratePackageOnBuild=false \ + --configuration "$CONFIGURATION" \ + -nodeReuse:false + dotnet sonarscanner end /d:sonar.token="$SONAR_TOKEN" + continue-on-error: true + + - name: Start Analysis + if: github.actor != 'dependabot[bot]' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_PROJECT_NAME: ${{ inputs.sonarcloud-project-name }} + BRANCH_NAME: ${{ github.ref_name }} + run: | + dotnet sonarscanner begin \ + /k:"$SONAR_PROJECT_NAME" \ + /o:"$lowerCaseOwner" \ + /d:sonar.token="$SONAR_TOKEN" \ + /d:sonar.host.url="https://sonarcloud.io" \ + /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" \ + /d:sonar.cs.vscoveragexml.reportsPaths="**/TestResults/**/coverage.xml" \ + /d:sonar.cs.vstest.reportsPaths="**/TestResults/**.trx" \ + /d:sonar.exclusions="**/*.yml,**/*.xml" \ + /d:sonar.branch.name="$BRANCH_NAME" + continue-on-error: true + + # ────────────────────────────────────────────────────────────── + # C. Build + # ────────────────────────────────────────────────────────────── + + - name: Determine version + id: determine-version + env: + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + RUN_NUMBER: ${{ github.run_number }} + run: | + if [[ "$REF_TYPE" == "tag" ]]; then + echo "version=$REF_NAME" >> $GITHUB_OUTPUT + else + echo "version=0.0.$RUN_NUMBER" >> $GITHUB_OUTPUT + fi + shell: bash + + - name: Build + env: + OVERRIDE_CATALOG_DOWNLOAD_TOKEN: ${{ secrets.OVERRIDE_CATALOG_DOWNLOAD_TOKEN }} + SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + VERSION: ${{ steps.determine-version.outputs.version }} + CONFIGURATION: ${{ inputs.configuration }} + DEBUG_FLAG: ${{ inputs.debug }} + run: | + dotnet build "$SOLUTION_PATH" \ + -p:Version="$VERSION" \ + -p:DefineConstants="DCFv1%3BDBInfo%3BALARM_SQUASHING" \ + --configuration "$CONFIGURATION" \ + -p:CatalogPublishKeyName="DATAMINER_TOKEN" \ + -p:CatalogDefaultDownloadKeyName="OVERRIDE_CATALOG_DOWNLOAD_TOKEN" \ + -p:SkylineDataMinerSdkDebug="$DEBUG_FLAG" \ + -nodeReuse:false + + # ────────────────────────────────────────────────────────────── + # D. Test & Quality Gate + # ────────────────────────────────────────────────────────────── + + - name: Unit Tests (VSTest) + id: unit-tests-vstest + if: steps.detect-test-mode.outputs.test-runner-mode == 'vstest' + env: + SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + CONFIGURATION: ${{ inputs.configuration }} + run: | + dotnet test "$SOLUTION_PATH" \ + --no-build \ + --configuration "$CONFIGURATION" \ + --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ + --logger "trx;logfilename=unitTestResults.trx" \ + --collect "XPlat Code Coverage" \ + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura,opencover + + - name: Unit Tests (MTP) + id: unit-tests-mtp + if: steps.detect-test-mode.outputs.test-runner-mode == 'mtp' + env: + SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + CONFIGURATION: ${{ inputs.configuration }} + run: | + dotnet test \ + --solution "$SOLUTION_PATH" \ + --no-build \ + --configuration "$CONFIGURATION" \ + --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ + --report-trx \ + --report-trx-filename unitTestResults.trx \ + --coverage \ + --coverage-output-format xml \ + --coverage-output coverage.xml \ + --ignore-exit-code 8 + + - name: Stop Analysis + if: github.actor != 'dependabot[bot]' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + dotnet sonarscanner end /d:sonar.token="$SONAR_TOKEN" + continue-on-error: true + + - name: SonarCloud Quality Gate check + if: github.actor != 'dependabot[bot]' + id: sonarcloud-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@cb3ed20f9fec62b4c3b8ad9e77656c6adaade913 # Master from 17 Nov 2025 + with: + scanMetadataReportFile: .sonarqube/out/.sonar/report-task.txt + continue-on-error: true + timeout-minutes: 5 + + - name: Quality Gate + id: quality-step + env: + GH_ACTOR: ${{ github.actor }} + TEST_VSTEST_OUTCOME: ${{ steps.unit-tests-vstest.outcome }} + TEST_MTP_OUTCOME: ${{ steps.unit-tests-mtp.outcome }} + QG_STATUS: ${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }} + QG_OUTCOME: ${{ steps.sonarcloud-quality-gate-check.outcome }} + run: | + import os + is_dependabot = os.environ["GH_ACTOR"] == "dependabot[bot]" + + test_outcome_vstest = os.environ["TEST_VSTEST_OUTCOME"] + test_outcome_mtp = os.environ["TEST_MTP_OUTCOME"] + test_failed = test_outcome_vstest == "failure" or test_outcome_mtp == "failure" + + if test_failed: + print("Quality gate failed due to:") + print("- Test failures") + + if not is_dependabot: + if os.environ["QG_STATUS"] == "FAILED": + print("Quality gate failed due to:") + print("- Code analysis quality gate failed") + if os.environ["QG_OUTCOME"] == "failure" and os.environ["QG_STATUS"] != "FAILED": + print("- Could not retrieve SonarCloud quality gate status, potentially due to reaching License max LoC. Ignoring SonarCloud quality gate status in this case.") + + if test_failed: + exit(1) + if not is_dependabot and os.environ["QG_STATUS"] == "FAILED": + exit(1) + shell: python + + # ────────────────────────────────────────────────────────────── + # E. Artifacts & Outputs + # ────────────────────────────────────────────────────────────── + + - name: Detect NuGet packages + if: github.actor != 'dependabot[bot]' + id: detect-nuget-packages + env: + CONFIGURATION: ${{ inputs.configuration }} + run: | + nuget_count=$(find . -path "*/bin/$CONFIGURATION/*.nupkg" | wc -l) + if [ "$nuget_count" -gt 0 ]; then + echo "Found $nuget_count NuGet package(s)" + echo "has-nuget-packages=true" >> $GITHUB_OUTPUT + else + echo "No NuGet packages found" + echo "has-nuget-packages=false" >> $GITHUB_OUTPUT + fi + shell: bash + + - name: Upload NuGet packages + if: github.actor != 'dependabot[bot]' && steps.detect-nuget-packages.outputs.has-nuget-packages == 'true' + uses: actions/upload-artifact@v7 + with: + name: NugetPackages + path: | + **/bin/${{ inputs.configuration }}/**/*.nupkg + + - name: Detect DataMiner packages + if: github.actor != 'dependabot[bot]' + id: detect-dataminer-packages + run: | + dm_count=$(find . -type f \( -name "*.dmapp" -o -name "*.dmtest" \) | wc -l) + if [ "$dm_count" -gt 0 ]; then + echo "Found $dm_count DataMiner package(s)" + echo "has-dataminer-packages=true" >> $GITHUB_OUTPUT + else + echo "No DataMiner packages found" + echo "has-dataminer-packages=false" >> $GITHUB_OUTPUT + fi + shell: bash + + - name: Create package name + if: github.actor != 'dependabot[bot]' && steps.detect-dataminer-packages.outputs.has-dataminer-packages == 'true' + id: packageName + env: + REPO: ${{ github.repository }} + run: | + $tempName = "$env:REPO" + $safeName = $tempName -replace '[\"\/\\<>|:*?]', '_' + echo "name=$safeName" >> $env:GITHUB_OUTPUT + shell: pwsh + + - name: Generate SBOM file + if: github.actor != 'dependabot[bot]' && steps.detect-dataminer-packages.outputs.has-dataminer-packages == 'true' + env: + SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + PACKAGE_NAME: ${{ steps.packageName.outputs.name }} + PACKAGE_VERSION: ${{ github.ref_name }} + DEBUG_FLAG: ${{ inputs.debug }} + run: | + find . -type f \( -name "*.dmapp" -o -name "*.dmtest" \) -print0 | while IFS= read -r -d '' file; do + echo "Generating SBOM for $file" + dataminer-sbom generate-and-add \ + --solution-path "$SOLUTION_PATH" \ + --package-file "$file" \ + --package-name "$PACKAGE_NAME" \ + --package-version "$PACKAGE_VERSION" \ + --package-supplier "Skyline Communications" \ + --debug "$DEBUG_FLAG" + done + + - name: Upload DataMiner packages + if: github.actor != 'dependabot[bot]' && steps.detect-dataminer-packages.outputs.has-dataminer-packages == 'true' + uses: actions/upload-artifact@v7 + with: + name: DataMiner Installation Packages (${{ inputs.configuration }} ${{ inputs.solution-filter-name }}) unsigned + path: | + **/bin/${{ inputs.configuration }}/*.dmapp + **/bin/${{ inputs.configuration }}/*.dmtest + **/bin/${{ inputs.configuration }}/*.zip + **/bin/${{ inputs.configuration }}/**/*.dmapp + **/bin/${{ inputs.configuration }}/**/*.dmtest + **/bin/${{ inputs.configuration }}/**/*.zip + continue-on-error: true + + # ════════════════════════════════════════════════════════════════ + # NuGet Signing (Windows required: https://github.com/dotnet/runtime/issues/48794) + # ════════════════════════════════════════════════════════════════ + + sign_nuget: + name: Sign NuGet Packages + if: needs.ci.outputs.has-nuget-packages == 'true' && github.actor != 'dependabot[bot]' + runs-on: windows-latest + needs: [ci, check_oidc] + steps: + - name: Azure Login + uses: azure/login@v3 + if: needs.check_oidc.outputs.use-oidc == 'true' + with: + client-id: ${{ needs.check_oidc.outputs.client-id }} + tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} + subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} + + - name: Retrieve signing secrets from Azure Key Vault + if: needs.check_oidc.outputs.use-oidc == 'true' + shell: bash + run: | + echo "Fetching secrets from Azure Key Vault..." + + secret_names=("signing-client-id" "signing-client-secret" "signing-tenant-id" "signing-key-vault-certificate" "signing-key-vault-url") + + for secret_name in "${secret_names[@]}"; do + env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) + echo "::add-mask::$secret_value" + echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" + done + + - name: Download Unsigned NuGet + uses: actions/download-artifact@v8 + with: + name: NugetPackages + path: _NuGetResults + + - name: Install dotnet sign + if: needs.check_oidc.outputs.use-oidc == 'true' + run: dotnet tool install sign --global --prerelease + + - name: Sign NuGet Package + if: needs.check_oidc.outputs.use-oidc == 'true' + env: + AZURE_TENANT_ID: ${{ env.SIGNING_TENANT_ID }} + AZURE_CLIENT_ID: ${{ env.SIGNING_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ env.SIGNING_CLIENT_SECRET }} + run: | + IFS=$'\n' + sign code azure-key-vault "_NuGetResults/**/*.nupkg" --publisher-name "Skyline Communications" --description "Skyline Signing" --description-url "https://www.skyline.be/" --azure-key-vault-certificate "$SIGNING_KEY_VAULT_CERTIFICATE" --azure-key-vault-url "$SIGNING_KEY_VAULT_URL" --output "_SignedNuGetResults" + unset IFS + shell: bash + + - name: Upload Signed NuGet + if: needs.check_oidc.outputs.use-oidc == 'true' + uses: actions/upload-artifact@v7 + with: + name: SignedNugetPackages + path: "${{ github.workspace }}/_SignedNuGetResults" + + - name: Upload Unsigned NuGet (signing not available) + if: needs.check_oidc.outputs.use-oidc != 'true' + uses: actions/upload-artifact@v7 + with: + name: SignedNugetPackages + path: "${{ github.workspace }}/_NuGetResults" + + # ════════════════════════════════════════════════════════════════ + # NuGet Push (on tag only) + # ════════════════════════════════════════════════════════════════ + + push_nuget: + name: Push NuGet Packages + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + needs: [sign_nuget, check_oidc] + steps: + - name: Azure Login + uses: azure/login@v3 + if: needs.check_oidc.outputs.use-oidc == 'true' + with: + client-id: ${{ needs.check_oidc.outputs.client-id }} + tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} + subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} + + - name: Retrieve needed secrets from Azure Key Vault + if: needs.check_oidc.outputs.use-oidc == 'true' + shell: bash + run: | + echo "Fetching secrets from Azure Key Vault..." + + secret_names=("nuget-api-key-github") + + for secret_name in "${secret_names[@]}"; do + env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) + echo "::add-mask::$secret_value" + echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" + done + + - name: Overwrite default secrets with user-defined secrets + shell: bash + env: + USER_NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} + run: | + if [[ -n "$USER_NUGET_API_KEY" ]]; then + echo "Using provided NUGET_API_KEY secret" + echo "NUGET_API_KEY_GITHUB=$USER_NUGET_API_KEY" >> "$GITHUB_ENV" + fi + + - name: Download Signed NuGet + uses: actions/download-artifact@v8 + with: + name: SignedNugetPackages + path: _SignedNuGetResults + + - name: Find NuGet packages + id: findcreatednuget + run: | + IFS=$'\n' + echo nugetPackageName=$(find _SignedNuGetResults -type f -name '*.nupkg') >> $GITHUB_OUTPUT + unset IFS + shell: bash + + - name: Push NuGet packages + env: + PUSH_SOURCE_INPUT: ${{ inputs.nuget-push-source }} + REPO_OWNER: ${{ github.repository_owner }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PACKAGE_NAMES: ${{ steps.findcreatednuget.outputs.nugetPackageName }} + run: | + # Determine push source + PUSH_SOURCE="$PUSH_SOURCE_INPUT" + if [ -z "$PUSH_SOURCE" ]; then + PUSH_SOURCE="https://nuget.pkg.github.com/$REPO_OWNER/index.json" + fi + + # Determine API key: use OIDC/user-provided key, else fall back to GITHUB_TOKEN + API_KEY="$NUGET_API_KEY_GITHUB" + if [ -z "$API_KEY" ]; then + API_KEY="$GH_TOKEN" + fi + + IFS=$'\n' + for i in $PACKAGE_NAMES; + do + dotnet nuget push "$i" --api-key "$API_KEY" --source "$PUSH_SOURCE" + done + unset IFS + shell: bash + + # ════════════════════════════════════════════════════════════════ + # DataMiner Catalog Upload (on tag only) + # ════════════════════════════════════════════════════════════════ + + upload_to_catalog: + name: Upload to Catalog + runs-on: windows-latest + if: github.ref_type == 'tag' && needs.ci.outputs.has-dataminer-packages == 'true' + needs: [ci, check_oidc, discover_projects] + steps: + - name: Azure Login + uses: azure/login@v3 + if: needs.check_oidc.outputs.use-oidc == 'true' + with: + client-id: ${{ needs.check_oidc.outputs.client-id }} + tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} + subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} + + - name: Retrieve needed secrets from Azure Key Vault + if: needs.check_oidc.outputs.use-oidc == 'true' + shell: bash + run: | + echo "Fetching secrets from Azure Key Vault..." + + secret_names=("dataminer-token" "signing-client-id" "signing-client-secret" "signing-tenant-id" "signing-key-vault-certificate" "signing-key-vault-url") + + for secret_name in "${secret_names[@]}"; do + env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) + echo "::add-mask::$secret_value" + echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" + done + + - name: Overwrite default secrets with user-defined secrets + shell: bash + env: + USER_DATAMINER_TOKEN: ${{ secrets.DATAMINER_TOKEN }} + run: | + if [[ -n "$USER_DATAMINER_TOKEN" ]]; then + echo "Using provided DATAMINER_TOKEN secret" + echo "DATAMINER_TOKEN=$USER_DATAMINER_TOKEN" >> "$GITHUB_ENV" + fi + + - name: Enable long paths + run: git config --global core.longpaths true + + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Download artifact from CI + uses: actions/download-artifact@v8 + with: + name: DataMiner Installation Packages (${{ inputs.configuration }} ${{ inputs.solution-filter-name }}) unsigned + path: downloaded_artifacts + + - name: Restore artifact contents to bin folders + run: | + Write-Output "Restoring DataMiner package files to original structure..." + + $downloadRoot = Resolve-Path "downloaded_artifacts" + $workspaceRoot = Resolve-Path "." + + Get-ChildItem -Path $downloadRoot -Recurse -File | Where-Object { $_.Extension -in '.zip', '.dmapp', '.dmtest' } | ForEach-Object { + $archiveFile = $_.FullName + $relativePath = $archiveFile.Substring($downloadRoot.Path.Length + 1) -replace '\\', '/' + $targetPath = Join-Path $workspaceRoot $relativePath + + $targetFolder = Split-Path $targetPath + if (-not (Test-Path $targetFolder)) { + New-Item -ItemType Directory -Path $targetFolder -Force | Out-Null + } + + Move-Item -Path $archiveFile -Destination $targetPath -Force + Write-Output "Moved $relativePath to workspace root" + } + + Write-Output "Removing downloaded_artifacts folder..." + Remove-Item -Path $downloadRoot -Recurse -Force + Write-Output "Cleanup complete." + shell: pwsh + + - name: Install Tools + if: needs.check_oidc.outputs.use-oidc == 'true' + run: | + dotnet tool install Skyline.DataMiner.CICD.Tools.PackageSign --global --version 2.* + + - name: Sign generated dmapp/dmtest packages + if: needs.check_oidc.outputs.use-oidc == 'true' + shell: pwsh + env: + AZURE_TENANT_ID: ${{ env.SIGNING_TENANT_ID }} + AZURE_CLIENT_ID: ${{ env.SIGNING_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ env.SIGNING_CLIENT_SECRET }} + AZURE_KEY_VAULT_URL: ${{ env.SIGNING_KEY_VAULT_URL }} + AZURE_KEY_VAULT_CERTIFICATE: ${{ env.SIGNING_KEY_VAULT_CERTIFICATE }} + DEBUG_FLAG: ${{ inputs.debug }} + run: | + dataminer-package-signature sign dmapp ` + --package-location "$env:GITHUB_WORKSPACE" ` + --debug $env:DEBUG_FLAG + + - name: Authenticate with GitHub CLI + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + "$env:GH_TOKEN" | gh auth login --with-token + + - name: Find Version Comment + id: findVersionComment + env: + REF_NAME: ${{ github.ref_name }} + run: | + echo "Checking for release notes associated with the reference: '$REF_NAME'" + + RELEASE_INFO=$(gh release view "$REF_NAME" --json body,name 2>/dev/null || echo "{}") + RELEASE_NOTE=$(echo "$RELEASE_INFO" | jq -r '.body // ""' 2>/dev/null || echo "") + RELEASE_TITLE=$(echo "$RELEASE_INFO" | jq -r '.name // ""' 2>/dev/null || echo "") + + escape_special_chars() { + echo "$1" | sed -e 's/,/%2c/g' -e 's/"/%22/g' -e 's/;/%3b/g' + } + + if [[ -n "$RELEASE_NOTE" ]]; then + ESCAPED_RELEASE_NOTE=$(escape_special_chars "$RELEASE_NOTE") + echo "Release description found for '$REF_NAME': $ESCAPED_RELEASE_NOTE" + echo "versionComment<> $GITHUB_OUTPUT + echo "$ESCAPED_RELEASE_NOTE" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + elif [[ -n "$RELEASE_TITLE" ]]; then + ESCAPED_RELEASE_TITLE=$(escape_special_chars "$RELEASE_TITLE") + echo "Release title found for '$REF_NAME': $ESCAPED_RELEASE_TITLE" + echo "versionComment=$ESCAPED_RELEASE_TITLE" >> $GITHUB_OUTPUT + else + echo "No release description or title found for '$REF_NAME'. Falling back to tag or commit message." + VERSION_COMMENT=$(git describe --tags --exact-match 2>/dev/null || git log -1 --pretty=format:%s) + ESCAPED_VERSION_COMMENT=$(escape_special_chars "$VERSION_COMMENT") + echo "Fallback version comment: $ESCAPED_VERSION_COMMENT" + echo "versionComment=$ESCAPED_VERSION_COMMENT" >> $GITHUB_OUTPUT + fi + shell: bash + + - name: Update Skyline.DataMiner.Sdk version in global.json (if present) + shell: pwsh + run: | + $globalJsonPath = Join-Path $env:GITHUB_WORKSPACE "global.json" + + if (-Not (Test-Path $globalJsonPath)) { + Write-Host "No global.json found. Skipping update." + return + } + + $jsonContent = Get-Content $globalJsonPath -Raw | ConvertFrom-Json + + if (-not $jsonContent.'msbuild-sdks') { + $jsonContent | Add-Member -MemberType NoteProperty -Name 'msbuild-sdks' -Value @{} + } + + $jsonContent.'msbuild-sdks'.'Skyline.DataMiner.Sdk' = $env:VERSION_SDK + + $updatedJson = $jsonContent | ConvertTo-Json -Depth 10 + + $updatedJson | Set-Content $globalJsonPath -Encoding UTF8 + + Write-Host "Updated global.json:" + Write-Host $updatedJson + + - name: Publish To Catalog + shell: pwsh + env: + SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + VERSION: ${{ github.ref_name }} + VERSION_COMMENT: ${{ steps.findVersionComment.outputs.versionComment }} + CONFIGURATION: ${{ inputs.configuration }} + DEBUG_FLAG: ${{ inputs.debug }} + run: | + dotnet publish ` + "$env:SOLUTION_PATH" ` + --no-build ` + -p:Version="$env:VERSION" ` + -p:VersionComment="$env:VERSION_COMMENT" ` + -p:CatalogPublishKeyName="DATAMINER_TOKEN" ` + --configuration "$env:CONFIGURATION" ` + -p:SkylineDataMinerSdkDebug="$env:DEBUG_FLAG" ` + -p:IsPublishable=false \ No newline at end of file diff --git a/.github/workflows/NuGet Solution Master Workflow.yml b/.github/workflows/NuGet Solution Master Workflow.yml index 4aef429..757218f 100644 --- a/.github/workflows/NuGet Solution Master Workflow.yml +++ b/.github/workflows/NuGet Solution Master Workflow.yml @@ -5,11 +5,6 @@ permissions: write-all on: workflow_call: - outputs: - quality_gate: - description: "Results from Skyline Quality Gate." - value: ${{ jobs.validate_skyline_quality_gate.outputs.quality }} - inputs: # No need to specify these as the workflow can access github.* (https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/accessing-contextual-information-about-workflow-runs#github-context) # When a reusable workflow is triggered by a caller workflow, the github context is always associated with the caller workflow. @@ -65,419 +60,58 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if obsolete inputs are still used - run: | - input_names=("referenceName" "runNumber" "referenceType" "repository" "owner") - - for input_name in "${input_names[@]}"; do - value="${{ inputs.referenceName }}" # placeholder, see note below - case $input_name in - referenceName) value="${{ inputs.referenceName }}" ;; - runNumber) value="${{ inputs.runNumber }}" ;; - referenceType) value="${{ inputs.referenceType }}" ;; - repository) value="${{ inputs.repository }}" ;; - owner) value="${{ inputs.owner }}" ;; - esac + env: + INPUT_REFERENCENAME: ${{ inputs.referenceName }} + INPUT_RUNNUMBER: ${{ inputs.runNumber }} + INPUT_REFERENCETYPE: ${{ inputs.referenceType }} + INPUT_REPOSITORY: ${{ inputs.repository }} + INPUT_OWNER: ${{ inputs.owner }} + run: | + declare -A input_values=( + [referenceName]="$INPUT_REFERENCENAME" + [runNumber]="$INPUT_RUNNUMBER" + [referenceType]="$INPUT_REFERENCETYPE" + [repository]="$INPUT_REPOSITORY" + [owner]="$INPUT_OWNER" + ) - if [ -n "$value" ]; then + for input_name in "${!input_values[@]}"; do + if [ -n "${input_values[$input_name]}" ]; then echo "::warning::The input '$input_name' can be safely removed as this is not required anymore." fi done - name: Check if obsolete secrets are still used if: github.repository_owner == 'SkylineCommunications' - run: | - secret_names=("pfx" "pfxPassword" "azureToken" "sonarCloudToken") - - for secret_name in "${secret_names[@]}"; do - value="${{ secrets.sonarCloudToken }}" # placeholder, see note below - case $secret_name in - sonarCloudToken) value="${{ secrets.sonarCloudToken }}" ;; - pfx) value="${{ secrets.pfx }}" ;; - pfxPassword) value="${{ secrets.pfxPassword }}" ;; - azureToken) value="${{ secrets.azureToken }}" ;; - esac + env: + SECRET_PFX: ${{ secrets.pfx }} + SECRET_PFXPASSWORD: ${{ secrets.pfxPassword }} + SECRET_AZURETOKEN: ${{ secrets.azureToken }} + SECRET_SONARCLOUDTOKEN: ${{ secrets.sonarCloudToken }} + run: | + declare -A secret_values=( + [pfx]="$SECRET_PFX" + [pfxPassword]="$SECRET_PFXPASSWORD" + [azureToken]="$SECRET_AZURETOKEN" + [sonarCloudToken]="$SECRET_SONARCLOUDTOKEN" + ) - if [ -n "$value" ]; then + for secret_name in "${!secret_values[@]}"; do + if [ -n "${secret_values[$secret_name]}" ]; then echo "::warning::The secret '$secret_name' can be safely removed as this is not required anymore." fi done - check_oidc: - name: Check OIDC - runs-on: ubuntu-latest - outputs: - client-id: ${{ steps.set_oidc.outputs.client-id }} - tenant-id: ${{ steps.set_oidc.outputs.tenant-id }} - subscription-id: ${{ steps.set_oidc.outputs.subscription-id }} - use-oidc: ${{ steps.set_oidc.outputs.use-oidc }} - steps: - - name: Set Azure OIDC parameters - id: set_oidc - run: | - echo "Determining Azure OIDC parameters..." - - if [[ -n "${{ inputs.oidc-client-id }}" ]]; then - echo "Using provided OIDC parameters" - { - echo "client-id=${{ inputs.oidc-client-id }}" - echo "tenant-id=${{ inputs.oidc-tenant-id }}" - echo "subscription-id=${{ inputs.oidc-subscription-id }}" - echo "use-oidc=true" - } >> "$GITHUB_OUTPUT" - elif [[ "${{ github.repository_owner }}" == "SkylineCommunications" ]]; then - echo "Using SkylineCommunications default OIDC parameters" - { - echo "client-id=c50da9cc-ba14-4138-8595-a62d97ab0e53" - echo "tenant-id=5f175691-8d1c-4932-b7c8-ce990839ac40" - echo "subscription-id=d6cbb8df-56eb-451d-9db7-67f49cba3220" - echo "use-oidc=true" - } >> "$GITHUB_OUTPUT" - else - echo "No OIDC parameters provided and owner does not match SkylineCommunications" - echo "use-oidc=false" >> "$GITHUB_OUTPUT" - fi - - validate_skyline_quality_gate: - name: Skyline Quality Gate - runs-on: windows-latest - needs: check_oidc - outputs: - quality: ${{ steps.quality-step.outputs.results }} - steps: - - name: Azure Login - uses: azure/login@v2 - if: needs.check_oidc.outputs.use-oidc == 'true' - with: - client-id: ${{ needs.check_oidc.outputs.client-id }} - tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} - subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} - - - name: Retrieve needed secrets from Azure Key Vault - if: needs.check_oidc.outputs.use-oidc == 'true' - shell: bash - run: | - echo "Fetching secrets from Azure Key Vault..." - - # List of secret names needed for this job - secret_names=("azure-token" "sonar-token") - - for secret_name in "${secret_names[@]}"; do - # Convert to uppercase and replace hyphens with underscores - env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') - - # Retrieve the secret value - secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) - - # Mask the secret value - echo "::add-mask::$secret_value" - - # Export as environment variable - echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" - done - - - name: Overwrite default secrets with user-defined secrets - shell: bash - run: | - if [[ -n "${{ secrets.azureToken }}" ]]; then - echo "Using provided azureToken secret" - echo "AZURE_TOKEN=${{ secrets.azureToken }}" >> "$GITHUB_ENV" - fi - - if [[ -n "${{ secrets.sonarCloudToken }}" ]]; then - echo "Using provided sonarCloudToken secret" - echo "SONAR_TOKEN=${{ secrets.sonarCloudToken }}" >> "$GITHUB_ENV" - fi - - - name: Enable long paths - run: git config --global core.longpaths true - - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - java-version: 17 - distribution: 'zulu' - - - name: Install .NET Tools - run: | - dotnet tool install -g dotnet-sonarscanner - dotnet tool install -g Skyline.DataMiner.CICD.Tools.NuGetToggleOnBuild - dotnet tool install -g Skyline.DataMiner.CICD.Tools.NuGetPreBuildApplyBranchOrTag - dotnet tool install -g Skyline.DataMiner.CICD.Tools.NuGetValidateSkylineSpecifications - - - name: Find solution file - id: findSlnFile - run: | - if [[ -z "${{ inputs.solutionName }}" ]]; then - echo solutionFilePath=$(find . -type f -name '*.sln' -o -name '*.slnx' | sort -r | head -n 1) >> $GITHUB_OUTPUT - else - echo solutionFilePath=$(find . -type f -name '${{ inputs.solutionName }}') >> $GITHUB_OUTPUT - fi - shell: bash - - - name: Validate NuGet Metadata - run: NuGetValidateSkylineSpecifications --workspace ${{ github.workspace }} --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" - - - name: Apply Branch and output path for pre-release NuGet - if: github.ref_type == 'branch' - run: NuGetPreBuildApplyBranchOrTag --workspace ${{ github.workspace }} --tag " " --branch "${{ github.ref_name }}" --build ${{ github.run_number }} --nugetResultFolder "${{ github.workspace }}/_NuGetResults" --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" - - - name: Apply Tag and output path for Release NuGet - if: github.ref_type == 'tag' - run: NuGetPreBuildApplyBranchOrTag --workspace ${{ github.workspace }} --tag "${{ github.ref_name }}" --branch " " --build ${{ github.run_number }} --nugetResultFolder "${{ github.workspace }}/_NuGetResults" --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" - - - name: Enable Skyline NuGet Registries - if: github.repository_owner == 'SkylineCommunications' - run: | - $sources = @( - @{ Name = "PrivateGitHubNugets"; URL = "https://nuget.pkg.github.com/SkylineCommunications/index.json"; Username = "USERNAME"; Password = "${{ secrets.GITHUB_TOKEN }}" }, - @{ Name = "CloudNuGets"; URL = "https://pkgs.dev.azure.com/skyline-cloud/Cloud_NuGets/_packaging/CloudNuGet/nuget/v3/index.json"; Username = "az"; Password = "${{ env.AZURE_TOKEN }}" }, - @{ Name = "PrivateAzureNuGets"; URL = "https://pkgs.dev.azure.com/skyline-cloud/_packaging/skyline-private-nugets/nuget/v3/index.json"; Username = "az"; Password = "${{ env.AZURE_TOKEN }}" } - ) - - foreach ($source in $sources) { - if ($source.Password -ne "") { - Write-Host "Checking source $($source.Name)..." - - if (dotnet nuget list source | Select-String -Pattern $source.Name) { - Write-Host "Updating existing source $($source.Name)." - dotnet nuget update source $source.Name --source $source.URL --username $source.Username --password $source.Password --store-password-in-clear-text - } else { - Write-Host "Adding new source $($source.Name)." - dotnet nuget add source $source.URL --name $source.Name --username $source.Username --password $source.Password --store-password-in-clear-text - } - } else { - Write-Host "Skipping $($source.Name) because the password is not set." - } - } - shell: pwsh - - - name: Prepare SonarCloud Variables - id: prepSonarCloudVar - run: | - import os - env_file = os.getenv('GITHUB_ENV') - with open(env_file, "a") as myfile: - myfile.write("lowerCaseOwner=" + str.lower("${{ github.repository_owner }}")) - shell: python - - - name: Get SonarCloud Status - id: get-sonarcloud-status - run: | - sonarCloudProjectStatus=$(curl -s -u "${{ env.SONAR_TOKEN }}:" "https://sonarcloud.io/api/qualitygates/project_status?projectKey=${{ inputs.sonarCloudProjectName }}&branch=${{ github.ref_name }}") - - # Check if the response is empty or not valid JSON - if [ -z "$sonarCloudProjectStatus" ] || ! echo "$sonarCloudProjectStatus" | jq . > /dev/null 2>&1; then - echo "Error: The SONAR_TOKEN is invalid, expired, or the response is empty. Please check: https://sonarcloud.io/account/security and update your token: https://github.com/${{ github.repository }}/settings/secrets/actions" >&2 - echo "Returned response: $sonarCloudProjectStatus" >&2 - exit 1 - fi - - # Check if the response contains errors - if echo "$sonarCloudProjectStatus" | jq -e '.errors' > /dev/null 2>&1; then - echo "Error: SonarCloud API returned errors. Initial analysis needed." >&2 - echo "Returned response: $sonarCloudProjectStatus" >&2 - echo "needsInitialAnalysis=true" >> $GITHUB_OUTPUT - exit 0 - fi - - # Check if project status is NONE (needs initial analysis) - projectStatus=$(echo "$sonarCloudProjectStatus" | jq -r '.projectStatus.status // empty') - if [ "$projectStatus" = "NONE" ]; then - echo "Project status is NONE. Initial analysis needed." - echo "needsInitialAnalysis=true" >> $GITHUB_OUTPUT - else - echo "needsInitialAnalysis=false" >> $GITHUB_OUTPUT - fi - - # Output the JSON response if valid - echo "Returned response: $sonarCloudProjectStatus" - shell: bash - - - name: Trigger Initial Analysis - if: steps.get-sonarcloud-status.outputs.needsInitialAnalysis == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner begin ` - /k:"${{ inputs.sonarCloudProjectName }}" ` - /o:"${{ env.lowerCaseOwner }}" ` - /d:sonar.token="${{ env.SONAR_TOKEN }}" ` - /d:sonar.host.url="https://sonarcloud.io" ` - /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" ` - /d:sonar.cs.vstest.reportsPaths="**/TestResults/**.trx" ` - /d:sonar.exclusions="**/*.yml,**/*.xml" ` - /d:sonar.branch.name="${{ github.ref_name }}" - dotnet build "${{ steps.findSlnFile.outputs.solutionFilePath }}" -p:DefineConstants="DCFv1%3BDBInfo%3BALARM_SQUASHING" --configuration Release -nodeReuse:false - dotnet sonarscanner end /d:sonar.token="${{ env.SONAR_TOKEN }}" - continue-on-error: true - - - name: Start Analysis - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner begin ` - /k:"${{ inputs.sonarCloudProjectName }}" ` - /o:"${{ env.lowerCaseOwner }}" ` - /d:sonar.token="${{ env.SONAR_TOKEN }}" ` - /d:sonar.host.url="https://sonarcloud.io" ` - /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" ` - /d:sonar.cs.vstest.reportsPaths="**/TestResults/**.trx" ` - /d:sonar.exclusions="**/*.yml,**/*.xml" ` - /d:sonar.branch.name="${{ github.ref_name }}" - continue-on-error: true - - - name: Building - run: dotnet build "${{ steps.findSlnFile.outputs.solutionFilePath }}" -p:DefineConstants="DCFv1%3BDBInfo%3BALARM_SQUASHING" --configuration Release -nodeReuse:false - - - uses: actions/upload-artifact@v6 - with: - name: NugetPackages - path: "${{ github.workspace }}/_NuGetResults" - - - name: Disable NuGet Creation on Subsequent Builds - run: NuGetToggleOnBuild --setToActive false --workspace ${{ github.workspace }} --onlyOnConfiguredNuGet false --solution-filepath "${{ steps.findSlnFile.outputs.solutionFilePath }}" - - - name: Unit Tests - # when not using MSTest you'll need to install coverlet.collector nuget in your test solutions - id: unit-tests - run: dotnet test "${{ steps.findSlnFile.outputs.solutionFilePath }}" --filter TestCategory!=IntegrationTest --logger "trx;logfilename=unitTestResults.trx" --collect "XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura,opencover - continue-on-error: true - - - name: Stop Analysis - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - run: | - dotnet sonarscanner end /d:sonar.token="${{ env.SONAR_TOKEN }}" - continue-on-error: true - - - name: SonarCloud Quality Gate check - id: sonarcloud-quality-gate-check - uses: sonarsource/sonarqube-quality-gate-action@master - with: - scanMetadataReportFile: .sonarqube/out/.sonar/report-task.txt - continue-on-error: true - # Force to fail step after specific time. - timeout-minutes: 5 - - - name: Upload Sonar report-task.txt artifact - continue-on-error: true - uses: actions/upload-artifact@v6 - with: - name: sonar-report-task - path: .sonarqube/out/.sonar/report-task.txt - if-no-files-found: ignore - retention-days: 14 - - - name: Quality Gate - id: quality-step - run: | - if "${{ steps.unit-tests.outcome }}" == "failure" or "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" == "FAILED": - print("Quality gate failed due to:") - if "${{ steps.unit-tests.outcome }}" == "failure": - print("- Test failures") - if "${{ steps.sonarcloud-quality-gate-check.outcome }}" == "failure" and "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" != "FAILED": - print("- Could not retrieve SonarCloud quality gate status, potentially due to reaching License max LoC. Ignoring SonarCloud quality gate status in this case.") - if "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" == "FAILED": - print("- Code analysis quality gate failed") - if "${{ steps.unit-tests.outcome }}" == "failure" or "${{ steps.sonarcloud-quality-gate-check.outputs.quality-gate-status }}" == "FAILED": - exit(1) - shell: python - - # Signing cannot be done from linux environment (https://github.com/dotnet/runtime/issues/48794) - sign: - # Don't run the signing when dependabot branch/pull request - if: ${{ github.actor != 'dependabot[bot]' }} - runs-on: windows-latest - needs: [validate_skyline_quality_gate, check_oidc] - steps: - - name: Azure Login - uses: azure/login@v2 - if: needs.check_oidc.outputs.use-oidc == 'true' - with: - client-id: ${{ needs.check_oidc.outputs.client-id }} - tenant-id: ${{ needs.check_oidc.outputs.tenant-id }} - subscription-id: ${{ needs.check_oidc.outputs.subscription-id }} - - - name: Retrieve needed secrets from Azure Key Vault - if: needs.check_oidc.outputs.use-oidc == 'true' - shell: bash - run: | - echo "Fetching secrets from Azure Key Vault..." - - # List of secret names needed for this job - secret_names=("signing-client-id" "signing-client-secret" "signing-tenant-id" "signing-key-vault-certificate" "signing-key-vault-url") - - for secret_name in "${secret_names[@]}"; do - # Convert to uppercase and replace hyphens with underscores - env_var_name=$(echo "$secret_name" | tr '[:lower:]' '[:upper:]' | tr '-' '_') - - # Retrieve the secret value - secret_value=$(az keyvault secret show --vault-name kv-master-cicd-secrets --name "$secret_name" --query value -o tsv) - - # Mask the secret value - echo "::add-mask::$secret_value" - - # Export as environment variable - echo "$env_var_name=$secret_value" >> "$GITHUB_ENV" - done - - - name: Download Unsigned NuGet - id: downloadUnsignedNuget - uses: actions/download-artifact@v7 - with: - name: NugetPackages - path: _NuGetResults - - - name: Install dotnet sign - run: dotnet tool install sign --global --prerelease - - - name: Sign NuGet Package - env: - AZURE_TENANT_ID: ${{ env.SIGNING_TENANT_ID }} - AZURE_CLIENT_ID: ${{ env.SIGNING_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ env.SIGNING_CLIENT_SECRET }} - run: | - IFS=$'\n' - sign code azure-key-vault "_NuGetResults/**/*.nupkg" --publisher-name "Skyline Communications" --description "Skyline Signing" --description-url "https://www.skyline.be/" --azure-key-vault-certificate "${{ env.SIGNING_KEY_VAULT_CERTIFICATE }}" --azure-key-vault-url "${{ env.SIGNING_KEY_VAULT_URL }}" --output "_SignedNuGetResults" - unset IFS - shell: bash - - - uses: actions/upload-artifact@v6 - with: - name: SignedNugetPackages - path: "${{ github.workspace }}/_SignedNuGetResults" - - push: - if: github.ref_type == 'tag' - name: push - runs-on: ubuntu-latest - needs: sign - steps: - - name: Download Signed NuGet - id: downloadSignedNuget - uses: actions/download-artifact@v7 - with: - name: SignedNugetPackages - path: _SignedNuGetResults - - - name: Find Nuget - id: findcreatednuget - run: | - IFS=$'\n' - echo nugetPackageName=$(find _SignedNuGetResults -type f -name '*.nupkg') >> $GITHUB_OUTPUT - unset IFS - shell: bash - - - name: Push to NuGet.org - run: | - IFS=$'\n' - for i in ${{ steps.findcreatednuget.outputs.nugetPackageName }}; - do - dotnet nuget push "$i" --api-key ${{ secrets.nugetApiKey }} --source https://api.nuget.org/v3/index.json - done - unset IFS - shell: bash + master_workflow: + uses: ./.github/workflows/Master Workflow.yml + with: + oidc-client-id: ${{ inputs.oidc-client-id }} + oidc-tenant-id: ${{ inputs.oidc-tenant-id }} + oidc-subscription-id: ${{ inputs.oidc-subscription-id }} + sonarcloud-project-name: ${{ inputs.sonarCloudProjectName }} + solution-filter-name: ${{ inputs.solutionName }} + nuget-push-source: "https://api.nuget.org/v3/index.json" + secrets: + NUGET_API_KEY: ${{ secrets.nugetApiKey }} + SONAR_TOKEN: ${{ secrets.sonarCloudToken }} + AZURE_TOKEN: ${{ secrets.azureToken }} \ No newline at end of file From df1d7cbec373b2720ae82d9fd29c5d013eea5b83 Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Mon, 30 Mar 2026 09:00:51 +0200 Subject: [PATCH 02/13] Add logging to test --- .github/workflows/Master Workflow.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 45c3df1..1b50cff 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -644,6 +644,15 @@ jobs: # D. Test & Quality Gate # ────────────────────────────────────────────────────────────── + - name: Log .NET versions + run: | + echo "Installed .NET SDKs:" + dotnet --list-sdks + echo "Installed .NET Runtimes:" + dotnet --list-runtimes + echo "Installed .NET versions:" + dotnet --version + - name: Unit Tests (VSTest) id: unit-tests-vstest if: steps.detect-test-mode.outputs.test-runner-mode == 'vstest' From 34ced894660637d1774546e9f77220264633a607 Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Mon, 30 Mar 2026 09:42:24 +0200 Subject: [PATCH 03/13] Allow different runner for CI job --- .github/workflows/Master Workflow.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 1b50cff..031e3fb 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -37,6 +37,11 @@ on: type: boolean default: false + runs-on: + required: false + type: string + default: 'ubuntu-latest' + # One or more lines with =. # Example: # PackageA.Install/CatalogInformation/manifest.yml=11111111-1111-1111-1111-111111111111 @@ -165,7 +170,7 @@ jobs: ci: name: CI needs: [check_oidc, discover_projects] - runs-on: ubuntu-latest + runs-on: ${{ inputs.runs-on }} outputs: has-nuget-packages: ${{ steps.detect-nuget-packages.outputs.has-nuget-packages }} has-dataminer-packages: ${{ steps.detect-dataminer-packages.outputs.has-dataminer-packages }} @@ -298,6 +303,7 @@ jobs: fetch-depth: 0 - name: Cache and Install Mono + if: runner.os == 'Linux' uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 with: packages: mono-complete @@ -644,15 +650,6 @@ jobs: # D. Test & Quality Gate # ────────────────────────────────────────────────────────────── - - name: Log .NET versions - run: | - echo "Installed .NET SDKs:" - dotnet --list-sdks - echo "Installed .NET Runtimes:" - dotnet --list-runtimes - echo "Installed .NET versions:" - dotnet --version - - name: Unit Tests (VSTest) id: unit-tests-vstest if: steps.detect-test-mode.outputs.test-runner-mode == 'vstest' From e1e6e2a5dc657a664e77870ab431094d1587a16a Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Mon, 30 Mar 2026 09:51:17 +0200 Subject: [PATCH 04/13] Default to bash for ci step unless specified on step --- .github/workflows/Master Workflow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 031e3fb..5756a12 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -171,6 +171,9 @@ jobs: name: CI needs: [check_oidc, discover_projects] runs-on: ${{ inputs.runs-on }} + defaults: + run: + shell: bash outputs: has-nuget-packages: ${{ steps.detect-nuget-packages.outputs.has-nuget-packages }} has-dataminer-packages: ${{ steps.detect-dataminer-packages.outputs.has-dataminer-packages }} From e9353e151ab45609875b2b633e7d6001770cba17 Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Mon, 30 Mar 2026 10:59:43 +0200 Subject: [PATCH 05/13] Fix issues --- .github/workflows/Master Workflow.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 5756a12..357747b 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -579,6 +579,7 @@ jobs: BRANCH_NAME: ${{ github.ref_name }} SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} CONFIGURATION: ${{ inputs.configuration }} + MSYS_NO_PATHCONV: '1' run: | dotnet sonarscanner begin \ /k:"$SONAR_PROJECT_NAME" \ @@ -601,6 +602,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_PROJECT_NAME: ${{ inputs.sonarcloud-project-name }} BRANCH_NAME: ${{ github.ref_name }} + MSYS_NO_PATHCONV: '1' run: | dotnet sonarscanner begin \ /k:"$SONAR_PROJECT_NAME" \ @@ -691,6 +693,7 @@ jobs: if: github.actor != 'dependabot[bot]' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MSYS_NO_PATHCONV: '1' run: | dotnet sonarscanner end /d:sonar.token="$SONAR_TOKEN" continue-on-error: true @@ -944,9 +947,11 @@ jobs: - name: Find NuGet packages id: findcreatednuget run: | - IFS=$'\n' - echo nugetPackageName=$(find _SignedNuGetResults -type f -name '*.nupkg') >> $GITHUB_OUTPUT - unset IFS + { + echo "nugetPackageName<> $GITHUB_OUTPUT shell: bash - name: Push NuGet packages From fbe1a737ad26c4f7beaa23d00dadf1440ccb5601 Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Tue, 31 Mar 2026 09:42:41 +0200 Subject: [PATCH 06/13] Remove dotnet build warnings on initial SonarCloud analysis --- .github/workflows/Master Workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 357747b..fb715ce 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -592,7 +592,8 @@ jobs: -p:GenerateDataMinerPackage=false \ -p:GeneratePackageOnBuild=false \ --configuration "$CONFIGURATION" \ - -nodeReuse:false + -nodeReuse:false \ + -p:WarningLevel=0 dotnet sonarscanner end /d:sonar.token="$SONAR_TOKEN" continue-on-error: true From 0900e4799cd8a4e4b34e8f4bbac8ee72830f73eb Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Tue, 31 Mar 2026 09:52:04 +0200 Subject: [PATCH 07/13] Mark validator as prerelease --- .github/workflows/Connector Master SDK Workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Connector Master SDK Workflow.yml b/.github/workflows/Connector Master SDK Workflow.yml index 7778dc8..da6945e 100644 --- a/.github/workflows/Connector Master SDK Workflow.yml +++ b/.github/workflows/Connector Master SDK Workflow.yml @@ -115,7 +115,7 @@ jobs: - name: Install .NET Tools run: | dotnet tool install -g dotnet-sonarscanner - dotnet tool install -g Skyline.DataMiner.CICD.Tools.Validator --version 3.* + dotnet tool install -g Skyline.DataMiner.CICD.Tools.Validator --prerelease dotnet tool install -g Skyline.DataMiner.CICD.Tools.Sbom --version 1.* - name: Enable Skyline NuGet Registries From 6dab1ff4fe55306191474e4a25a20832617ffda4 Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Tue, 31 Mar 2026 11:48:25 +0200 Subject: [PATCH 08/13] Revert "Mark validator as prerelease" This reverts commit 0900e4799cd8a4e4b34e8f4bbac8ee72830f73eb. --- .github/workflows/Connector Master SDK Workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Connector Master SDK Workflow.yml b/.github/workflows/Connector Master SDK Workflow.yml index da6945e..7778dc8 100644 --- a/.github/workflows/Connector Master SDK Workflow.yml +++ b/.github/workflows/Connector Master SDK Workflow.yml @@ -115,7 +115,7 @@ jobs: - name: Install .NET Tools run: | dotnet tool install -g dotnet-sonarscanner - dotnet tool install -g Skyline.DataMiner.CICD.Tools.Validator --prerelease + dotnet tool install -g Skyline.DataMiner.CICD.Tools.Validator --version 3.* dotnet tool install -g Skyline.DataMiner.CICD.Tools.Sbom --version 1.* - name: Enable Skyline NuGet Registries From d009c0feecfb1f57d6936f1e7f674119d64513f5 Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Thu, 2 Apr 2026 09:35:53 +0200 Subject: [PATCH 09/13] Use prerelease of Sdk --- .github/workflows/Master Workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index fb715ce..919415a 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -68,7 +68,7 @@ on: required: false env: - VERSION_SDK: '2.4.7' + VERSION_SDK: '2.5.0-alpha' VERSION_APPPACKAGEINSTALLER: '4.0.0' jobs: From d79ca9c1921ec64a4eff553caf53d5d33fbd8e1d Mon Sep 17 00:00:00 2001 From: Michiel Oda Date: Thu, 2 Apr 2026 09:51:33 +0200 Subject: [PATCH 10/13] Also set PackageVersion (as NuGets take preference to that one) --- .github/workflows/Master Workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 919415a..70691f3 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -68,7 +68,7 @@ on: required: false env: - VERSION_SDK: '2.5.0-alpha' + VERSION_SDK: '0.0.800' VERSION_APPPACKAGEINSTALLER: '4.0.0' jobs: @@ -645,6 +645,7 @@ jobs: run: | dotnet build "$SOLUTION_PATH" \ -p:Version="$VERSION" \ + -p:PackageVersion="$VERSION" \ -p:DefineConstants="DCFv1%3BDBInfo%3BALARM_SQUASHING" \ --configuration "$CONFIGURATION" \ -p:CatalogPublishKeyName="DATAMINER_TOKEN" \ From 66b6cd0051f58f1ae6db1d061fc3b2fe1c4652ae Mon Sep 17 00:00:00 2001 From: Jan Staelens Date: Thu, 2 Apr 2026 17:48:43 +0200 Subject: [PATCH 11/13] dotnet test per project. don't test integration test projects. --- .github/workflows/Master Workflow.yml | 82 ++++++++++++++++++++------- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 70691f3..57094f1 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -656,21 +656,26 @@ jobs: # ────────────────────────────────────────────────────────────── # D. Test & Quality Gate # ────────────────────────────────────────────────────────────── - - name: Unit Tests (VSTest) id: unit-tests-vstest if: steps.detect-test-mode.outputs.test-runner-mode == 'vstest' env: SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} CONFIGURATION: ${{ inputs.configuration }} + shell: bash run: | - dotnet test "$SOLUTION_PATH" \ - --no-build \ - --configuration "$CONFIGURATION" \ - --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ - --logger "trx;logfilename=unitTestResults.trx" \ - --collect "XPlat Code Coverage" \ - -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura,opencover + while IFS= read -r project; do + [[ "$project" == *".Integration"* ]] && continue + + dotnet test "$project" \ + --no-build \ + --configuration "$CONFIGURATION" \ + --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ + --logger "trx;logfilename=$(basename "$project" .csproj).trx" \ + --results-directory "TestResults" \ + --collect "XPlat Code Coverage" \ + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura,opencover + done < <(dotnet sln "$SOLUTION_PATH" list | tail -n +3 | grep '\.csproj$') - name: Unit Tests (MTP) id: unit-tests-mtp @@ -678,18 +683,57 @@ jobs: env: SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} CONFIGURATION: ${{ inputs.configuration }} + shell: bash run: | - dotnet test \ - --solution "$SOLUTION_PATH" \ - --no-build \ - --configuration "$CONFIGURATION" \ - --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ - --report-trx \ - --report-trx-filename unitTestResults.trx \ - --coverage \ - --coverage-output-format xml \ - --coverage-output coverage.xml \ - --ignore-exit-code 8 + while IFS= read -r project; do + [[ "$project" == *".Integration"* ]] && continue + + dotnet test "$project" \ + --no-build \ + --configuration "$CONFIGURATION" \ + --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ + --results-directory "TestResults" \ + --report-trx \ + --report-trx-filename "$(basename "$project" .csproj).trx" \ + --coverage \ + --coverage-output-format xml \ + --coverage-output "TestResults/$(basename "$project" .csproj).coverage.xml" \ + --ignore-exit-code 8 + done < <(dotnet sln "$SOLUTION_PATH" list | tail -n +3 | grep '\.csproj$') + + # - name: Unit Tests (VSTest) + # id: unit-tests-vstest + # if: steps.detect-test-mode.outputs.test-runner-mode == 'vstest' + # env: + # SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + # CONFIGURATION: ${{ inputs.configuration }} + # run: | + # dotnet test "$SOLUTION_PATH" \ + # --no-build \ + # --configuration "$CONFIGURATION" \ + # --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ + # --logger "trx;logfilename=unitTestResults.trx" \ + # --collect "XPlat Code Coverage" \ + # -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura,opencover + + # - name: Unit Tests (MTP) + # id: unit-tests-mtp + # if: steps.detect-test-mode.outputs.test-runner-mode == 'mtp' + # env: + # SOLUTION_PATH: ${{ needs.discover_projects.outputs.solution-path }} + # CONFIGURATION: ${{ inputs.configuration }} + # run: | + # dotnet test \ + # --solution "$SOLUTION_PATH" \ + # --no-build \ + # --configuration "$CONFIGURATION" \ + # --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ + # --report-trx \ + # --report-trx-filename unitTestResults.trx \ + # --coverage \ + # --coverage-output-format xml \ + # --coverage-output coverage.xml \ + # --ignore-exit-code 8 - name: Stop Analysis if: github.actor != 'dependabot[bot]' From f8e783297dd25611b4ade20ddf7151a6d1e708e0 Mon Sep 17 00:00:00 2001 From: Jan Staelens Date: Fri, 3 Apr 2026 10:29:51 +0200 Subject: [PATCH 12/13] Simplify coverage output in dotnet test command Refactor dotnet test command to simplify coverage output. --- .github/workflows/Master Workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 57094f1..4407b87 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -688,7 +688,7 @@ jobs: while IFS= read -r project; do [[ "$project" == *".Integration"* ]] && continue - dotnet test "$project" \ + dotnet test --project "$project" \ --no-build \ --configuration "$CONFIGURATION" \ --filter "TestCategory!=IntegrationTest&TestCategory!=IntegrationTests" \ @@ -697,7 +697,7 @@ jobs: --report-trx-filename "$(basename "$project" .csproj).trx" \ --coverage \ --coverage-output-format xml \ - --coverage-output "TestResults/$(basename "$project" .csproj).coverage.xml" \ + --coverage-output coverage.xml \ --ignore-exit-code 8 done < <(dotnet sln "$SOLUTION_PATH" list | tail -n +3 | grep '\.csproj$') @@ -1212,4 +1212,4 @@ jobs: -p:CatalogPublishKeyName="DATAMINER_TOKEN" ` --configuration "$env:CONFIGURATION" ` -p:SkylineDataMinerSdkDebug="$env:DEBUG_FLAG" ` - -p:IsPublishable=false \ No newline at end of file + -p:IsPublishable=false From 82dfed22202788f17caaf5688ab124f4f9ebb567 Mon Sep 17 00:00:00 2001 From: Jan Staelens Date: Fri, 3 Apr 2026 10:31:12 +0200 Subject: [PATCH 13/13] Update Master Workflow.yml --- .github/workflows/Master Workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Master Workflow.yml b/.github/workflows/Master Workflow.yml index 4407b87..293c2b2 100644 --- a/.github/workflows/Master Workflow.yml +++ b/.github/workflows/Master Workflow.yml @@ -665,7 +665,7 @@ jobs: shell: bash run: | while IFS= read -r project; do - [[ "$project" == *".Integration"* ]] && continue + [[ "$project" == *"Integration"* ]] && continue dotnet test "$project" \ --no-build \ @@ -686,7 +686,7 @@ jobs: shell: bash run: | while IFS= read -r project; do - [[ "$project" == *".Integration"* ]] && continue + [[ "$project" == *"Integration"* ]] && continue dotnet test --project "$project" \ --no-build \