Expanded Azure DevOps pipeline template including stand-alone deployment pipeline

Templates/AzureDevOpsPipeline/azure-pipeline-deploy.yml

@@ -0,0 +1,141 @@
+#
+# DEPLOYMENT PIPELINE
+#
+# This is a manually triggered pipeline that deploys a referenced archive of the application into the selected runtime environment
+# using IBM Wazi Deploy
+#
+#  It leverages the conventions of the Common Backend scripts, and assumes that the archive (a.k.a the package)
+#  has been uploaded to Artifactory or Nexus
+# 
+#  It relates to the environments, that have been configured for the project in the ADO project
+
+
+# Configuration
+
+  ### Variables to be created in Azure pipeline configuration
+
+  # ) agentPool                            # Agent pool name for Azure Agents to connect to MVS
+  # ) zosSSHConnection                     # zOS - SSH connection name
+  # ) zosSSHKnownHost                      # Known host entry for secure shell connections
+  # ) zosSSHPrivateKeySecureFile           # Reference to uploaded Private SSH Key that in ADO Pipeline/Libary/SecureFile that is
+  #                                          that installed for sftp processes to fetch logs
+  # ) pipelineWorkspace                    # Root directory on z/OS Unix System services to perform build and deployments. E.g. `/u/ado/workspace`
+  # ) zosHostname                          # zOS - Host IP for SFTP connection
+  # ) zosSFTPUser                          # zOS - Host user for SFTP connection
+
+  ### Environment variables to be added in z/OS Unix System Services $HOME/.profile
+  #  To locate the backend scripts. See also Installation of backend scripts.
+  #  PIPELINE_SCRIPTS:
+  #  Ex: export PIPELINE_SCRIPTS=/var/dbb/Common-Backend-Scripts
+  #      export PATH=$PIPELINE_SCRIPTS:$PATH
+
+trigger: none
+
+parameters:
+  - name: deploymentEnvironment
+    displayName: "Target deployment environment - Environment where the package should be deployed to"
+    type: string
+    default: Acceptance
+    values: 
+     - Integration
+     - Acceptance
+     - Production  
+  - name: archiveType
+    displayName: "Archive Type - a preliminary build or release candidate"
+    type: string
+    default: build
+    values:
+      - build
+      - release
+  - name: archiveReference
+    displayName: "Archive Reference - either the branch name (like 'main') or release name (like 'rel-1.0.0') for the archive"
+    type: string
+    default: main
+  - name: archiveBuildIdentifier
+    displayName: "Archive Identifier - Identifier for the archive, like build number or timestamp"
+    type: string
+    default: "build-identifier"
+
+# pipeline templates
+resources:
+  repositories:
+    - repository: templates
+      type: git
+      name: PipelineCore/PipelineCore
+
+# Agent pool where the pipeline runs
+pool:
+  name: $(agentPool)
+
+# Variables used in the pipeline
+variables:
+
+  ### Variables generated during run time
+  organization: $[ replace(variables['organization_tmp'], '/', '') ] # Azure organization name
+  project: $(System.TeamProject) # Project name
+  application: ${{ variables['Build.Repository.Name'] }} # Application name
+  repo: "git@ssh.dev.azure.com:v3/$(organization)/$(project)/$(application)" # Git repository path
+  uniqueWorkspaceId: $(pipelineWorkspace)/$(application)/deployments/${{ parameters.archiveType }}/${{ parameters.archiveReference }}/build-$(Build.BuildNumber) # Calculated workspace path
+  packageName: $(application) # default name of the Package
+  buildNumber: ${{ parameters.archiveBuildIdentifier }} # Build reference
+  pipelineType: ${{ parameters.archiveType }}
+  branch: ${{ parameters.archiveReference }}
+
+# Pipeline stages
+stages:
+  - stage: Setup
+    jobs:
+      - job: Setup
+        continueOnError: false
+        steps:
+          - checkout: none
+          # Display CI-CD pipeline parms
+          - script: |
+              echo " "
+              echo " Deployment pipeline parameters"
+              echo "*******************************************"
+              echo "organization               = $(organization)"
+              echo "project                    = $(project)"
+              echo "application                = $(application)"
+              echo "repo                       = $(repo)"
+              echo " Archive information"
+              echo "archiveReference           = $${{ parameters.archiveReference }}"
+              echo "archiveBuildIdentifier     = $(buildNumber)"
+              echo "archiveType                = $(pipelineType)"
+              echo " Connectivity"
+              echo "zosSSHConnection           = $(zosSSHConnection)"
+              echo "zosHostname                = $(zosHostname)"
+              echo "zosSFTPUser                = $(zosSFTPUser)"
+              echo "ussWorkspaceRoot           = $(pipelineWorkspace)"
+              echo "ussWorkspaceDir            = $(uniqueWorkspaceId)"
+              echo " "
+            displayName: "Deploy pipeline parameters"
+
+  # Production environment
+  - stage: Deploy
+    displayName: "Deploy Package"
+    dependsOn: 
+      - Setup
+    jobs:
+      - template: deployment/deployPackage.yml@templates
+        parameters:
+          environmentName: ${{ parameters.deploymentEnvironment }}
+          packageReference: "${{ parameters.archiveReference }}"
+          packageIdentifier: "${{ parameters.archiveBuildIdentifier }}"
+          packageType: "${{ parameters.archiveType }}"
+
+  # Cleanup
+  - stage: Cleanup
+    condition: always()
+    jobs:
+      - job: Cleanup
+        displayName: "Cleanup Deployment Workspace"
+        steps:
+          - checkout: none
+          - task: SSH@0
+            inputs:
+              sshEndpoint: $(zosSSHConnection)
+              runOptions: "commands"
+              commands: ". ./.profile && deleteWorkspace.sh -w $(uniqueWorkspaceId)"
+              readyTimeout: "20000"
+            displayName: "Delete Deployment folder on USS"

Templates/AzureDevOpsPipeline/azure-pipelines.yml

@@ -1,12 +1,11 @@
-# Branches that triggers this pipleline
+# Branches that triggers this pipeline
 trigger:
   - feature/*
   - release/*
   - hotfix/*
   - epic/*
   - main
 
-# Pipeline input parameters to drive build and release pipelines
 parameters:
   - name: pipelineType
     displayName: "Pipeline Type"
@@ -26,7 +25,7 @@ parameters:
       - patch
 
   - name: verboseLogging
-    displayName: "zAppBuild verbose logging"
+    displayName: "Verbose Logging"
     type: boolean
     default: false
 
@@ -60,37 +59,31 @@ variables:
   #  Ex: export PIPELINE_SCRIPTS=/var/dbb/Common-Backend-Scripts
   #      export PATH=$PIPELINE_SCRIPTS:$PATH
 
-  ### Input parameter variables:
-  pipelineType: ${{ parameters.pipelineType }}
-  releaseType: ${{ parameters.releaseType }}
-  # Computation DBB verbose logging
-  ${{ if eq(parameters.verboseLogging, true) }}: 
-    verbose: "-v"
-  ${{ if eq(parameters.verboseLogging, false) }}:
-    verbose: ""
-
   ### Variables generated during run time
   ${{ if startsWith(variables['Build.SourceBranch'], 'refs/heads/') }}: # Extract branch name for push
     branch: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ]
   ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: # Extract branch name for pull request
     branch: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ]
   buildNumber: $(Build.BuildNumber) # Build number
-  organization_tmp: $[ replace(variables['System.TeamFoundationCollectionUri'], 'https://dev.azure.com/', '') ] # Generate Azure repository uri for cloning
-  organization: $[ replace(variables['organization_tmp'], '/', '') ] # Azure organization name
   project: $(System.TeamProject) # Project name
   application: ${{ variables['Build.Repository.Name'] }} # Application name
-  repo: "git@ssh.dev.azure.com:v3/$(organization)/$(project)/$(application)" # Git repository path
-  uniqueWorkspaceId: $(pipelineWorkspace)/$(application)/$(branch)/build-$(buildNumber) # Calculated worksapce path
-  packageName: $(application) # Name of the Package
+  repo: ${{ variables['Build.Repository.Uri'] }}" # retrieves the http uri of the triggering project
+  uniqueWorkspaceId: $(pipelineWorkspace)/$(application)/$(branch)/build-$(buildNumber) # Calculated workspace path
+  # packageName: $(application) # Name of the Package
+  pipelineType: ${{ parameters.pipelineType }}
+  releaseType: ${{ parameters.releaseType }}
   isMain: $[contains(variables['Build.SourceBranch'], 'main')] # Variable to indicate pipeline for the main branch
-
-  # Azure package name. Package names must contain only lowercase alphanumeric segments separated by a dash, dot or underscore.
-  # reformattedBranchName: $[ replace(variables['branch'], '/', '-')]
-  # lowerApplicationName: $[ lower(variables['application']) ]
-  # lowerReformattedBranchName: $[ lower(variables['reformattedBranchName']) ]
-  # azureArtifactName: $(lowerReformattedBranchName)-$(lowerApplicationName).tar
-
-
+  # Computation DBB verbose logging
+  ${{ if eq(parameters.verboseLogging, true) }}: 
+    verbose: "-v"
+  ${{ if eq(parameters.verboseLogging, false) }}:
+    verbose: ""
+  # Azure artifact upload. Package names must contain only lowercase alphanumeric segments separated by a dash, dot or underscore.
+  # Kept for reference for users wanting to upload to Azure Artifacts instead of Artifactory/Nexus
+  #reformattedBranchName: $[ replace(variables['branch'], '/', '-')]
+  #lowerApplicatioName: $[ lower(variables['application']) ]
+  #lowerReformattedBranchName: $[ lower(variables['reformattedBranchName']) ]
+  #azureArtifactName: $(lowerReformattedBranchName)-$(lowerApplicatioName).tar
 # Pipeline stages
 stages:
   - stage: Setup
@@ -112,7 +105,6 @@ stages:
               echo "verbose                    = ${{ parameters.verboseLogging }} "
               echo "branch                     = $(branch)"
               echo "buildNumber                = $(buildNumber)"
-              echo "organization               = $(organization)"
               echo "project                    = $(project)"
               echo "application                = $(application)"
               echo "repo                       = $(repo)"
@@ -133,16 +125,15 @@ stages:
             inputs:
               sshEndpoint: $(zosSSHConnection)
               runOptions: "commands"
-              commands: ". ./.profile && gitClone.sh -w $(uniqueWorkspaceId) -r $(repo) -b $(branch)"
+              # Showcases how the System.Accesstoken can be passed to authenticate the pipeline for cloning via HTTP
+              commands: ". ./.profile && gitClone.sh -c \"AUTHORIZATION: Bearer $(System.AccessToken)\" -w $(uniqueWorkspaceId) -r $(repo) -b $(branch)"
               readyTimeout: "20000"
             displayName: "Clone application repo"
 
   ## Build stage
   # Invoke build and create a release candidate tag
 
   - stage: Build
-    dependsOn:
-      - Setup
     jobs:
       - job: Build
         displayName: "Build"
@@ -152,8 +143,8 @@ stages:
             inputs:
               sshEndpoint: $(zosSSHConnection)
               runOptions: "commands"
-              commands: ". ./.profile && dbbBuild.sh -w $(uniqueWorkspaceId) -a $(application) -b $(branch) -p $(pipelineType) ${{variables.verbose}}"
-              #commands: ". ./.profile && zBuilder.sh -w $(uniqueWorkspaceId) -a $(application) -b $(branch) -p $(pipelineType) ${{variables.verbose}}"
+             #commands: ". ./.profile && dbbBuild.sh -w $(uniqueWorkspaceId) -a $(application) -b $(branch) -p $(pipelineType) ${{variables.verbose}}"
+              commands: ". ./.profile && zBuilder.sh -w $(uniqueWorkspaceId) -a $(application) -b $(branch) -p $(pipelineType) ${{variables.verbose}}"
               readyTimeout: "20000"
               failOnStdErr: false
             displayName: "Build application"
@@ -215,30 +206,30 @@ stages:
         displayName: "Package Outputs"
         steps:
           - checkout: none
-          #- task: SSH@0
-          #  inputs:
-          #    sshEndpoint: $(zosSSHConnection)
-          #    runOptions: 'commands'
-          #    commands: '. ./.profile && ucdPackaging.sh -v $(packageName) -c $(application) -w $(uniqueWorkspaceId)  -b $(branch) -e $artifactRepoConfig'
-          #    readyTimeout: '20000'
-          #  displayName: 'Package the build artifacts for Deployment'
           - task: SSH@0
             condition: succeeded()
             inputs:
               sshEndpoint: $(zosSSHConnection)
               runOptions: "commands"
               commands: ". ./.profile && packageBuildOutputs.sh -w $(uniqueWorkspaceId) -a $(application) -b $(branch) -p $(pipelineType) -r $(packageVersion) -i $(buildNumber)"
-              # For reference when creating a local tar file, skipping the upload
-              #commands: ". ./.profile && packageBuildOutputs.sh -w $(uniqueWorkspaceId) -t $(application).tar -a $(application) -b $(branch) -p $(pipelineType) -v $(packageVersion)"
               readyTimeout: "20000"
-            displayName: "Package the build artifacts for Deployment"
+            displayName: "Package the build artifacts"
+          # Alternative way to use DevOps Deploy packaging
+          #- task: SSH@0
+          #  inputs:
+          #    sshEndpoint: $(zosSSHConnection)
+          #    runOptions: 'commands'
+          #    commands: '. ./.profile && ucdPackaging.sh -v $(packageName) -c $(application) -w $(uniqueWorkspaceId)  -b $(branch) -e $artifactRepoConfig'
+          #    readyTimeout: '20000'
+          #  displayName: 'Package the build artifacts for Deployment'
         variables:
             ${{ if eq(variables['pipelineType'], 'release') }}: # Retrieve the version name from the release candidate
               packageVersion: $[ stageDependencies.Build.Tag.outputs['computeReleaseName.nextReleaseVersion'] ]
-            ${{ if not(eq(variables['pipelineType'], 'release')) }}: # Default for all other cases
-              packageVersion: $(packageName)
+            ${{ if not(eq(variables['pipelineType'], 'release')) }}: # For all other cases
+              packageVersion: $(branch)
 
-        # Alternate upload option to demonstrate to upload package to Azure DevOps Artifacts
+        # Demonstrate to upload package to Azure DevOps Artifacts
+        ## TODO - Requires more work to distinguish build and release packages
 #      - job: PublishPackage
 #        displayName: "Publish deployment package"
 #        dependsOn: Package
@@ -280,13 +271,16 @@ stages:
       Environment: Integration
       ${{ if eq(variables['pipelineType'], 'release') }}: # Retrieve the version name from the release candidate
          packageVersion: $[ stageDependencies.Build.Tag.outputs['computeReleaseName.nextReleaseVersion'] ]
-      ${{ if not(eq(variables['pipelineType'], 'release')) }}: # Default for all other cases
-         packageVersion: $(packageName)
+      ${{ if not(eq(variables['pipelineType'], 'release')) }}: # For all other cases
+         packageVersion: $(branch)
+
     jobs:
       - template: deployment/deployPackage.yml@templates
         parameters:
           environmentName: ${{ variables.Environment }}
-          packageVersion: "$(packageVersion)"
+          packageReference: $(packageVersion)
+          packageIdentifier: $(buildNumber)
+          packageType: $(pipelineType)
 
   ## Deployment to higher test environments
   # deployments to these test environment can only happen when the development team requests a release pipeline
@@ -303,33 +297,41 @@ stages:
       ${{ if eq(variables['pipelineType'], 'release') }}: # Retrieve the version name from the release candidate
          packageVersion: $[ stageDependencies.Build.Tag.outputs['computeReleaseName.nextReleaseVersion'] ]
       ${{ if not(eq(variables['pipelineType'], 'release')) }}: # For all other cases
-         packageVersion: $(packageName)
+         packageVersion: $(branch)
     jobs:
       - template: deployment/deployPackage.yml@templates
         parameters:
           environmentName: ${{ variables.Environment }}
-          packageVersion: "$(packageVersion)"
+          packageReference: $(packageVersion)
+          packageIdentifier: $(buildNumber)
+          packageType: $(pipelineType)
 
   # Add additional deployment stages as you need.
+
+  # The below deployment is expected to be run through the deployment pipeline to drive 
+  # deployments independent of the CI pipeline
 
-  # Production environment
-  - stage: Deploy_PRODUCTION
-    displayName: "Deploy Production"
-    dependsOn: 
-      - Build
-      - Deploy_ACCEPTANCE
-    condition: and(succeeded(), eq(variables.isMain, true), eq(variables['pipelineType'], 'release'))
-    variables:
-      Environment: Production
-      ${{ if eq(variables['pipelineType'], 'release') }}: # Retrieve the version name from the release candidate
-         packageVersion: $[ stageDependencies.Build.Tag.outputs['computeReleaseName.nextReleaseVersion'] ]
-      ${{ if not(eq(variables['pipelineType'], 'release')) }}: # For all other cases
-         packageVersion: $(packageName)      
-    jobs:
-      - template: deployment/deployPackage.yml@templates
-        parameters:
-          environmentName: ${{ variables.Environment }}
-          packageVersion: "$(packageVersion)"
+#  # Production environment
+#  - stage: Deploy_PRODUCTION
+#    displayName: "Deploy Production"
+#    dependsOn: 
+#      - Build
+#      - Deploy_ACCEPTANCE
+#    condition: and(succeeded(), eq(variables.isMain, true), eq(variables['pipelineType'], 'release'))
+#    variables:
+#      Environment: Production
+#      ${{ if eq(variables['pipelineType'], 'release') }}: # Retrieve the version name from the release candidate
+#         packageVersion: $[ stageDependencies.Build.Tag.outputs['computeReleaseName.nextReleaseVersion'] ]
+#      ${{ if not(eq(variables['pipelineType'], 'release')) }}: # For all other cases
+#         packageVersion: $(branch)      
+#    jobs:
+#      - template: deployment/deployPackage.yml@templates
+#        parameters:
+#          environmentName: ${{ variables.Environment }}
+#          packageVersion: $(packageVersion)
+#          packageReference: $(packageVersion)
+#          packageIdentifier: $(buildNumber)
+#          packageType: $(pipelineType)
 
   # Cleanup
   - stage: Cleanup