diff --git a/.github/workflows/cicd-3-test-deploy.yaml b/.github/workflows/cicd-3-test-deploy.yaml deleted file mode 100644 index 22dc1437..00000000 --- a/.github/workflows/cicd-3-test-deploy.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: "3. CD | Deploy to Test" - -on: - workflow_run: - workflows: ["2. CD | Deploy to Dev"] - types: [completed] - -concurrency: - group: test-deployments - cancel-in-progress: false - -permissions: - contents: read - id-token: write - actions: read - -jobs: - metadata: - name: "Resolve metadata from triggering run" - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - outputs: - terraform_version: ${{ steps.vars.outputs.terraform_version }} - tag: ${{ steps.tag.outputs.name }} - steps: - - name: "Checkout exact commit from CI/CD publish" - uses: actions/checkout@v6 - with: - ref: ${{ github.event.workflow_run.head_sha }} - - - name: "Set CI/CD variables" - id: vars - run: | - echo "terraform_version=$(grep '^terraform' .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT - - - name: "Resolve the dev-* tag for this commit" - id: tag - run: | - git fetch --tags --force - SHA="${{ github.event.workflow_run.head_sha }}" - TAG=$(git tag --points-at "$SHA" | grep '^dev-' | sort -r | head -n1 || true) - if [ -z "$TAG" ]; then - echo "No dev-* tag found on $SHA" >&2 - exit 1 - fi - echo "name=$TAG" >> $GITHUB_OUTPUT - echo "Resolved tag: $TAG" - - deploy: - name: "Deploy to TEST (approval required)" - runs-on: ubuntu-latest - needs: [metadata] - environment: test - timeout-minutes: 10080 - permissions: - id-token: write - contents: read - steps: - - name: "Checkout same commit" - uses: actions/checkout@v6 - with: - ref: ${{ github.event.workflow_run.head_sha }} - - - name: "Setup Terraform" - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: ${{ needs.metadata.outputs.terraform_version }} - - - name: "Configure AWS Credentials" - uses: aws-actions/configure-aws-credentials@v6 - with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role - aws-region: eu-west-2 - - - name: "Download lambda artefact from dev workflow" - uses: actions/download-artifact@v7 - with: - name: lambda-${{ needs.metadata.outputs.tag }} - path: ./dist - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ github.token }} - - - name: "Terraform Apply (TEST)" - env: - ENVIRONMENT: test - WORKSPACE: "default" - TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }} - TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }} - TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }} - TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }} - TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }} - TF_VAR_OPERATOR_EMAILS: ${{ vars.SECRET_ROTATION_OPERATOR_EMAILS }} - TF_VAR_PROXYGEN_PRIVATE_KEY_PTL: ${{ secrets.PROXYGEN_PRIVATE_KEY_PTL }} - TF_VAR_PROXYGEN_PRIVATE_KEY_PROD: ${{ secrets.PROXYGEN_PRIVATE_KEY_PROD }} - - run: | - mkdir -p ./build - echo "Deploying tag: ${{ needs.metadata.outputs.tag }}" - echo "Running: make terraform env=$ENVIRONMENT workspace=$WORKSPACE stack=networking tf-command=apply" - make terraform env=$ENVIRONMENT stack=networking tf-command=apply workspace=$WORKSPACE - echo "Running: make terraform env=$ENVIRONMENT workspace=$WORKSPACE stack=api-layer tf-command=apply" - make terraform env=$ENVIRONMENT stack=api-layer tf-command=apply workspace=$WORKSPACE - working-directory: ./infrastructure - - - name: "Validate Feature Toggles" - env: - ENV: test - run: | - pip install boto3 - python scripts/feature_toggle/validate_toggles.py - - - name: "Extract S3 bucket name from Terraform output" - id: tf_output - run: | - BUCKET=$(terraform output -raw lambda_artifact_bucket) - echo "bucket_name=$BUCKET" >> $GITHUB_OUTPUT - working-directory: ./infrastructure/stacks/api-layer - - - name: "Upload lambda artifact to S3" - run: | - aws s3 cp ./dist/lambda.zip \ - s3://${{ steps.tf_output.outputs.bucket_name }}/artifacts/${{ needs.metadata.outputs.tag }}/lambda.zip \ - --region eu-west-2 - - regression-tests: - name: "Regression Tests" - needs: deploy - uses: ./.github/workflows/regression-tests.yml - with: - ENVIRONMENT: "test" - VERSION_NUMBER: "main" - secrets: inherit - diff --git a/.github/workflows/cicd-4-preprod-deploy.yaml b/.github/workflows/cicd-4-preprod-deploy.yaml deleted file mode 100644 index 9896be0b..00000000 --- a/.github/workflows/cicd-4-preprod-deploy.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: "4. CD | Deploy to PreProd" - -concurrency: - group: preprod-deploy - cancel-in-progress: false - -on: - workflow_run: - workflows: ["3. CD | Deploy to Test"] - types: [completed] - workflow_dispatch: - inputs: - ref: - description: "dev-* tag to deploy to PreProd" - required: true - release_type: - description: "rc|patch|minor|major" - required: true - default: "rc" - reason: - description: "Why are you doing a manual deployment?" - required: true - default: "To roll back to a previous commit" - -permissions: - contents: write - id-token: write - actions: read - -jobs: - metadata: - name: "Resolve ref + stale guard + release type" - runs-on: ubuntu-latest - outputs: - ref: ${{ steps.resolver.outputs.this_ref }} - this_sha: ${{ steps.resolver.outputs.this_sha }} - latest_sha: ${{ steps.resolver.outputs.latest_test_sha }} - release_type: ${{ steps.release_type.outputs.release_type }} - if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} - env: - TEST_WORKFLOW_ID: "190123511" # this will need updating if the workflow is recreated - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - steps: - - name: Checkout (full history & tags) - uses: actions/checkout@v6 - with: { fetch-depth: 0 } - - - name: Force HTTPS remote for act - if: env.ACT == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - set -euo pipefail - echo "::add-mask::${GITHUB_TOKEN}" - git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" - git ls-remote --tags origin >/dev/null - - - name: Debug event - if: env.ACT == 'true' - run: | - echo "GITHUB_EVENT_NAME=${GITHUB_EVENT_NAME}" - echo "Payload:" && cat "$GITHUB_EVENT_PATH" || true - - - name: Resolve THIS vs LATEST TEST + stale guard (auto only) - id: resolver - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - EVENT_NAME: ${{ github.event_name }} - WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - MANUAL_REF: ${{ github.event.inputs.ref }} - WORKFLOW_NAME: "3. CD | Deploy to Test" - BRANCH: "main" - LIMIT: "100" - run: python3 scripts/workflow/pre-release_resolver.py - - - name: Resolve release_type (labels → default rc) - id: release_type - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: "main" - AGGREGATE: "true" - THIS_SHA: ${{ steps.resolver.outputs.this_sha }} - LATEST_TEST_SHA: ${{ steps.resolver.outputs.latest_test_sha }} - MANUAL_RELEASE_TYPE: ${{ github.event.inputs.release_type }} - run: python3 scripts/workflow/release_type_resolver.py - - deploy: - name: "Call base-deploy.yml (PreProd)" - needs: [metadata] - uses: ./.github/workflows/base-deploy.yml - with: - environment: preprod - ref: ${{ needs.metadata.outputs.ref }} - release_type: ${{ needs.metadata.outputs.release_type }} - secrets: inherit - if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} diff --git a/infrastructure/stacks/api-layer/athena.tf b/infrastructure/stacks/api-layer/athena.tf new file mode 100644 index 00000000..1a924f5c --- /dev/null +++ b/infrastructure/stacks/api-layer/athena.tf @@ -0,0 +1,103 @@ +resource "aws_iam_openid_connect_provider" "tableau_idp" { + url = "https://your-idp-domain.com" + client_id_list = ["your-client-id"] + thumbprint_list = ["a01152157448772d219323f136284e963b53b843"] +} + +data "aws_iam_policy_document" "tableau_trust_policy" { + statement { + sid = "AllowAthenaJwtPlugin" + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.tableau_idp.arn] + } + + condition { + test = "StringEquals" + variable = "${replace(aws_iam_openid_connect_provider.tableau_idp.url, "https://", "")}:aud" + values = ["your-client-id"] + } + + condition { + test = "StringEquals" + variable = "sts:RoleSessionName" + values = ["AthenaJWT"] + } + } +} + +resource "aws_iam_role" "tableau_athena_role" { + name = "tableau-athena-federated-role" + assume_role_policy = data.aws_iam_policy_document.tableau_trust_policy.json +} + +resource "aws_iam_role_policy" "tableau_athena_policy" { + name = "TableauAthenaAccess" + role = aws_iam_role.tableau_athena_role.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + sid = "AthenaQueryActions" + Effect = "Allow" + Action = [ + "athena:GetQueryExecution", + "athena:GetQueryResults", + "athena:StartQueryExecution", + "athena:GetWorkGroup", + "athena:StopQueryExecution", + "athena:GetDataCatalog" + ] + Resource = [ + "arn:aws:athena:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:workgroup/primary" + ] + }, + { + sid = "GlueMetadataDiscovery" + Effect = "Allow" + Action = [ + "glue:GetDatabase", + "glue:GetTable", + "glue:GetTables", + "glue:GetDatabases" + ] + Resource = [ + "arn:aws:glue:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:catalog", + "arn:aws:glue:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:database/elid_dq", + "arn:aws:glue:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:table/elid_dq/cohort_metrics" + ] + }, + { + sid = "DataBucketAccess" + Effect = "Allow" + Action = [ + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket" + ] + Resource = [ + "arn:aws:s3:::${module.s3_dq_metrics_bucket.storage_bucket_name}", + "arn:aws:s3:::${module.s3_dq_metrics_bucket.storage_bucket_name}/*" + ] + }, + { + sid = "AthenaResultsStaging" + Effect = "Allow" + Action = [ + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject" + ] + Resource = [ + "arn:aws:s3:::${module.s3_athena_dq_query_bucket.storage_bucket_name}", + "arn:aws:s3:::${module.s3_athena_dq_query_bucket.storage_bucket_name}/*" + ] + } + ] + }) +} diff --git a/infrastructure/stacks/api-layer/s3_buckets.tf b/infrastructure/stacks/api-layer/s3_buckets.tf index c2d92454..3fec75b3 100644 --- a/infrastructure/stacks/api-layer/s3_buckets.tf +++ b/infrastructure/stacks/api-layer/s3_buckets.tf @@ -57,3 +57,12 @@ module "s3_dq_metrics_bucket" { stack_name = local.stack_name workspace = terraform.workspace } + +module "s3_athena_dq_query_bucket" { + source = "../../modules/s3" + bucket_name = "athena-stage" + environment = var.environment + project_name = var.project_name + stack_name = local.stack_name + workspace = terraform.workspace +} diff --git a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf index ac774d86..8086e612 100644 --- a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf +++ b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf @@ -235,6 +235,10 @@ resource "aws_iam_policy" "s3_management" { "arn:aws:s3:::*eligibility-signposting-api-${var.environment}-dq-metrics/*", "arn:aws:s3:::*eligibility-signposting-api-${var.environment}-dq-metrics-access-logs", "arn:aws:s3:::*eligibility-signposting-api-${var.environment}-dq-metrics-access-logs/*", + "arn:aws:s3:::*eligibility-signposting-api-${var.environment}-athena-stage", + "arn:aws:s3:::*eligibility-signposting-api-${var.environment}-athena-stage/*", + "arn:aws:s3:::*eligibility-signposting-api-${var.environment}-athena-stage-access-logs", + "arn:aws:s3:::*eligibility-signposting-api-${var.environment}-athena-stage-access-logs/*", ] } ] @@ -743,6 +747,65 @@ resource "aws_iam_policy" "cloudwatch_management" { tags = merge(local.tags, { Name = "cloudwatch-management" }) } +# Athena/Glue Infrastructure Management Policy for GitHub Actions +resource "aws_iam_policy" "athena_glue_management" { + name = "athena-glue-management" + description = "Allows GitHub Actions to create and manage Athena/Glue resources" + path = "/service-policies/" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + # 1. Permission to manage the Glue Metadata (The "Athena Database/Table") + Effect = "Allow", + Action = [ + "glue:CreateDatabase", + "glue:DeleteDatabase", + "glue:GetDatabase", + "glue:UpdateDatabase", + "glue:CreateTable", + "glue:DeleteTable", + "glue:UpdateTable", + "glue:GetTable", + "glue:GetTables", + "glue:BatchCreatePartition", + "glue:CreatePartition", + "glue:DeletePartition", + "glue:GetPartitions" + ], + Resource = [ + "arn:aws:glue:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:catalog", + "arn:aws:glue:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:database/elid_dq", + "arn:aws:glue:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:table/elid_dq/*" + ] + }, + { + # 2. Permission to manage Athena Workgroups or Named Queries + Effect = "Allow", + Action = [ + "athena:CreateWorkGroup", + "athena:DeleteWorkGroup", + "athena:UpdateWorkGroup", + "athena:GetWorkGroup", + "athena:CreateNamedQuery", + "athena:DeleteNamedQuery", + "athena:GetNamedQuery", + "athena:ListDataCatalogs", + "athena:CreateDataCatalog", + "athena:DeleteDataCatalog" + ], + Resource = [ + "arn:aws:athena:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:workgroup/*", + "arn:aws:athena:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:datacatalog/*" + ] + } + ] + }) + + tags = merge(local.tags, { Name = "athena-glue-management" }) +} + # Attach the policies to the role resource "aws_iam_role_policy_attachment" "terraform_state" { role = aws_iam_role.github_actions.name @@ -788,3 +851,8 @@ resource "aws_iam_role_policy_attachment" "cloudwatch_management" { role = aws_iam_role.github_actions.name policy_arn = aws_iam_policy.cloudwatch_management.arn } + +resource "aws_iam_role_policy_attachment" "athena_glue_management" { + role = aws_iam_role.github_actions.name + policy_arn = aws_iam_policy.athena_glue_management.arn +} diff --git a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf index 91c1e94d..7ae5df2c 100644 --- a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf +++ b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf @@ -221,6 +221,23 @@ data "aws_iam_policy_document" "permissions_boundary" { "states:CreateStateMachine", "states:TagResource", "states:UpdateStateMachine", + + # Athena + "athena:CreateWorkGroup", + "athena:UpdateWorkGroup", + "athena:GetQueryExecution", + "athena:GetQueryResults", + "athena:StartQueryExecution", + "athena:GetWorkGroup", + "athena:StopQueryExecution", + "athena:GetDataCatalog", + + # Glue + "glue:CreateDatabase", + "glue:GetDatabase", + "glue:GetTable", + "glue:GetTables", + "glue:GetDatabases" ] resources = ["*"]