diff --git a/admin/deploy/templates/admin.deployment.yaml b/admin/deploy/templates/admin.deployment.yaml index 2f171fa17a..9dbedd8896 100644 --- a/admin/deploy/templates/admin.deployment.yaml +++ b/admin/deploy/templates/admin.deployment.yaml @@ -101,7 +101,11 @@ spec: readOnly: true {{- if .Values.audit.connectSocket }} - name: mdsd-asa-run-vol + {{- if .Values.audit.defenderEnabled }} mountPath: /var/run/mdsd + {{- else }} + mountPath: /var/run/mdsd/asa + {{- end }} {{- end }} volumes: - name: fpa-cert @@ -113,6 +117,10 @@ spec: {{- if .Values.audit.connectSocket }} - name: mdsd-asa-run-vol hostPath: + {{- if .Values.audit.defenderEnabled }} path: /var/run/mdsd + {{- else }} + path: /var/run/mdsd/asa + {{- end }} type: Directory {{- end }} diff --git a/admin/testdata/helmtest_connect_socket.yaml b/admin/testdata/helmtest_connect_socket.yaml new file mode 100644 index 0000000000..2eb296b52e --- /dev/null +++ b/admin/testdata/helmtest_connect_socket.yaml @@ -0,0 +1,8 @@ +values: ../values.yaml +name: admin-api-connect-socket +namespace: aro-hcp-admin-api +testData: + adminApi: + audit: + connectSocket: true + defenderEnabled: false diff --git a/admin/testdata/helmtest_connect_socket_defender.yaml b/admin/testdata/helmtest_connect_socket_defender.yaml new file mode 100644 index 0000000000..cc5b37d22b --- /dev/null +++ b/admin/testdata/helmtest_connect_socket_defender.yaml @@ -0,0 +1,8 @@ +values: ../values.yaml +name: admin-api-connect-socket-defender +namespace: aro-hcp-admin-api +testData: + adminApi: + audit: + connectSocket: true + defenderEnabled: true diff --git a/admin/testdata/zz_fixture_TestHelmTemplate_admin_api_connect_socket.yaml b/admin/testdata/zz_fixture_TestHelmTemplate_admin_api_connect_socket.yaml new file mode 100644 index 0000000000..50a3f32a62 --- /dev/null +++ b/admin/testdata/zz_fixture_TestHelmTemplate_admin_api_connect_socket.yaml @@ -0,0 +1,312 @@ +--- +# Source: ARO HCP Admin API/templates/admin.serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + azure.workload.identity/client-id: '__adminApiMsiClientId__' + azure.workload.identity/tenant-id: '__tenantId__' + name: admin-api + namespace: 'aro-hcp-admin-api' +--- +# Source: ARO HCP Admin API/templates/admin.service.yaml +apiVersion: v1 +kind: Service +metadata: + name: admin-api + namespace: 'aro-hcp-admin-api' + labels: + app: admin-api +spec: + selector: + app: admin-api + ports: + - port: 8443 + targetPort: 8443 + protocol: TCP +--- +# Source: ARO HCP Admin API/templates/admin.deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-api + namespace: 'aro-hcp-admin-api' + labels: + app: admin-api +spec: + replicas: 2 + revisionHistoryLimit: 3 + selector: + matchLabels: + app: admin-api + strategy: + rollingUpdate: + maxSurge: 50% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + labels: + app: admin-api + azure.workload.identity/use: "true" + annotations: + checksum/fpa-spc: '3a8a42649d4184782a080188d109778f9fc6d3991745b5ce9858c9c0eb78e68a' + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: 'topology.kubernetes.io/zone' + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: admin-api + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: admin-api + serviceAccountName: admin-api + containers: + - name: service + image: "arohcpsvcdev.azurecr.io/arohcpadminapi@sha256:1234567890" + imagePullPolicy: IfNotPresent + args: + - "--location" + - "westus3" + env: + - name: CLUSTERS_SERVICE_URL + value: "http://clusters-service.clusters-service.svc.cluster.local:8000" + - name: COSMOS_URL + value: "__cosmosDBDocumentEndpoint__" + - name: COSMOS_NAME + value: "arohcpdev-rp-usw3" + - name: KUSTO_ENDPOINT + value: "__kustoEndpoint__" + - name: FPA_CERT_BUNDLE_PATH + value: "/secrets/fpa-cert/bundle" + - name: FPA_CLIENT_ID + value: "b3cb2fab-15cb-4583-ad06-f91da9bfe2d1" + - name: AUDIT_CONNECT_SOCKET + value: "true" + ports: + - containerPort: 8443 + name: http + protocol: TCP + - containerPort: 8444 + name: metrics + protocol: TCP + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1 + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + livenessProbe: + httpGet: + path: /healthz/live + port: 8444 + initialDelaySeconds: 15 + periodSeconds: 20 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz/ready + port: 8444 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: fpa-cert + mountPath: /secrets/fpa-cert + readOnly: true + - name: mdsd-asa-run-vol + mountPath: /var/run/mdsd/asa + volumes: + - name: fpa-cert + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: fpa-cert + - name: mdsd-asa-run-vol + hostPath: + path: /var/run/mdsd/asa + type: Directory +--- +# Source: ARO HCP Admin API/templates/admin.secret-refresher.yaml +################################ +# +# This keeps the certificate secret fresh because the secret is mounted from the keyVault (via the SecretProviderClass) and +# if the certificate changes in the keyvault this will trigger a refresh of the kubernetes secret. +# +# Note: the istio plugin doesn't support using the SecretProviderClass directly. When it does this can be removed. +# +################################ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-api-certificate-refresher + namespace: aks-istio-ingress +spec: + replicas: 1 + selector: + matchLabels: + app: admin-api-certificate-refresher + template: + metadata: + labels: + app: admin-api-certificate-refresher + spec: + containers: + - command: + - "/bin/sleep" + - "infinity" + image: mcr.microsoft.com/cbl-mariner/busybox:1.35 + name: init-container-msg-container-init + volumeMounts: + - name: secrets-store01-inline + mountPath: "/mnt/secrets-store" + readOnly: true + volumes: + - name: secrets-store01-inline + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "admin-api-scp" +--- +# Source: ARO HCP Admin API/templates/acrpullbinding.yaml +apiVersion: acrpull.microsoft.com/v1beta2 +kind: AcrPullBinding +metadata: + name: pull-binding + namespace: 'aro-hcp-admin-api' +spec: + acr: + environment: PublicCloud + server: 'arohcpsvcdev.azurecr.io' + scope: 'repository:arohcpadminapi:pull' + auth: + workloadIdentity: + serviceAccountRef: 'admin-api' + clientID: '__imagePullerMsiClientId__' + tenantID: '__tenantId__' + serviceAccountName: 'admin-api' +--- +# Source: ARO HCP Admin API/templates/admin.httproute.yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: admin-api + namespace: aro-hcp-admin-api +spec: + parentRefs: + - name: ops-ingress-gateway + namespace: aks-istio-ingress + sectionName: admin-api-https + hostnames: + - "admin.westus3.hcpsvc.osadev.cloud" + rules: + - matches: + - path: + type: PathPrefix + value: / + filters: + - type: RequestHeaderModifier + requestHeaderModifier: + add: + - name: mise-inbound-policies-to-filter + value: "Geneva Actions" + backendRefs: + - name: admin-api + port: 8443 +--- +# Source: ARO HCP Admin API/templates/admin.fpa.secretproviderclass.yaml +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: fpa-cert + namespace: 'aro-hcp-admin-api' +spec: + parameters: + clientID: '__adminApiMsiClientId__' + cloudName: 'AzurePublicCloud' + keyvaultName: 'aro-hcp-dev-svc-kv' + objects: |- + array: + - | + objectName: 'firstPartyCert2' + objectType: secret + objectAlias: bundle + tenantId: '__tenantId__' + usePodIdentity: "false" + provider: azure +--- +# Source: ARO HCP Admin API/templates/admin.secretproviderclass.yaml +################################ +# +# The addition of the secretObjects is to facilitate the istio plugin as it can't yet consume the SecretProviderClass directly. +# When it does this can be simplified and the secret.refresher removed. +# +################################ +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: admin-api-scp + namespace: aks-istio-ingress +spec: + parameters: + usePodIdentity: "false" + useVMManagedIdentity: "true" + userAssignedIdentityID: '__csiSecretStoreClientId__' + keyvaultName: 'aro-hcp-dev-svc-kv' + objects: |- + array: + - | + objectName: 'admin-api-cert-dev-usw3' + objectType: secret + objectAlias: admin-api-cert + tenantId: '__tenantId__' + provider: azure + secretObjects: + - secretName: admin-api-credential + type: kubernetes.io/tls + data: + - objectName: admin-api-cert + key: tls.crt + - objectName: admin-api-cert + key: tls.key +--- +# Source: ARO HCP Admin API/templates/admin.virtualservice.yaml +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: admin-api-vs + namespace: 'aro-hcp-admin-api' +spec: + hosts: + - "admin.westus3.hcpsvc.osadev.cloud" + gateways: + - aks-istio-ingress/aro-hcp-gateway-external + http: + - match: + - uri: + regex: '.+' + headers: + request: + add: + mise-inbound-policies-to-filter: "Geneva Actions" + route: + - destination: + host: admin-api + port: + number: 8443 + diff --git a/admin/testdata/zz_fixture_TestHelmTemplate_admin_api_connect_socket_defender.yaml b/admin/testdata/zz_fixture_TestHelmTemplate_admin_api_connect_socket_defender.yaml new file mode 100644 index 0000000000..f094b11068 --- /dev/null +++ b/admin/testdata/zz_fixture_TestHelmTemplate_admin_api_connect_socket_defender.yaml @@ -0,0 +1,312 @@ +--- +# Source: ARO HCP Admin API/templates/admin.serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + azure.workload.identity/client-id: '__adminApiMsiClientId__' + azure.workload.identity/tenant-id: '__tenantId__' + name: admin-api + namespace: 'aro-hcp-admin-api' +--- +# Source: ARO HCP Admin API/templates/admin.service.yaml +apiVersion: v1 +kind: Service +metadata: + name: admin-api + namespace: 'aro-hcp-admin-api' + labels: + app: admin-api +spec: + selector: + app: admin-api + ports: + - port: 8443 + targetPort: 8443 + protocol: TCP +--- +# Source: ARO HCP Admin API/templates/admin.deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-api + namespace: 'aro-hcp-admin-api' + labels: + app: admin-api +spec: + replicas: 2 + revisionHistoryLimit: 3 + selector: + matchLabels: + app: admin-api + strategy: + rollingUpdate: + maxSurge: 50% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + labels: + app: admin-api + azure.workload.identity/use: "true" + annotations: + checksum/fpa-spc: '3a8a42649d4184782a080188d109778f9fc6d3991745b5ce9858c9c0eb78e68a' + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: 'topology.kubernetes.io/zone' + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: admin-api + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: admin-api + serviceAccountName: admin-api + containers: + - name: service + image: "arohcpsvcdev.azurecr.io/arohcpadminapi@sha256:1234567890" + imagePullPolicy: IfNotPresent + args: + - "--location" + - "westus3" + env: + - name: CLUSTERS_SERVICE_URL + value: "http://clusters-service.clusters-service.svc.cluster.local:8000" + - name: COSMOS_URL + value: "__cosmosDBDocumentEndpoint__" + - name: COSMOS_NAME + value: "arohcpdev-rp-usw3" + - name: KUSTO_ENDPOINT + value: "__kustoEndpoint__" + - name: FPA_CERT_BUNDLE_PATH + value: "/secrets/fpa-cert/bundle" + - name: FPA_CLIENT_ID + value: "b3cb2fab-15cb-4583-ad06-f91da9bfe2d1" + - name: AUDIT_CONNECT_SOCKET + value: "true" + ports: + - containerPort: 8443 + name: http + protocol: TCP + - containerPort: 8444 + name: metrics + protocol: TCP + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1 + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + livenessProbe: + httpGet: + path: /healthz/live + port: 8444 + initialDelaySeconds: 15 + periodSeconds: 20 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz/ready + port: 8444 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: fpa-cert + mountPath: /secrets/fpa-cert + readOnly: true + - name: mdsd-asa-run-vol + mountPath: /var/run/mdsd + volumes: + - name: fpa-cert + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: fpa-cert + - name: mdsd-asa-run-vol + hostPath: + path: /var/run/mdsd + type: Directory +--- +# Source: ARO HCP Admin API/templates/admin.secret-refresher.yaml +################################ +# +# This keeps the certificate secret fresh because the secret is mounted from the keyVault (via the SecretProviderClass) and +# if the certificate changes in the keyvault this will trigger a refresh of the kubernetes secret. +# +# Note: the istio plugin doesn't support using the SecretProviderClass directly. When it does this can be removed. +# +################################ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admin-api-certificate-refresher + namespace: aks-istio-ingress +spec: + replicas: 1 + selector: + matchLabels: + app: admin-api-certificate-refresher + template: + metadata: + labels: + app: admin-api-certificate-refresher + spec: + containers: + - command: + - "/bin/sleep" + - "infinity" + image: mcr.microsoft.com/cbl-mariner/busybox:1.35 + name: init-container-msg-container-init + volumeMounts: + - name: secrets-store01-inline + mountPath: "/mnt/secrets-store" + readOnly: true + volumes: + - name: secrets-store01-inline + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "admin-api-scp" +--- +# Source: ARO HCP Admin API/templates/acrpullbinding.yaml +apiVersion: acrpull.microsoft.com/v1beta2 +kind: AcrPullBinding +metadata: + name: pull-binding + namespace: 'aro-hcp-admin-api' +spec: + acr: + environment: PublicCloud + server: 'arohcpsvcdev.azurecr.io' + scope: 'repository:arohcpadminapi:pull' + auth: + workloadIdentity: + serviceAccountRef: 'admin-api' + clientID: '__imagePullerMsiClientId__' + tenantID: '__tenantId__' + serviceAccountName: 'admin-api' +--- +# Source: ARO HCP Admin API/templates/admin.httproute.yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: admin-api + namespace: aro-hcp-admin-api +spec: + parentRefs: + - name: ops-ingress-gateway + namespace: aks-istio-ingress + sectionName: admin-api-https + hostnames: + - "admin.westus3.hcpsvc.osadev.cloud" + rules: + - matches: + - path: + type: PathPrefix + value: / + filters: + - type: RequestHeaderModifier + requestHeaderModifier: + add: + - name: mise-inbound-policies-to-filter + value: "Geneva Actions" + backendRefs: + - name: admin-api + port: 8443 +--- +# Source: ARO HCP Admin API/templates/admin.fpa.secretproviderclass.yaml +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: fpa-cert + namespace: 'aro-hcp-admin-api' +spec: + parameters: + clientID: '__adminApiMsiClientId__' + cloudName: 'AzurePublicCloud' + keyvaultName: 'aro-hcp-dev-svc-kv' + objects: |- + array: + - | + objectName: 'firstPartyCert2' + objectType: secret + objectAlias: bundle + tenantId: '__tenantId__' + usePodIdentity: "false" + provider: azure +--- +# Source: ARO HCP Admin API/templates/admin.secretproviderclass.yaml +################################ +# +# The addition of the secretObjects is to facilitate the istio plugin as it can't yet consume the SecretProviderClass directly. +# When it does this can be simplified and the secret.refresher removed. +# +################################ +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: admin-api-scp + namespace: aks-istio-ingress +spec: + parameters: + usePodIdentity: "false" + useVMManagedIdentity: "true" + userAssignedIdentityID: '__csiSecretStoreClientId__' + keyvaultName: 'aro-hcp-dev-svc-kv' + objects: |- + array: + - | + objectName: 'admin-api-cert-dev-usw3' + objectType: secret + objectAlias: admin-api-cert + tenantId: '__tenantId__' + provider: azure + secretObjects: + - secretName: admin-api-credential + type: kubernetes.io/tls + data: + - objectName: admin-api-cert + key: tls.crt + - objectName: admin-api-cert + key: tls.key +--- +# Source: ARO HCP Admin API/templates/admin.virtualservice.yaml +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: admin-api-vs + namespace: 'aro-hcp-admin-api' +spec: + hosts: + - "admin.westus3.hcpsvc.osadev.cloud" + gateways: + - aks-istio-ingress/aro-hcp-gateway-external + http: + - match: + - uri: + regex: '.+' + headers: + request: + add: + mise-inbound-policies-to-filter: "Geneva Actions" + route: + - destination: + host: admin-api + port: + number: 8443 + diff --git a/admin/values.yaml b/admin/values.yaml index e9f9cd13dd..2997954fe6 100644 --- a/admin/values.yaml +++ b/admin/values.yaml @@ -1,6 +1,7 @@ location: "{{ .region }}" audit: connectSocket: {{ .adminApi.audit.connectSocket }} + defenderEnabled: {{ .defenderEnabled }} deployment: replicas: {{ .adminApi.k8s.replicas }} requests: diff --git a/config/config.msft.clouds-overlay.yaml b/config/config.msft.clouds-overlay.yaml index 75a3891820..a5fc6960c1 100644 --- a/config/config.msft.clouds-overlay.yaml +++ b/config/config.msft.clouds-overlay.yaml @@ -4,6 +4,7 @@ clouds: int: # this is the MSFT INT environment defaults: + defenderEnabled: true # E2E e2e: regionTest: diff --git a/config/config.schema.json b/config/config.schema.json index 29f6b27079..ad4e49cb0d 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -2997,6 +2997,10 @@ "automationDryRun": { "type": "boolean", "description": "Whether automation account runs in dry-run mode" + }, + "defenderEnabled": { + "type": "boolean", + "description": "Whether Microsoft Defender for Cloud is active in the environment" } }, "additionalProperties": false, @@ -3031,6 +3035,7 @@ "msiCredentialsRefresher", "keyVaultDNSSuffix", "acrDNSSuffix", - "e2e" + "e2e", + "defenderEnabled" ] } \ No newline at end of file diff --git a/config/config.yaml b/config/config.yaml index 81723db562..5fa6491190 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -838,6 +838,8 @@ defaults: operatorVersionsToMirror: "4.18,4.19,4.20" # Automation Account automationDryRun: true + # Microsoft Defender for Cloud + defenderEnabled: false # Mock Managed Identities - not relevant for most MSFT envs miMockClientId: "" miMockPrincipalId: "" diff --git a/config/rendered/dev/cspr/westus3.yaml b/config/rendered/dev/cspr/westus3.yaml index ebbba5832a..d3d37e486d 100755 --- a/config/rendered/dev/cspr/westus3.yaml +++ b/config/rendered/dev/cspr/westus3.yaml @@ -186,6 +186,7 @@ cxKeyVault: softDelete: false tagKey: aroHCPPurpose tagValue: cx +defenderEnabled: false dns: baseDnsZoneRG: global cxParentZoneName: hcp.osadev.cloud diff --git a/config/rendered/dev/dev/westus3.yaml b/config/rendered/dev/dev/westus3.yaml index ad15f3bdbb..b55a1b3c02 100755 --- a/config/rendered/dev/dev/westus3.yaml +++ b/config/rendered/dev/dev/westus3.yaml @@ -186,6 +186,7 @@ cxKeyVault: softDelete: false tagKey: aroHCPPurpose tagValue: cx +defenderEnabled: false dns: baseDnsZoneRG: global cxParentZoneName: hcp.osadev.cloud diff --git a/config/rendered/dev/perf/westus3.yaml b/config/rendered/dev/perf/westus3.yaml index 068578be80..29adba9c08 100755 --- a/config/rendered/dev/perf/westus3.yaml +++ b/config/rendered/dev/perf/westus3.yaml @@ -186,6 +186,7 @@ cxKeyVault: softDelete: false tagKey: aroHCPPurpose tagValue: cx +defenderEnabled: false dns: baseDnsZoneRG: global cxParentZoneName: hcp.osadev.cloud diff --git a/config/rendered/dev/pers/westus3.yaml b/config/rendered/dev/pers/westus3.yaml index dea1f353cb..bff6edd547 100755 --- a/config/rendered/dev/pers/westus3.yaml +++ b/config/rendered/dev/pers/westus3.yaml @@ -186,6 +186,7 @@ cxKeyVault: softDelete: false tagKey: aroHCPPurpose tagValue: cx +defenderEnabled: false dns: baseDnsZoneRG: global cxParentZoneName: hcp.osadev.cloud diff --git a/config/rendered/dev/prow/westus3.yaml b/config/rendered/dev/prow/westus3.yaml index f8f87ace82..eff17b0936 100755 --- a/config/rendered/dev/prow/westus3.yaml +++ b/config/rendered/dev/prow/westus3.yaml @@ -186,6 +186,7 @@ cxKeyVault: softDelete: false tagKey: aroHCPPurpose tagValue: cx +defenderEnabled: false dns: baseDnsZoneRG: global cxParentZoneName: hcp.osadev.cloud diff --git a/config/rendered/dev/swft/uksouth.yaml b/config/rendered/dev/swft/uksouth.yaml index 19d66f78b3..9898522736 100755 --- a/config/rendered/dev/swft/uksouth.yaml +++ b/config/rendered/dev/swft/uksouth.yaml @@ -186,6 +186,7 @@ cxKeyVault: softDelete: false tagKey: aroHCPPurpose tagValue: cx +defenderEnabled: false dns: baseDnsZoneRG: global cxParentZoneName: hcp.osadev.cloud diff --git a/docs/exec-plans/2026-02/config-defenderEnabled.md b/docs/exec-plans/2026-02/config-defenderEnabled.md new file mode 100644 index 0000000000..06c97e7172 --- /dev/null +++ b/docs/exec-plans/2026-02/config-defenderEnabled.md @@ -0,0 +1,259 @@ +# Plan: Add `defenderEnabled` Config Flag for Conditional MDSD Mount Path + +## Context + +PR 4164 changed the frontend MDSD audit socket mount path from `/var/run/mdsd/asa` to `/var/run/mdsd` +(parent directory) to prevent breakage when Microsoft Defender for Cloud crashes and changes the inode +of the `asa` subdirectory. PR 4160 introduced the same socket mounting pattern for the Admin API. + +This path change is only safe/needed when Defender for Cloud is active in the environment (MSFT-managed +environments). In environments without Defender (dev, cspr, etc.), the narrower `/var/run/mdsd/asa` mount +is preferable. The fix: a new `defenderEnabled` config flag that drives path selection. + +**Scope**: only two places mount this socket: +- `frontend/deploy/templates/frontend.deployment.yaml` +- `admin/deploy/templates/admin.deployment.yaml` + +(The `observability/arobit` daemonset mounts `/var/run/mdsd/` for a different purpose and is unaffected.) + +--- + +## Changes + +### 1. `config/config.schema.json` + +Add `defenderEnabled` as a top-level boolean property in the `defaults` object definition (same level as +`automationDryRun`): + +```json +"defenderEnabled": { + "type": "boolean", + "description": "Whether Microsoft Defender for Cloud is active in the environment" +} +``` + +Also add it to the `required` array for `defaults`. + +### 2. `config/config.yaml` + +Add in the top-level `defaults` section near the Automation Account block: + +```yaml + # Microsoft Defender for Cloud + defenderEnabled: false +``` + +### 3. `config/config.msft.clouds-overlay.yaml` + +Add `defenderEnabled: true` **only** under the `int` environment. Leave existing `frontend.audit.connectSocket` +entries untouched; do NOT add `adminApi.audit.connectSocket` (that remains false everywhere for now): + +```yaml +clouds: + public: + environments: + int: + defaults: + defenderEnabled: true + frontend: + audit: + connectSocket: true # already present, no change +``` + +`stg` and `prod` keep `defenderEnabled: false` (not added there). + +### 4. `frontend/values.yaml` + +Add to the `audit` section: + +```yaml +audit: + connectSocket: {{ .frontend.audit.connectSocket }} + defenderEnabled: {{ .defenderEnabled }} +``` + +### 5. `admin/values.yaml` + +Add to the `audit` section: + +```yaml +audit: + connectSocket: {{ .adminApi.audit.connectSocket }} + defenderEnabled: {{ .defenderEnabled }} +``` + +### 6. `frontend/deploy/templates/frontend.deployment.yaml` + +Replace both the `volumeMounts` and `volumes` path with a conditional: + +```yaml +{{- if .Values.audit.connectSocket }} +volumeMounts: + - name: mdsd-asa-run-vol + {{- if .Values.audit.defenderEnabled }} + mountPath: /var/run/mdsd + {{- else }} + mountPath: /var/run/mdsd/asa + {{- end }} +{{- end }} +... +{{- if .Values.audit.connectSocket }} +volumes: + - name: mdsd-asa-run-vol + hostPath: + {{- if .Values.audit.defenderEnabled }} + path: /var/run/mdsd + {{- else }} + path: /var/run/mdsd/asa + {{- end }} + type: Directory +{{- end }} +``` + +Note: placing `defenderEnabled` under `audit` in values (`.Values.audit.defenderEnabled`) keeps it +logically grouped. + +### 7. `admin/deploy/templates/admin.deployment.yaml` + +Same conditional pattern applied to the `volumeMounts` and `volumes` sections (identical logic as frontend). + +### 8. Test fixtures — frontend + +**`frontend/testdata/helmtest_connect_socket.yaml`** — add `defenderEnabled: false` override to make +the test explicit (tests use the rendered `dev/dev/westus3.yaml` config which will have `false`, but +being explicit is better): + +```yaml +testData: + frontend: + audit: + connectSocket: true + defenderEnabled: false +``` + +Add new test file: +- `frontend/testdata/helmtest_connect_socket_defender.yaml` with `connectSocket: true` + `defenderEnabled: true` + +The corresponding `zz_fixture_*` files are regenerated automatically by `make materialize` (see Execution below). + +### 9. Test fixtures — admin + +Add new test files: +- `admin/testdata/helmtest_connect_socket.yaml` with `connectSocket: true` + `defenderEnabled: false` +- `admin/testdata/helmtest_connect_socket_defender.yaml` with `connectSocket: true` + `defenderEnabled: true` + +The corresponding `zz_fixture_*` files are regenerated automatically by `make materialize`. + +--- + +## Execution + +Make all code and config changes first, then run a single command that validates the schema, renders +all configs, and regenerates all helm test fixtures in one step: + +```bash +cd config && make materialize +``` + +`make materialize` internally calls `make update-helm-fixtures` which runs +`UPDATE=true go test --count=1 ./...` in `tooling/helmtest`. **Do not run helmtest with `UPDATE=true` +separately** — it will fail for tests that rely on the base rendered config (e.g. `frontend-mise-enabled`, +`dev-westus3-svc-1-admin-api`) if `make materialize` hasn't been run first to populate `defenderEnabled` +in the rendered YAML. + +To confirm fixtures are stable after generation: + +```bash +cd tooling/helmtest && go test -run TestHelmTemplate -count=1 ./... +``` + +--- + +## Critical Files + +| File | Change | +|------|--------| +| `config/config.schema.json` | Add `defenderEnabled: boolean` to defaults, add to `required` | +| `config/config.yaml` | Add `defenderEnabled: false` to defaults | +| `config/config.msft.clouds-overlay.yaml` | Add `defenderEnabled: true` for `int` only | +| `frontend/values.yaml` | Add `audit.defenderEnabled` | +| `admin/values.yaml` | Add `audit.defenderEnabled` | +| `frontend/deploy/templates/frontend.deployment.yaml` | Conditional path | +| `admin/deploy/templates/admin.deployment.yaml` | Conditional path | +| `frontend/testdata/helmtest_connect_socket.yaml` | Add `defenderEnabled: false` | +| `frontend/testdata/helmtest_connect_socket_defender.yaml` | New test case | +| `admin/testdata/helmtest_connect_socket.yaml` | New test case | +| `admin/testdata/helmtest_connect_socket_defender.yaml` | New test case | +| `config/rendered/**` | Re-rendered by `make materialize` | +| `frontend/testdata/zz_fixture_*` | Regenerated by `make materialize` | +| `admin/testdata/zz_fixture_*` | Regenerated by `make materialize` | + +--- + +## Verification + +1. Run `make materialize` — this validates the schema and renders configs. Check exit code is 0. + +2. Run helm tests to confirm all fixtures are stable: + ```bash + cd tooling/helmtest && go test -run TestHelmTemplate -count=1 ./... + ``` + +3. Spot-check rendered `config/rendered/dev/dev/westus3.yaml` has `defenderEnabled: false`. + +4. Verify generated fixtures produce correct paths: + - `frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket.yaml`: + `mountPath: /var/run/mdsd/asa` (non-defender default) + - `frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket_defender.yaml`: + `mountPath: /var/run/mdsd` (defender path) + - Same pattern in the two admin fixtures. + +5. Verify MSFT environment values using `config/Makefile`'s `render-partial-config` target, which + applies `config.msft.clouds-overlay.yaml` and works without MSFT credentials (uses + `--skip-schema-validation` and `--ev2-cloud public` with placeholder ev2 values): + + ```bash + for env in int stg prod; do + make -C config render-partial-config \ + ARO_HCP_CLOUD=public \ + ARO_HCP_DEPLOY_ENV=$env \ + LOCATION=eastus \ + CONFIG_OUTPUT=/tmp/config-${env}.yaml + echo -n "$env: " + python3 -c " + import yaml + with open('/tmp/config-${env}.yaml') as f: + d = yaml.safe_load(f) + print('defenderEnabled={} connectSocket={}'.format(d['defenderEnabled'], d['frontend']['audit']['connectSocket'])) + " + done + ``` + + Expected output: + ``` + int: defenderEnabled=True connectSocket=True + stg: defenderEnabled=False connectSocket=True + prod: defenderEnabled=False connectSocket=True + ``` + + `stg` and `prod` have `connectSocket=true` but `defenderEnabled=false`, so they mount + `/var/run/mdsd/asa`. They can be switched to `defenderEnabled=true` in a follow-up once `int` + is confirmed stable. + +--- + +## Notes on Original Plan vs Actual Implementation + +The following corrections were discovered during execution: + +- **Test update flag**: The original plan referenced `go test ./... -update`. The actual mechanism is + an environment variable: `UPDATE=true go test ./...`. There is no `-update` CLI flag. +- **`make materialize` handles fixture regeneration**: `make materialize` already calls + `make update-helm-fixtures` → `UPDATE=true go test`. Running helmtest with UPDATE separately is + redundant and error-prone (it fails if done before `make materialize` because the base rendered + config lacks `defenderEnabled`). +- **`public/int` rendered configs not available locally**: `config/rendered/` only contains `dev` + configs. Use `make -C config render-partial-config` with `ARO_HCP_CLOUD=public` and + `ARO_HCP_DEPLOY_ENV=int/stg/prod` to produce partial renders of MSFT environment configs locally + without needing MSFT credentials. This is also the same mechanism used by the prow + `aro-hcp-write-config` step in CI. diff --git a/frontend/deploy/templates/frontend.deployment.yaml b/frontend/deploy/templates/frontend.deployment.yaml index 106c7a8258..39b67936b6 100644 --- a/frontend/deploy/templates/frontend.deployment.yaml +++ b/frontend/deploy/templates/frontend.deployment.yaml @@ -86,7 +86,11 @@ spec: {{- if .Values.audit.connectSocket }} volumeMounts: - name: mdsd-asa-run-vol + {{- if .Values.audit.defenderEnabled }} mountPath: /var/run/mdsd + {{- else }} + mountPath: /var/run/mdsd/asa + {{- end }} {{- end }} livenessProbe: httpGet: @@ -104,8 +108,12 @@ spec: {{- if .Values.audit.connectSocket }} volumes: - name: mdsd-asa-run-vol - hostPath: + hostPath: + {{- if .Values.audit.defenderEnabled }} path: /var/run/mdsd + {{- else }} + path: /var/run/mdsd/asa + {{- end }} type: Directory {{- end }} restartPolicy: Always diff --git a/frontend/testdata/helmtest_connect_socket.yaml b/frontend/testdata/helmtest_connect_socket.yaml index a8a9b54c6a..a9b4a62afa 100644 --- a/frontend/testdata/helmtest_connect_socket.yaml +++ b/frontend/testdata/helmtest_connect_socket.yaml @@ -5,3 +5,4 @@ testData: frontend: audit: connectSocket: true + defenderEnabled: false diff --git a/frontend/testdata/helmtest_connect_socket_defender.yaml b/frontend/testdata/helmtest_connect_socket_defender.yaml new file mode 100644 index 0000000000..01a1c48049 --- /dev/null +++ b/frontend/testdata/helmtest_connect_socket_defender.yaml @@ -0,0 +1,8 @@ +values: ../values.yaml +name: frontend-connect-socket-defender +namespace: aro-hcp +testData: + frontend: + audit: + connectSocket: true + defenderEnabled: true diff --git a/frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket.yaml b/frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket.yaml index b5f0be7ead..78bc869e28 100644 --- a/frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket.yaml +++ b/frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket.yaml @@ -156,7 +156,7 @@ spec: type: RuntimeDefault volumeMounts: - name: mdsd-asa-run-vol - mountPath: /var/run/mdsd + mountPath: /var/run/mdsd/asa livenessProbe: httpGet: path: /healthz @@ -172,8 +172,8 @@ spec: periodSeconds: 10 volumes: - name: mdsd-asa-run-vol - hostPath: - path: /var/run/mdsd + hostPath: + path: /var/run/mdsd/asa type: Directory restartPolicy: Always terminationGracePeriodSeconds: 30 diff --git a/frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket_defender.yaml b/frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket_defender.yaml new file mode 100644 index 0000000000..95f46f904c --- /dev/null +++ b/frontend/testdata/zz_fixture_TestHelmTemplate_frontend_connect_socket_defender.yaml @@ -0,0 +1,431 @@ +--- +# Source: frontend/templates/frontend.poddisruptionbudget.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: aro-hcp-frontend + namespace: 'aro-hcp' +spec: + minAvailable: 1 + selector: + matchLabels: + app: aro-hcp-frontend +--- +# Source: frontend/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + azure.workload.identity/client-id: '__frontendMsiClientId__' + azure.workload.identity/tenant-id: '__frontendMsiTenantId__' + name: frontend + namespace: 'aro-hcp' +--- +# Source: frontend/templates/frontend.configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: frontend-config + namespace: 'aro-hcp' +data: + DB_NAME: 'arohcpdev-rp-usw3' + DB_URL: '__cosmosDBDocumentEndpoint__' + FRONTEND_MI_CLIENT_ID: '__frontendMsiClientId__' + LOCATION: 'westus3' +--- +# Source: frontend/templates/frontend.service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app: aro-hcp-frontend + name: aro-hcp-frontend + namespace: 'aro-hcp' +spec: + ports: + - port: 8443 + protocol: TCP + targetPort: 8443 + selector: + app: aro-hcp-frontend + type: ClusterIP +--- +# Source: frontend/templates/metrics.service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app: aro-hcp-frontend + port: metrics + name: aro-hcp-frontend-metrics + namespace: 'aro-hcp' +spec: + ports: + - port: 8081 + protocol: TCP + targetPort: 8081 + name: metrics + selector: + app: aro-hcp-frontend +--- +# Source: frontend/templates/frontend.deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: aro-hcp-frontend + name: aro-hcp-frontend + namespace: 'aro-hcp' +spec: + progressDeadlineSeconds: 600 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: aro-hcp-frontend + strategy: + rollingUpdate: + maxSurge: 50% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + labels: + app: aro-hcp-frontend + azure.workload.identity/use: "true" + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: 'topology.kubernetes.io/zone' + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: aro-hcp-frontend + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: aro-hcp-frontend + serviceAccountName: 'frontend' + containers: + - name: aro-hcp-frontend + image: 'arohcpsvcdev.azurecr.io/arohcpfrontend@sha256:1234567890' + imagePullPolicy: Always + args: ["--clusters-service-url", "http://clusters-service.clusters-service.svc.cluster.local:8000"] + env: + - name: DB_NAME + valueFrom: + configMapKeyRef: + name: frontend-config + key: DB_NAME + - name: DB_URL + valueFrom: + configMapKeyRef: + name: frontend-config + key: DB_URL + - name: LOCATION + valueFrom: + configMapKeyRef: + name: frontend-config + key: LOCATION + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "" + - name: OTEL_TRACES_EXPORTER + value: "" + - name: AUDIT_CONNECT_SOCKET + value: "true" + ports: + - containerPort: 8443 + protocol: TCP + - containerPort: 8081 + protocol: TCP + resources: + limits: + memory: 1Gi + requests: + cpu: 100m + memory: 500Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: mdsd-asa-run-vol + mountPath: /var/run/mdsd + livenessProbe: + httpGet: + path: /healthz + port: 8443 + initialDelaySeconds: 15 + periodSeconds: 20 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /healthz + port: 8443 + initialDelaySeconds: 5 + periodSeconds: 10 + volumes: + - name: mdsd-asa-run-vol + hostPath: + path: /var/run/mdsd + type: Directory + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +# Source: frontend/templates/frontend.secret-refresher.yaml +################################ +# +# This keeps the certificate secret fresh because the secret is mounted from the keyVault (via the SecretProviderClass) and +# it's if the certificate changes in the keyvault this will trigger the refreshing of the kubernetes secret. +# +# Note: the istio plugin doesn't support using the SecretProviderClass directly. When it does this can be removed. +# +################################ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend-certificate-refresher + namespace: aks-istio-ingress +spec: + replicas: 1 + selector: + matchLabels: + app: frontend-certificate-refresher + template: + metadata: + labels: + app: frontend-certificate-refresher + spec: + containers: + - command: + - "/bin/sleep" + - "infinity" + image: mcr.microsoft.com/cbl-mariner/busybox:1.35 + name: init-container-msg-container-init + volumeMounts: + - name: secrets-store01-inline + mountPath: "/mnt/secrets-store" + readOnly: true + volumes: + - name: secrets-store01-inline + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "frontend-scp" +--- +# Source: frontend/templates/acrpullbinding.yaml +apiVersion: acrpull.microsoft.com/v1beta2 +kind: AcrPullBinding +metadata: + name: frontend-pull-binding + namespace: 'aro-hcp' +spec: + acr: + environment: PublicCloud + server: 'arohcpsvcdev.azurecr.io' + scope: 'repository:arohcpfrontend:pull' + auth: + workloadIdentity: + serviceAccountRef: 'frontend' + clientID: '__imagePullerMsiClientId__' + tenantID: '__imagePullerMsiTenantId__' + serviceAccountName: 'frontend' +--- +# Source: frontend/templates/allow-admin.authorizationpolicy.yaml +apiVersion: security.istio.io/v1 +kind: AuthorizationPolicy +metadata: + name: allow-admin-api-to-frontend + namespace: 'aro-hcp' +spec: + action: "ALLOW" + rules: + - from: + - source: + principals: + - "cluster.local/ns/aro-hcp-admin-api/sa/admin-api" + to: + - operation: + ports: + - "8443" + selector: + matchLabels: + app: "aro-hcp-frontend" +--- +# Source: frontend/templates/allow-ingress.authorizationpolicy.yaml +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-istio-ingress + namespace: 'aro-hcp' +spec: + action: ALLOW + rules: + - from: + - source: + namespaces: ["aks-istio-ingress"] + to: + - operation: + methods: ["GET", "PUT", "POST", "PATCH", "DELETE"] + ports: + - "8443" +--- +# Source: frontend/templates/allow-metrics.authorizationpolicy.yaml +apiVersion: security.istio.io/v1 +kind: AuthorizationPolicy +metadata: + name: allow-metrics-frontend + namespace: 'aro-hcp' +spec: + action: "ALLOW" + rules: + - to: + - operation: + paths: ["/metrics"] + methods: ["GET"] + ports: ["8081"] + selector: + matchLabels: + app: "aro-hcp-frontend" +--- +# Source: frontend/templates/authorizationpolicy.yaml +apiVersion: security.istio.io/v1 +kind: AuthorizationPolicy +metadata: + name: allow-nothing + namespace: 'aro-hcp' +spec: {} +--- +# Source: frontend/templates/frontend.gateway.yaml +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: aro-hcp-gateway-external + namespace: aks-istio-ingress + annotations: + helm.sh/resource-policy: keep +spec: + selector: + istio: aks-istio-ingressgateway-external + servers: + - port: + number: 443 + name: frontend-https + protocol: HTTPS + tls: + mode: SIMPLE + credentialName: frontend-credential + hosts: + - "rp.westus3.hcpsvc.osadev.cloud" + # TODO: ops-ingress phase 4: cleanup, remove routing for admin API + - port: + number: 443 + name: admin-api-https + protocol: HTTPS + tls: + mode: SIMPLE + credentialName: admin-api-credential + hosts: + - "admin.westus3.hcpsvc.osadev.cloud" +--- +# Source: frontend/templates/peerauthentication.yaml +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: aro-hcp-frontend-metrics + namespace: 'aro-hcp' +spec: + selector: + matchLabels: + app: aro-hcp-frontend + portLevelMtls: + 8081: + mode: PERMISSIVE +--- +# Source: frontend/templates/frontend.secretproviderclass.yaml +################################ +# +# The addition of the secretObjects is to facilitate the istio plugin as it can't yet consume the SecretProviderClass directly. +# When it does this can be simplified and the secret.refresher removed. +# +################################ +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: frontend-scp + namespace: aks-istio-ingress +spec: + parameters: + usePodIdentity: "false" + useVMManagedIdentity: "true" + userAssignedIdentityID: '__csiSecretStoreClientId__' + keyvaultName: 'aro-hcp-dev-svc-kv' + objects: |- + array: + - | + objectName: 'frontend-cert-dev-usw3' + objectType: secret + objectAlias: frontend-cert + tenantId: '__tenantId__' + provider: azure + secretObjects: + - secretName: frontend-credential + type: kubernetes.io/tls + data: + - objectName: frontend-cert + key: tls.crt + - objectName: frontend-cert + key: tls.key +--- +# Source: frontend/templates/servicemonitor.yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: aro-hcp-frontend + namespace: 'aro-hcp' +spec: + endpoints: + - interval: 30s + path: /metrics + port: metrics + scheme: http + namespaceSelector: + matchNames: + - 'aro-hcp' + selector: + matchLabels: + app: aro-hcp-frontend + port: metrics +--- +# Source: frontend/templates/frontend.virtualservice.yaml +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: aro-hcp-vs-frontend + namespace: 'aro-hcp' +spec: + hosts: + - "rp.westus3.hcpsvc.osadev.cloud" + gateways: + - aks-istio-ingress/aro-hcp-gateway-external + http: + - match: + - uri: + regex: '.+' + headers: + request: + add: + mise-inbound-policies-to-filter: "ARM Policy" + route: + - destination: + host: aro-hcp-frontend + port: + number: 8443 + diff --git a/frontend/values.yaml b/frontend/values.yaml index 2cb52aba7f..fd0e20d051 100644 --- a/frontend/values.yaml +++ b/frontend/values.yaml @@ -33,6 +33,7 @@ tracing: exporter: "{{ .frontend.tracing.exporter }}" audit: connectSocket: {{ .frontend.audit.connectSocket }} + defenderEnabled: {{ .defenderEnabled }} adminApi: namespace: "{{ .adminApi.k8s.namespace }}" serviceAccount: "{{ .adminApi.k8s.serviceAccountName }}"