From 0a23d00c01e5569a9f2840484c4750e9346bd2c1 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 31 Oct 2025 14:05:37 -0500 Subject: [PATCH 1/5] Initial pass, eliminate secrets and configmaps that are linked to `local_settings.py` --- helm/common/Chart.yaml | 2 +- helm/common/README.md | 2 +- helm/common/templates/_generate_test_data.tpl | 21 ----------------- helm/fence/Chart.yaml | 2 +- helm/fence/README.md | 6 ++--- helm/fence/values.yaml | 23 ------------------- helm/gen3/Chart.yaml | 6 ++--- helm/gen3/README.md | 6 ++--- wip/gen3-test-data-job/templates/_jobs.tpl | 21 ----------------- wip/gen3-test-data-job/templates/jobs.yaml | 22 +----------------- 10 files changed, 13 insertions(+), 98 deletions(-) diff --git a/helm/common/Chart.yaml b/helm/common/Chart.yaml index a220155a4..8be4f91e6 100644 --- a/helm/common/Chart.yaml +++ b/helm/common/Chart.yaml @@ -15,7 +15,7 @@ type: library # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.24 +version: 0.1.25 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/common/README.md b/helm/common/README.md index 1bf51d26a..d3fd45b93 100644 --- a/helm/common/README.md +++ b/helm/common/README.md @@ -1,6 +1,6 @@ # common -![Version: 0.1.24](https://img.shields.io/badge/Version-0.1.24-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.25](https://img.shields.io/badge/Version-0.1.25-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for provisioning databases in gen3 diff --git a/helm/common/templates/_generate_test_data.tpl b/helm/common/templates/_generate_test_data.tpl index 2a2760efe..e7545a1a7 100644 --- a/helm/common/templates/_generate_test_data.tpl +++ b/helm/common/templates/_generate_test_data.tpl @@ -25,18 +25,9 @@ spec: # DEPRECATED! Remove when all commons are no longer using local_settings.py # for fence. # ----------------------------------------------------------------------------- - - name: old-config-volume - secret: - secretName: "fence-secret" - name: creds-volume secret: secretName: "fence-creds" - - name: config-helper - configMap: - name: config-helper - - name: json-secret-volume - secret: - secretName: "fence-json-secret" # ----------------------------------------------------------------------------- - name: config-volume secret: @@ -113,22 +104,10 @@ spec: # DEPRECATED! Remove when all commons are no longer using local_settings.py # for fence. # ----------------------------------------------------------------------------- - - name: "old-config-volume" - readOnly: true - mountPath: "/var/www/fence/local_settings.py" - subPath: local_settings.py - name: "creds-volume" readOnly: true mountPath: "/var/www/fence/creds.json" subPath: creds.json - - name: "config-helper" - readOnly: true - mountPath: "/var/www/fence/config_helper.py" - subPath: config_helper.py - - name: "json-secret-volume" - readOnly: true - mountPath: "/var/www/fence/fence_credentials.json" - subPath: fence_credentials.json # ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index c421f6cdd..4240b177b 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.62 +version: 0.1.63 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/fence/README.md b/helm/fence/README.md index 6c5736e6f..42ac6bcc0 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -1,6 +1,6 @@ # fence -![Version: 0.1.62](https://img.shields.io/badge/Version-0.1.62-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.63](https://img.shields.io/badge/Version-0.1.63-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Fence @@ -217,5 +217,5 @@ A Helm chart for gen3 Fence | usersync.syncFromDbgap | bool | `false` | Whether to sync data from dbGaP. | | usersync.userYamlS3Path | string | `"s3://cdis-gen3-users/helm-test/user.yaml"` | Path to the user.yaml file in S3. | | usersync.usersync | bool | `false` | Whether to run Fence usersync or not. | -| volumeMounts | list | `[{"mountPath":"/var/www/fence/local_settings.py","name":"old-config-volume","readOnly":true,"subPath":"local_settings.py"},{"mountPath":"/var/www/fence/fence_credentials.json","name":"json-secret-volume","readOnly":true,"subPath":"fence_credentials.json"},{"mountPath":"/var/www/fence/creds.json","name":"creds-volume","readOnly":true,"subPath":"creds.json"},{"mountPath":"/var/www/fence/config_helper.py","name":"config-helper","readOnly":true,"subPath":"config_helper.py"},{"mountPath":"/fence/fence/static/img/logo.svg","name":"logo-volume","readOnly":true,"subPath":"logo.svg"},{"mountPath":"/fence/fence/static/privacy_policy.md","name":"privacy-policy","readOnly":true,"subPath":"privacy_policy.md"},{"mountPath":"/var/www/fence/fence-config-secret.yaml","name":"config-volume","readOnly":true,"subPath":"fence-config.yaml"},{"mountPath":"/var/www/fence/yaml_merge.py","name":"yaml-merge","readOnly":true,"subPath":"yaml_merge.py"},{"mountPath":"/var/www/fence/fence_google_app_creds_secret.json","name":"fence-google-app-creds-secret-volume","readOnly":true,"subPath":"fence_google_app_creds_secret.json"},{"mountPath":"/var/www/fence/fence_google_storage_creds_secret.json","name":"fence-google-storage-creds-secret-volume","readOnly":true,"subPath":"fence_google_storage_creds_secret.json"},{"mountPath":"/fence/keys/key/jwt_private_key.pem","name":"fence-jwt-keys","readOnly":true,"subPath":"jwt_private_key.pem"},{"mountPath":"/var/www/fence/fence-config-public.yaml","name":"config-volume-public","readOnly":true,"subPath":"fence-config-public.yaml"}]` | Volumes to mount to the container. | -| volumes | list | `[{"name":"old-config-volume","secret":{"secretName":"fence-secret"}},{"name":"json-secret-volume","secret":{"optional":true,"secretName":"fence-json-secret"}},{"name":"creds-volume","secret":{"secretName":"fence-creds"}},{"configMap":{"name":"config-helper","optional":true},"name":"config-helper"},{"configMap":{"name":"logo-config"},"name":"logo-volume"},{"name":"config-volume","secret":{"secretName":"fence-config"}},{"name":"fence-google-app-creds-secret-volume","secret":{"secretName":"fence-google-app-creds-secret"}},{"name":"fence-google-storage-creds-secret-volume","secret":{"secretName":"fence-google-storage-creds-secret"}},{"name":"fence-jwt-keys","secret":{"secretName":"fence-jwt-keys"}},{"configMap":{"name":"privacy-policy"},"name":"privacy-policy"},{"configMap":{"name":"fence-yaml-merge","optional":false},"name":"yaml-merge"},{"configMap":{"name":"manifest-fence","optional":true},"name":"config-volume-public"}]` | Volumes to attach to the container. | +| volumeMounts | list | `[{"mountPath":"/var/www/fence/creds.json","name":"creds-volume","readOnly":true,"subPath":"creds.json"},{"mountPath":"/fence/fence/static/img/logo.svg","name":"logo-volume","readOnly":true,"subPath":"logo.svg"},{"mountPath":"/fence/fence/static/privacy_policy.md","name":"privacy-policy","readOnly":true,"subPath":"privacy_policy.md"},{"mountPath":"/var/www/fence/fence-config-secret.yaml","name":"config-volume","readOnly":true,"subPath":"fence-config.yaml"},{"mountPath":"/var/www/fence/yaml_merge.py","name":"yaml-merge","readOnly":true,"subPath":"yaml_merge.py"},{"mountPath":"/var/www/fence/fence_google_app_creds_secret.json","name":"fence-google-app-creds-secret-volume","readOnly":true,"subPath":"fence_google_app_creds_secret.json"},{"mountPath":"/var/www/fence/fence_google_storage_creds_secret.json","name":"fence-google-storage-creds-secret-volume","readOnly":true,"subPath":"fence_google_storage_creds_secret.json"},{"mountPath":"/fence/keys/key/jwt_private_key.pem","name":"fence-jwt-keys","readOnly":true,"subPath":"jwt_private_key.pem"},{"mountPath":"/var/www/fence/fence-config-public.yaml","name":"config-volume-public","readOnly":true,"subPath":"fence-config-public.yaml"}]` | Volumes to mount to the container. | +| volumes | list | `[{"name":"creds-volume","secret":{"secretName":"fence-creds"}},{"configMap":{"name":"logo-config"},"name":"logo-volume"},{"name":"config-volume","secret":{"secretName":"fence-config"}},{"name":"fence-google-app-creds-secret-volume","secret":{"secretName":"fence-google-app-creds-secret"}},{"name":"fence-google-storage-creds-secret-volume","secret":{"secretName":"fence-google-storage-creds-secret"}},{"name":"fence-jwt-keys","secret":{"secretName":"fence-jwt-keys"}},{"configMap":{"name":"privacy-policy"},"name":"privacy-policy"},{"configMap":{"name":"fence-yaml-merge","optional":false},"name":"yaml-merge"},{"configMap":{"name":"manifest-fence","optional":true},"name":"config-volume-public"}]` | Volumes to attach to the container. | diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml index 273aec01e..dcea975cd 100644 --- a/helm/fence/values.yaml +++ b/helm/fence/values.yaml @@ -389,20 +389,9 @@ env: # -- (list) Volumes to attach to the container. volumes: - - name: old-config-volume - secret: - secretName: "fence-secret" - - name: json-secret-volume - secret: - secretName: "fence-json-secret" - optional: true - name: creds-volume secret: secretName: "fence-creds" - - name: config-helper - configMap: - name: config-helper - optional: true - name: logo-volume configMap: name: "logo-config" @@ -432,22 +421,10 @@ volumes: # -- (list) Volumes to mount to the container. volumeMounts: - - name: "old-config-volume" - readOnly: true - mountPath: "/var/www/fence/local_settings.py" - subPath: local_settings.py - - name: "json-secret-volume" - readOnly: true - mountPath: "/var/www/fence/fence_credentials.json" - subPath: fence_credentials.json - name: "creds-volume" readOnly: true mountPath: "/var/www/fence/creds.json" subPath: creds.json - - name: "config-helper" - readOnly: true - mountPath: "/var/www/fence/config_helper.py" - subPath: config_helper.py - name: "logo-volume" readOnly: true mountPath: "/fence/fence/static/img/logo.svg" diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index fe9736cb1..546aab711 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -37,7 +37,7 @@ dependencies: repository: "file://../cohort-middleware" condition: cohort-middleware.enabled - name: common - version: 0.1.24 + version: 0.1.25 repository: file://../common - name: dashboard version: 0.1.9 @@ -56,7 +56,7 @@ dependencies: repository: "file://../frontend-framework" condition: frontend-framework.enabled - name: fence - version: 0.1.62 + version: 0.1.63 repository: "file://../fence" condition: fence.enabled - name: gen3-user-data-library @@ -169,7 +169,7 @@ type: application # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.76 +version: 0.2.77 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 8dc4340fd..115c183b5 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -1,6 +1,6 @@ # gen3 -![Version: 0.2.76](https://img.shields.io/badge/Version-0.2.76-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.2.77](https://img.shields.io/badge/Version-0.2.77-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) Helm chart to deploy Gen3 Data Commons @@ -26,12 +26,12 @@ Helm chart to deploy Gen3 Data Commons | file://../aws-es-proxy | aws-es-proxy | 0.1.31 | | file://../cedar | cedar | 0.1.13 | | file://../cohort-middleware | cohort-middleware | 0.1.12 | -| file://../common | common | 0.1.24 | +| file://../common | common | 0.1.25 | | file://../dashboard | dashboard | 0.1.9 | | file://../datareplicate | datareplicate | 0.0.33 | | file://../dicom-server | dicom-server | 0.1.20 | | file://../etl | etl | 0.1.17 | -| file://../fence | fence | 0.1.62 | +| file://../fence | fence | 0.1.63 | | file://../frontend-framework | frontend-framework | 0.1.14 | | file://../gen3-analysis | gen3-analysis | 0.1.2 | | file://../gen3-network-policies | gen3-network-policies | 0.1.2 | diff --git a/wip/gen3-test-data-job/templates/_jobs.tpl b/wip/gen3-test-data-job/templates/_jobs.tpl index 8629182b6..260a2ff47 100644 --- a/wip/gen3-test-data-job/templates/_jobs.tpl +++ b/wip/gen3-test-data-job/templates/_jobs.tpl @@ -25,18 +25,9 @@ spec: # DEPRECATED! Remove when all commons are no longer using local_settings.py # for fence. # ----------------------------------------------------------------------------- - - name: old-config-volume - secret: - secretName: "fence-secret" - name: creds-volume secret: secretName: "fence-creds" - - name: config-helper - configMap: - name: config-helper - - name: json-secret-volume - secret: - secretName: "fence-json-secret" # ----------------------------------------------------------------------------- - name: config-volume secret: @@ -113,22 +104,10 @@ spec: # DEPRECATED! Remove when all commons are no longer using local_settings.py # for fence. # ----------------------------------------------------------------------------- - - name: "old-config-volume" - readOnly: true - mountPath: "/var/www/fence/local_settings.py" - subPath: local_settings.py - name: "creds-volume" readOnly: true mountPath: "/var/www/fence/creds.json" subPath: creds.json - - name: "config-helper" - readOnly: true - mountPath: "/var/www/fence/config_helper.py" - subPath: config_helper.py - - name: "json-secret-volume" - readOnly: true - mountPath: "/var/www/fence/fence_credentials.json" - subPath: fence_credentials.json # ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true diff --git a/wip/gen3-test-data-job/templates/jobs.yaml b/wip/gen3-test-data-job/templates/jobs.yaml index 5b86fbf9a..47318f449 100644 --- a/wip/gen3-test-data-job/templates/jobs.yaml +++ b/wip/gen3-test-data-job/templates/jobs.yaml @@ -20,18 +20,9 @@ spec: # DEPRECATED! Remove when all commons are no longer using local_settings.py # for fence. # ----------------------------------------------------------------------------- - - name: old-config-volume - secret: - secretName: "fence-secret" - name: creds-volume secret: secretName: "fence-creds" - - name: config-helper - configMap: - name: config-helper - - name: json-secret-volume - secret: - secretName: "fence-json-secret" # ----------------------------------------------------------------------------- - name: config-volume secret: @@ -108,22 +99,11 @@ spec: # DEPRECATED! Remove when all commons are no longer using local_settings.py # for fence. # ----------------------------------------------------------------------------- - - name: "old-config-volume" - readOnly: true - mountPath: "/var/www/fence/local_settings.py" - subPath: local_settings.py + - name: "creds-volume" readOnly: true mountPath: "/var/www/fence/creds.json" subPath: creds.json - - name: "config-helper" - readOnly: true - mountPath: "/var/www/fence/config_helper.py" - subPath: config_helper.py - - name: "json-secret-volume" - readOnly: true - mountPath: "/var/www/fence/fence_credentials.json" - subPath: fence_credentials.json # ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true From 53d417d59c650a718d0b172662892701c39a47dc Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 7 Nov 2025 14:15:06 -0600 Subject: [PATCH 2/5] Remove unused config files and volumemounts --- helm/cluster-level-resources/README.md | 22 +- helm/common/templates/_generate_test_data.tpl | 17 - helm/fence/README.md | 4 +- helm/fence/fence-secret/config_helper.py | 376 ------------------ helm/fence/fence-secret/fence_settings.py | 170 -------- helm/fence/templates/fence-creds.yaml | 19 - helm/fence/templates/usersync-cron.yaml | 7 - helm/fence/values.yaml | 7 - helm/gen3/Chart.yaml | 4 +- helm/gen3/README.md | 4 +- helm/peregrine/Chart.yaml | 2 +- helm/peregrine/README.md | 2 +- .../peregrine-secret/config_helper.py | 376 ------------------ helm/peregrine/peregrine-secret/settings.py | 5 - helm/sheepdog/Chart.yaml | 2 +- helm/sheepdog/README.md | 2 +- .../sheepdog/sheepdog-secret/config_helper.py | 376 ------------------ helm/sheepdog/sheepdog-secret/settings.py | 5 - wip/gen3-test-data-job/templates/_jobs.tpl | 17 - wip/gen3-test-data-job/templates/jobs.yaml | 18 - 20 files changed, 21 insertions(+), 1414 deletions(-) delete mode 100644 helm/fence/fence-secret/config_helper.py delete mode 100644 helm/fence/fence-secret/fence_settings.py delete mode 100644 helm/fence/templates/fence-creds.yaml delete mode 100644 helm/peregrine/peregrine-secret/config_helper.py delete mode 100644 helm/sheepdog/sheepdog-secret/config_helper.py diff --git a/helm/cluster-level-resources/README.md b/helm/cluster-level-resources/README.md index c0653d75f..625f5ba05 100644 --- a/helm/cluster-level-resources/README.md +++ b/helm/cluster-level-resources/README.md @@ -1,6 +1,6 @@ # cluster-level-resources -![Version: 0.6.17](https://img.shields.io/badge/Version-0.6.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.17.0](https://img.shields.io/badge/AppVersion-1.17.0-informational?style=flat-square) +![Version: 0.6.18](https://img.shields.io/badge/Version-0.6.18-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.17.0](https://img.shields.io/badge/AppVersion-1.17.0-informational?style=flat-square) An app-of-apps Helm chart that allows for flexible deployment of resources that support Gen3 @@ -11,14 +11,14 @@ An app-of-apps Helm chart that allows for flexible deployment of resources that | accountNumber | string | `"xxxxxxxxxxxx"` | | | alb-controller.configuration.enabled | bool | `false` | | | alb-controller.enabled | bool | `false` | | -| alb-controller.targetRevision | string | `"1.11.0"` | | +| alb-controller.targetRevision | string | `"1.13.4"` | | | alloy-configmap-data | string | `"logging {\n level = \"info\"\n format = \"json\"\n write_to = [loki.write.endpoint.receiver]\n}\n\n/////////////////////// OTLP START ///////////////////////\n\notelcol.receiver.otlp \"default\" {\n grpc {}\n http {}\n\n output {\n metrics = [otelcol.processor.batch.default.input]\n traces = [otelcol.processor.batch.default.input]\n }\n}\n\notelcol.processor.batch \"default\" {\n output {\n metrics = [otelcol.exporter.prometheus.default.input]\n traces = [otelcol.exporter.otlp.tempo.input]\n }\n}\n\notelcol.exporter.prometheus \"default\" {\n forward_to = [prometheus.remote_write.default.receiver]\n}\n\notelcol.exporter.otlp \"tempo\" {\n client {\n endpoint = \"http://monitoring-tempo-distributor.monitoring:4317\"\n // Configure TLS settings for communicating with the endpoint.\n tls {\n // The connection is insecure.\n insecure = true\n // Do not verify TLS certificates when connecting.\n insecure_skip_verify = true\n }\n }\n}\n\n\n/////////////////////// OTLP END ///////////////////////\n\n// discover all pods, to be used later in this config\ndiscovery.kubernetes \"pods\" {\n role = \"pod\"\n}\n\n// discover all services, to be used later in this config\ndiscovery.kubernetes \"services\" {\n role = \"service\"\n}\n\n// discover all nodes, to be used later in this config\ndiscovery.kubernetes \"nodes\" {\n role = \"node\"\n}\n\n// Generic scrape of any pod with Annotation \"prometheus.io/scrape: true\"\ndiscovery.relabel \"annotation_autodiscovery_pods\" {\n targets = discovery.kubernetes.pods.targets\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scrape\"]\n regex = \"true\"\n action = \"keep\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_job\"]\n action = \"replace\"\n target_label = \"job\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_instance\"]\n action = \"replace\"\n target_label = \"instance\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_path\"]\n action = \"replace\"\n target_label = \"__metrics_path__\"\n }\n\n // Choose the pod port\n // The discovery generates a target for each declared container port of the pod.\n // If the metricsPortName annotation has value, keep only the target where the port name matches the one of the annotation.\n rule {\n source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n target_label = \"__tmp_port\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_portName\"]\n regex = \"(.+)\"\n target_label = \"__tmp_port\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_container_port_name\"]\n action = \"keepequal\"\n target_label = \"__tmp_port\"\n }\n\n // If the metrics port number annotation has a value, override the target address to use it, regardless whether it is\n // one of the declared ports on that Pod.\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n regex = \"(\\\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})\"\n replacement = \"[$2]:$1\" // IPv6\n target_label = \"__address__\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_port\", \"__meta_kubernetes_pod_ip\"]\n regex = \"(\\\\d+);((([0-9]+?)(\\\\.|$)){4})\" // IPv4, takes priority over IPv6 when both exists\n replacement = \"$2:$1\"\n target_label = \"__address__\"\n }\n\n rule {\n source_labels = [\"__meta_kubernetes_pod_annotation_prometheus_io_scheme\"]\n action = \"replace\"\n target_label = \"__scheme__\"\n }\n\n\n // add labels\n rule {\n source_labels = [\"__meta_kubernetes_pod_name\"]\n target_label = \"pod\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_container_name\"]\n target_label = \"container\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_controller_name\"]\n target_label = \"controller\"\n }\n\n rule {\n source_labels = [\"__meta_kubernetes_namespace\"]\n target_label = \"namespace\"\n }\n\n\n rule {\n source_labels = [\"__meta_kubernetes_pod_label_app\"]\n target_label = \"app\"\n }\n\n // map all labels\n rule {\n action = \"labelmap\"\n regex = \"__meta_kubernetes_pod_label_(.+)\"\n }\n}\n\n// Generic scrape of any service with\n// Annotation Autodiscovery\ndiscovery.relabel \"annotation_autodiscovery_services\" {\n targets = discovery.kubernetes.services.targets\n rule {\n source_labels = [\"__meta_kubernetes_service_annotation_prometheus_io_scrape\"]\n regex = \"true\"\n action = \"keep\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_service_annotation_prometheus_io_job\"]\n action = \"replace\"\n target_label = \"job\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_service_annotation_prometheus_io_instance\"]\n action = \"replace\"\n target_label = \"instance\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_service_annotation_prometheus_io_path\"]\n action = \"replace\"\n target_label = \"__metrics_path__\"\n }\n\n // Choose the service port\n rule {\n source_labels = [\"__meta_kubernetes_service_port_name\"]\n target_label = \"__tmp_port\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_service_annotation_prometheus_io_portName\"]\n regex = \"(.+)\"\n target_label = \"__tmp_port\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_service_port_name\"]\n action = \"keepequal\"\n target_label = \"__tmp_port\"\n }\n\n rule {\n source_labels = [\"__meta_kubernetes_service_port_number\"]\n target_label = \"__tmp_port\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_service_annotation_prometheus_io_port\"]\n regex = \"(.+)\"\n target_label = \"__tmp_port\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_service_port_number\"]\n action = \"keepequal\"\n target_label = \"__tmp_port\"\n }\n\n rule {\n source_labels = [\"__meta_kubernetes_service_annotation_prometheus_io_scheme\"]\n action = \"replace\"\n target_label = \"__scheme__\"\n }\n}\n\nprometheus.scrape \"metrics\" {\n job_name = \"integrations/autodiscovery_metrics\"\n targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output)\n honor_labels = true\n clustering {\n enabled = true\n }\n forward_to = [prometheus.relabel.metrics_service.receiver]\n}\n\n\n// Node Exporter\n// TODO: replace with https://grafana.com/docs/alloy/latest/reference/components/prometheus.exporter.unix/\ndiscovery.relabel \"node_exporter\" {\n targets = discovery.kubernetes.pods.targets\n rule {\n source_labels = [\"__meta_kubernetes_pod_label_app_kubernetes_io_instance\"]\n regex = \"monitoring-extras\"\n action = \"keep\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_label_app_kubernetes_io_name\"]\n regex = \"node-exporter\"\n action = \"keep\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_node_name\"]\n action = \"replace\"\n target_label = \"instance\"\n }\n}\n\nprometheus.scrape \"node_exporter\" {\n job_name = \"integrations/node_exporter\"\n targets = discovery.relabel.node_exporter.output\n scrape_interval = \"60s\"\n clustering {\n enabled = true\n }\n forward_to = [prometheus.relabel.node_exporter.receiver]\n}\n\nprometheus.relabel \"node_exporter\" {\n rule {\n source_labels = [\"__name__\"]\n regex = \"up|node_cpu.*|node_network.*|node_exporter_build_info|node_filesystem.*|node_memory.*|process_cpu_seconds_total|process_resident_memory_bytes\"\n action = \"keep\"\n }\n forward_to = [prometheus.relabel.metrics_service.receiver]\n}\n\n\n// cAdvisor\n// discovery.relabel \"cadvisor\" {\n// targets = discovery.kubernetes.nodes.targets\n// rule {\n// target_label = \"__address__\"\n// replacement = \"kubernetes.default.svc.cluster.local:443\"\n// }\n// rule {\n// source_labels = [\"__meta_kubernetes_node_name\"]\n// regex = \"(.+)\"\n// replacement = \"/api/v1/nodes/${1}/proxy/metrics/cadvisor\"\n// target_label = \"__metrics_path__\"\n// }\n// }\n\n// prometheus.scrape \"cadvisor\" {\n// job_name = \"integrations/kubernetes/cadvisor\"\n// targets = discovery.relabel.cadvisor.output\n// scheme = \"https\"\n// scrape_interval = \"60s\"\n// bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n// tls_config {\n// insecure_skip_verify = true\n// }\n// clustering {\n// enabled = true\n// }\n// forward_to = [prometheus.relabel.cadvisor.receiver]\n//}\n\n//prometheus.relabel \"cadvisor\" {\n// rule {\n// source_labels = [\"__name__\"]\n// regex = \"up|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total|container_cpu_usage_seconds_total|container_fs_reads_bytes_total|container_fs_reads_total|container_fs_writes_bytes_total|container_fs_writes_total|container_memory_cache|container_memory_rss|container_memory_swap|container_memory_working_set_bytes|container_network_receive_bytes_total|container_network_receive_packets_dropped_total|container_network_receive_packets_total|container_network_transmit_bytes_total|container_network_transmit_packets_dropped_total|container_network_transmit_packets_total|machine_memory_bytes\"\n// action = \"keep\"\n// }\n// forward_to = [prometheus.relabel.metrics_service.receiver]\n// }\n\n// Logs from all pods\ndiscovery.relabel \"all_pods\" {\n targets = discovery.kubernetes.pods.targets\n rule {\n source_labels = [\"__meta_kubernetes_namespace\"]\n target_label = \"namespace\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_name\"]\n target_label = \"pod\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_container_name\"]\n target_label = \"container\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_pod_controller_name\"]\n target_label = \"controller\"\n }\n\n rule {\n source_labels = [\"__meta_kubernetes_pod_label_app\"]\n target_label = \"app\"\n }\n\n // map all labels\n rule {\n action = \"labelmap\"\n regex = \"__meta_kubernetes_pod_label_(.+)\"\n }\n\n}\n\nloki.source.kubernetes \"pods\" {\n targets = discovery.relabel.all_pods.output\n forward_to = [loki.write.endpoint.receiver]\n}\n\n// kube-state-metrics\ndiscovery.relabel \"relabel_kube_state_metrics\" {\n targets = discovery.kubernetes.services.targets\n rule {\n source_labels = [\"__meta_kubernetes_namespace\"]\n regex = \"monitoring\"\n action = \"keep\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_service_name\"]\n regex = \"monitoring-extras-kube-state-metrics\"\n action = \"keep\"\n }\n}\n\nprometheus.scrape \"kube_state_metrics\" {\n targets = discovery.relabel.relabel_kube_state_metrics.output\n job_name = \"kube-state-metrics\"\n metrics_path = \"/metrics\"\n forward_to = [prometheus.remote_write.default.receiver]\n}\n\n// Kubelet\ndiscovery.relabel \"kubelet\" {\n targets = discovery.kubernetes.nodes.targets\n rule {\n target_label = \"__address__\"\n replacement = \"kubernetes.default.svc.cluster.local:443\"\n }\n rule {\n source_labels = [\"__meta_kubernetes_node_name\"]\n regex = \"(.+)\"\n replacement = \"/api/v1/nodes/${1}/proxy/metrics\"\n target_label = \"__metrics_path__\"\n }\n}\n\nprometheus.scrape \"kubelet\" {\n job_name = \"integrations/kubernetes/kubelet\"\n targets = discovery.relabel.kubelet.output\n scheme = \"https\"\n scrape_interval = \"60s\"\n bearer_token_file = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n tls_config {\n insecure_skip_verify = true\n }\n clustering {\n enabled = true\n }\n forward_to = [prometheus.relabel.kubelet.receiver]\n}\n\nprometheus.relabel \"kubelet\" {\n rule {\n source_labels = [\"__name__\"]\n regex = \"up|container_cpu_usage_seconds_total|kubelet_certificate_manager_client_expiration_renew_errors|kubelet_certificate_manager_client_ttl_seconds|kubelet_certificate_manager_server_ttl_seconds|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_count|kubelet_node_config_error|kubelet_node_name|kubelet_pleg_relist_duration_seconds_bucket|kubelet_pleg_relist_duration_seconds_count|kubelet_pleg_relist_interval_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_start_duration_seconds_count|kubelet_pod_worker_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_count|kubelet_running_container_count|kubelet_running_containers|kubelet_running_pod_count|kubelet_running_pods|kubelet_runtime_operations_errors_total|kubelet_runtime_operations_total|kubelet_server_expiration_renew_errors|kubelet_volume_stats_available_bytes|kubelet_volume_stats_capacity_bytes|kubelet_volume_stats_inodes|kubelet_volume_stats_inodes_used|kubernetes_build_info|namespace_workload_pod|rest_client_requests_total|storage_operation_duration_seconds_count|storage_operation_errors_total|volume_manager_total_volumes\"\n action = \"keep\"\n }\n forward_to = [prometheus.relabel.metrics_service.receiver]\n}\n\n// Cluster Events\nloki.source.kubernetes_events \"cluster_events\" {\n job_name = \"integrations/kubernetes/eventhandler\"\n log_format = \"logfmt\"\n forward_to = [loki.write.endpoint.receiver]\n}\n\n\n// Why is this needed?\nprometheus.relabel \"metrics_service\" {\n forward_to = [prometheus.remote_write.default.receiver]\n}\n\n\n// Write Endpoints\n// prometheus write endpoint\nprometheus.remote_write \"default\" {\n external_labels = {\n cluster = \"{{ .Values.cluster }}\",\n project = \"{{ .Values.project }}\",\n }\n endpoint {\n url = \"https://mimir.planx-pla.net/api/v1/push\"\n\n headers = {\n \"X-Scope-OrgID\" = \"anonymous\",\n }\n\n }\n}\n\n// loki write endpoint\nloki.write \"endpoint\" {\n external_labels = {\n cluster = \"{{ .Values.cluster }}\",\n project = \"{{ .Values.project }}\",\n }\n endpoint {\n url = \"https://loki.planx-pla.net/loki/api/v1/push\"\n }\n}\n"` | | | aws-s3-mountpoint.configuration.enabled | bool | `false` | | | aws-s3-mountpoint.enabled | bool | `false` | | -| aws-s3-mountpoint.targetRevision | string | `"1.11.0"` | | +| aws-s3-mountpoint.targetRevision | string | `"2.2.0"` | | | calico.configuration.enabled | bool | `false` | | | calico.enabled | bool | `false` | | -| calico.targetRevision | string | `"v3.29.1"` | | +| calico.targetRevision | string | `"v3.30.3"` | | | cert-manager.configuration.enabled | bool | `false` | | | cert-manager.enabled | bool | `false` | | | cert-manager.targetRevision | string | `"v1.17.2"` | | @@ -27,7 +27,7 @@ An app-of-apps Helm chart that allows for flexible deployment of resources that | configuration.configurationRevision | string | `"master"` | | | coreDNS.configuration.enabled | bool | `false` | | | coreDNS.enabled | bool | `false` | | -| coreDNS.targetRevision | string | `"v1.37.0"` | | +| coreDNS.targetRevision | string | `"1.43.1"` | | | crossplane-crds.awsAccessKeyID | string | `"YOUR_ACCESS_KEY_ID"` | | | crossplane-crds.awsSecretAccessKey | string | `"YOUR_SECRET_ACCESS_KEY"` | | | crossplane-crds.configuration.enabled | bool | `false` | | @@ -42,7 +42,7 @@ An app-of-apps Helm chart that allows for flexible deployment of resources that | custom-objects.enabled | bool | `false` | | | ebs-csi-driver.configuration.enabled | bool | `false` | | | ebs-csi-driver.enabled | bool | `false` | | -| ebs-csi-driver.targetRevision | string | `"2.38.1"` | | +| ebs-csi-driver.targetRevision | string | `"2.48.0"` | | | eksClusterEndpoint | string | `""` | | | external-secrets.configuration.enabled | bool | `false` | | | external-secrets.enabled | bool | `false` | | @@ -121,21 +121,21 @@ An app-of-apps Helm chart that allows for flexible deployment of resources that | karpenter.awsRegion | string | `"us-east-1"` | | | karpenter.configuration.enabled | bool | `false` | | | karpenter.controller.image.digest | string | `"sha256:0c142050d872cb0ac7b30a188ec36aa765b449718cde0c7e49f7495b28f47c29"` | | -| karpenter.controller.image.tag | string | `"v0.32.9"` | | +| karpenter.controller.image.tag | string | `"1.0.8"` | | | karpenter.enabled | bool | `false` | | | karpenter.resources.limits.cpu | string | `"1"` | | | karpenter.resources.limits.memory | string | `"1Gi"` | | | karpenter.resources.requests.cpu | string | `"1"` | | | karpenter.resources.requests.memory | string | `"1Gi"` | | -| karpenter.targetRevision | string | `"v0.32.9"` | | +| karpenter.targetRevision | string | `"1.0.8"` | | | kube-proxy.configuration.enabled | bool | `false` | | | kube-proxy.enabled | bool | `false` | | | kube-proxy.image.repo | string | `"602401143452.dkr.ecr.us-east-1.amazonaws.com/eks/kube-proxy"` | | -| kube-proxy.image.tag | string | `"v1.31.2-minimal-eksbuild.3"` | | +| kube-proxy.image.tag | string | `"v1.33.3-minimal-eksbuild.11"` | | | kube-state-metrics.configuration.enabled | bool | `false` | | | kube-state-metrics.enabled | bool | `false` | | -| kube-state-metrics.targetRevision | string | `"5.28.0"` | | +| kube-state-metrics.targetRevision | string | `"6.3.0"` | | | project | string | `"unfunded"` | | | vpc-cni.configuration.enabled | bool | `false` | | | vpc-cni.enabled | bool | `false` | | -| vpc-cni.targetRevision | string | `"v1.16.2"` | | +| vpc-cni.targetRevision | string | `"v1.20.4"` | | diff --git a/helm/common/templates/_generate_test_data.tpl b/helm/common/templates/_generate_test_data.tpl index e7545a1a7..f5bcc7a6c 100644 --- a/helm/common/templates/_generate_test_data.tpl +++ b/helm/common/templates/_generate_test_data.tpl @@ -21,14 +21,6 @@ spec: name: "fence-yaml-merge" - name: shared-data emptyDir: {} -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: creds-volume - secret: - secretName: "fence-creds" -# ----------------------------------------------------------------------------- - name: config-volume secret: secretName: "fence-config" @@ -100,15 +92,6 @@ spec: key: fence-config-public.yaml optional: true volumeMounts: -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json -# ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true mountPath: "/var/www/fence/fence-config-secret.yaml" diff --git a/helm/fence/README.md b/helm/fence/README.md index 42ac6bcc0..ac38c5c8f 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -217,5 +217,5 @@ A Helm chart for gen3 Fence | usersync.syncFromDbgap | bool | `false` | Whether to sync data from dbGaP. | | usersync.userYamlS3Path | string | `"s3://cdis-gen3-users/helm-test/user.yaml"` | Path to the user.yaml file in S3. | | usersync.usersync | bool | `false` | Whether to run Fence usersync or not. | -| volumeMounts | list | `[{"mountPath":"/var/www/fence/creds.json","name":"creds-volume","readOnly":true,"subPath":"creds.json"},{"mountPath":"/fence/fence/static/img/logo.svg","name":"logo-volume","readOnly":true,"subPath":"logo.svg"},{"mountPath":"/fence/fence/static/privacy_policy.md","name":"privacy-policy","readOnly":true,"subPath":"privacy_policy.md"},{"mountPath":"/var/www/fence/fence-config-secret.yaml","name":"config-volume","readOnly":true,"subPath":"fence-config.yaml"},{"mountPath":"/var/www/fence/yaml_merge.py","name":"yaml-merge","readOnly":true,"subPath":"yaml_merge.py"},{"mountPath":"/var/www/fence/fence_google_app_creds_secret.json","name":"fence-google-app-creds-secret-volume","readOnly":true,"subPath":"fence_google_app_creds_secret.json"},{"mountPath":"/var/www/fence/fence_google_storage_creds_secret.json","name":"fence-google-storage-creds-secret-volume","readOnly":true,"subPath":"fence_google_storage_creds_secret.json"},{"mountPath":"/fence/keys/key/jwt_private_key.pem","name":"fence-jwt-keys","readOnly":true,"subPath":"jwt_private_key.pem"},{"mountPath":"/var/www/fence/fence-config-public.yaml","name":"config-volume-public","readOnly":true,"subPath":"fence-config-public.yaml"}]` | Volumes to mount to the container. | -| volumes | list | `[{"name":"creds-volume","secret":{"secretName":"fence-creds"}},{"configMap":{"name":"logo-config"},"name":"logo-volume"},{"name":"config-volume","secret":{"secretName":"fence-config"}},{"name":"fence-google-app-creds-secret-volume","secret":{"secretName":"fence-google-app-creds-secret"}},{"name":"fence-google-storage-creds-secret-volume","secret":{"secretName":"fence-google-storage-creds-secret"}},{"name":"fence-jwt-keys","secret":{"secretName":"fence-jwt-keys"}},{"configMap":{"name":"privacy-policy"},"name":"privacy-policy"},{"configMap":{"name":"fence-yaml-merge","optional":false},"name":"yaml-merge"},{"configMap":{"name":"manifest-fence","optional":true},"name":"config-volume-public"}]` | Volumes to attach to the container. | +| volumeMounts | list | `[{"mountPath":"/fence/fence/static/img/logo.svg","name":"logo-volume","readOnly":true,"subPath":"logo.svg"},{"mountPath":"/fence/fence/static/privacy_policy.md","name":"privacy-policy","readOnly":true,"subPath":"privacy_policy.md"},{"mountPath":"/var/www/fence/fence-config-secret.yaml","name":"config-volume","readOnly":true,"subPath":"fence-config.yaml"},{"mountPath":"/var/www/fence/yaml_merge.py","name":"yaml-merge","readOnly":true,"subPath":"yaml_merge.py"},{"mountPath":"/var/www/fence/fence_google_app_creds_secret.json","name":"fence-google-app-creds-secret-volume","readOnly":true,"subPath":"fence_google_app_creds_secret.json"},{"mountPath":"/var/www/fence/fence_google_storage_creds_secret.json","name":"fence-google-storage-creds-secret-volume","readOnly":true,"subPath":"fence_google_storage_creds_secret.json"},{"mountPath":"/fence/keys/key/jwt_private_key.pem","name":"fence-jwt-keys","readOnly":true,"subPath":"jwt_private_key.pem"},{"mountPath":"/var/www/fence/fence-config-public.yaml","name":"config-volume-public","readOnly":true,"subPath":"fence-config-public.yaml"}]` | Volumes to mount to the container. | +| volumes | list | `[{"configMap":{"name":"logo-config"},"name":"logo-volume"},{"name":"config-volume","secret":{"secretName":"fence-config"}},{"name":"fence-google-app-creds-secret-volume","secret":{"secretName":"fence-google-app-creds-secret"}},{"name":"fence-google-storage-creds-secret-volume","secret":{"secretName":"fence-google-storage-creds-secret"}},{"name":"fence-jwt-keys","secret":{"secretName":"fence-jwt-keys"}},{"configMap":{"name":"privacy-policy"},"name":"privacy-policy"},{"configMap":{"name":"fence-yaml-merge","optional":false},"name":"yaml-merge"},{"configMap":{"name":"manifest-fence","optional":true},"name":"config-volume-public"}]` | Volumes to attach to the container. | diff --git a/helm/fence/fence-secret/config_helper.py b/helm/fence/fence-secret/config_helper.py deleted file mode 100644 index 6b303beac..000000000 --- a/helm/fence/fence-secret/config_helper.py +++ /dev/null @@ -1,376 +0,0 @@ -import json -import os -import copy -import argparse -import re -import types - -# -# make it easy to change this for testing -XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") - - -def default_search_folders(app_name): - """ - Return the list of folders to search for configuration files - """ - return [ - "%s/cdis/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/cdis/%s" % app_name, - "%s/gen3/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/gen3/%s" % app_name, - "/var/www/%s" % app_name, - "/etc/gen3/%s" % app_name, - ] - - -def find_paths(file_name, app_name, search_folders=None): - """ - Search the given folders for file_name - search_folders defaults to default_search_folders if not specified - return the first path to file_name found - """ - search_folders = search_folders or default_search_folders(app_name) - possible_files = [os.path.join(folder, file_name) for folder in search_folders] - return [path for path in possible_files if os.path.exists(path)] - - -def load_json(file_name, app_name, search_folders=None): - """ - json.load(file_name) after finding file_name in search_folders - - return the loaded json data or None if file not found - """ - actual_files = find_paths(file_name, app_name, search_folders) - if not actual_files: - return None - with open(actual_files[0], "r") as reader: - return json.load(reader) - - -def inject_creds_into_fence_config(creds_file_path, config_file_path): - creds_file = open(creds_file_path, "r") - creds = json.load(creds_file) - creds_file.close() - - # get secret values from creds.json file - db_host = _get_nested_value(creds, "db_host") - db_username = _get_nested_value(creds, "db_username") - db_password = _get_nested_value(creds, "db_password") - db_database = _get_nested_value(creds, "db_database") - hostname = _get_nested_value(creds, "hostname") - indexd_password = _get_nested_value(creds, "indexd_password") - google_client_secret = _get_nested_value(creds, "google_client_secret") - google_client_id = _get_nested_value(creds, "google_client_id") - hmac_key = _get_nested_value(creds, "hmac_key") - db_path = "postgresql://{}:{}@{}:5432/{}".format( - db_username, db_password, db_host, db_database - ) - - config_file = open(config_file_path, "r").read() - - print(" DB injected with value(s) from creds.json") - config_file = _replace(config_file, "DB", db_path) - - print(" BASE_URL injected with value(s) from creds.json") - config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - - print(" ENCRYPTION_KEY injected with value(s) from creds.json") - config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) - - print( - " OPENID_CONNECT/google/client_secret injected with value(s) " - "from creds.json" - ) - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_secret", google_client_secret - ) - - print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_id", google_client_id - ) - - open(config_file_path, "w+").write(config_file) - - -def set_prod_defaults(config_file_path): - config_file = open(config_file_path, "r").read() - - print( - " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " - "var/www/fence/fence_google_app_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", - "/var/www/fence/fence_google_app_creds_secret.json", - ) - - print( - " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " - "var/www/fence/fence_google_storage_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", - "/var/www/fence/fence_google_storage_creds_secret.json", - ) - - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - - print(" ARBORIST set as http://arborist-service/") - config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") - - print(" HTTP_PROXY/host set as cloud-proxy.internal.io") - config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") - - print(" HTTP_PROXY/port set as 3128") - config_file = _replace(config_file, "HTTP_PROXY/port", 3128) - - print(" DEBUG set to false") - config_file = _replace(config_file, "DEBUG", False) - - print(" MOCK_AUTH set to false") - config_file = _replace(config_file, "MOCK_AUTH", False) - - print(" MOCK_GOOGLE_AUTH set to false") - config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) - - print(" AUTHLIB_INSECURE_TRANSPORT set to true") - config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) - - print(" SESSION_COOKIE_SECURE set to true") - config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) - - print(" ENABLE_CSRF_PROTECTION set to true") - config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) - - open(config_file_path, "w+").write(config_file) - - -def inject_other_files_into_fence_config(other_files, config_file_path): - additional_cfgs = _get_all_additional_configs(other_files) - - config_file = open(config_file_path, "r").read() - - for key, value in additional_cfgs.iteritems(): - print(" {} set to {}".format(key, value)) - config_file = _nested_replace(config_file, key, value) - - open(config_file_path, "w+").write(config_file) - - -def _get_all_additional_configs(other_files): - """ - Attempt to parse given list of files and extract configuration variables and values - """ - additional_configs = dict() - for file_path in other_files: - try: - file_ext = file_path.strip().split(".")[-1] - if file_ext == "json": - json_file = open(file_path, "r") - configs = json.load(json_file) - json_file.close() - elif file_ext == "py": - configs = from_pyfile(file_path) - else: - print( - "Cannot load config vars from a file with extention: {}".format( - file_ext - ) - ) - except Exception as exc: - # if there's any issue reading the file, exit - print( - "Error reading {}. Cannot get configuration. Skipping this file. " - "Details: {}".format(other_files, str(exc)) - ) - continue - - if configs: - additional_configs.update(configs) - - return additional_configs - - -def _nested_replace(config_file, key, value, replacement_path=None): - replacement_path = replacement_path or key - try: - for inner_key, inner_value in value.iteritems(): - temp_path = replacement_path - temp_path = temp_path + "/" + inner_key - config_file = _nested_replace( - config_file, inner_key, inner_value, temp_path - ) - except AttributeError: - # not a dict so replace - if value is not None: - config_file = _replace(config_file, replacement_path, value) - - return config_file - - -def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): - """ - Replace a nested value in a YAML file string with the given value without - losing comments. Uses a regex to do the replacement. - - Args: - yaml_config (str): a string representing a full configuration file - path_to_key (str): nested/path/to/key. The value of this key will be - replaced - replacement_value (str): Replacement value for the key from - path_to_key - """ - nested_path_to_replace = path_to_key.split("/") - - # our regex looks for a specific number of spaces to ensure correct - # level of nesting. It matches to the end of the line - search_string = ( - " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" - ) - matches = re.search(search_string, yaml_config[start:]) - - # early return if we haven't found anything - if not matches: - return yaml_config - - # if we're on the last item in the path, we need to get the value and - # replace it in the original file - if len(nested_path_to_replace) == 1: - # replace the current key:value with the new replacement value - match_start = start + matches.start(0) + len(" " * nested_level) - match_end = start + matches.end(0) - yaml_config = ( - yaml_config[:match_start] - + "{}: {}\n".format( - nested_path_to_replace[0], - _get_yaml_replacement_value(replacement_value, nested_level), - ) - + yaml_config[match_end:] - ) - - return yaml_config - - # set new start point to past current match and move on to next match - start = matches.end(0) - nested_level += 1 - del nested_path_to_replace[0] - - return _replace( - yaml_config, - "/".join(nested_path_to_replace), - replacement_value, - start, - nested_level, - ) - - -def from_pyfile(filename, silent=False): - """ - Modeled after flask's ability to load in python files: - https://github.com/pallets/flask/blob/master/flask/config.py - - Some alterations were made but logic is essentially the same - """ - filename = os.path.abspath(filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except IOError as e: - print("Unable to load configuration file ({})".format(e.strerror)) - if silent: - return False - raise - return _from_object(d) - - -def _from_object(obj): - configs = {} - for key in dir(obj): - if key.isupper(): - configs[key] = getattr(obj, key) - return configs - - -def _get_yaml_replacement_value(value, nested_level=0): - if isinstance(value, str): - return "'" + value + "'" - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, list) or isinstance(value, set): - output = "" - for item in value: - # spaces for nested level then spaces and hyphen for each list item - output += ( - "\n" - + " " * nested_level - + " - " - + _get_yaml_replacement_value(item) - + "" - ) - return output - else: - return value - - -def _get_nested_value(dictionary, nested_path): - """ - Return a value from a dictionary given a path-like nesting of keys. - - Will default to an empty string if value cannot be found. - - Args: - dictionary (dict): a dictionary - nested_path (str): nested/path/to/key - - Returns: - ?: Value from dict - """ - replacement_value_path = nested_path.split("/") - replacement_value = copy.deepcopy(dictionary) - - for item in replacement_value_path: - replacement_value = replacement_value.get(item, {}) - - if replacement_value == {}: - replacement_value = "" - - return replacement_value - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", - "--creds_file_to_inject", - default="creds.json", - help="creds file to inject into the configuration yaml", - ) - parser.add_argument( - "--other_files_to_inject", - nargs="+", - help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " - "inject into the configuration yaml", - ) - parser.add_argument( - "-c", "--config_file", default="config.yaml", help="configuration yaml" - ) - args = parser.parse_args() - - inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) - set_prod_defaults(args.config_file) - - if args.other_files_to_inject: - inject_other_files_into_fence_config( - args.other_files_to_inject, args.config_file - ) diff --git a/helm/fence/fence-secret/fence_settings.py b/helm/fence/fence-secret/fence_settings.py deleted file mode 100644 index eb2cf818d..000000000 --- a/helm/fence/fence-secret/fence_settings.py +++ /dev/null @@ -1,170 +0,0 @@ -from boto.s3.connection import OrdinaryCallingFormat -import config_helper - -APP_NAME = "fence" - -DB = "postgresql://{{db_username}}:{{db_password}}@{{db_host}}:5432/{{db_database}}" - -MOCK_AUTH = False -MOCK_STORAGE = True - -EMAIL_SERVER = "localhost" - -SEND_FROM = "phillis.tt@gmail.com" - -SEND_TO = "phillis.tt@gmail.com" - -CEPH = { - "aws_access_key_id": "", - "aws_secret_access_key": "", - "host": "", - "port": 443, - "is_secure": True, - "calling_format": OrdinaryCallingFormat(), -} - -AWS = {"aws_access_key_id": "", "aws_secret_access_key": ""} - -HMAC_ENCRYPTION_KEY = "{{hmac_key}}" - - -HOSTNAME = "{{hostname}}" -BASE_URL = "https://{{hostname}}/user" - -OPENID_CONNECT = { - "google": { - "client_id": "{{google_client_id}}", - "client_secret": "{{google_client_secret}}", - "redirect_url": "https://" + HOSTNAME + "/user/login/google/login/", - } -} - -HTTP_PROXY = {"host": "cloud-proxy.internal.io", "port": 3128} - -DEFAULT_DBGAP = { - "sftp": { - "host": "", - "username": "", - "password": "", - "port": 22, - "proxy": "", - "proxy_user": "", - }, - "decrypt_key": "", -} - -STORAGE_CREDENTIALS = {} -# aws_credentials should be a dict looks like: -# { identifier: { 'aws_access_key_id': 'XXX', 'aws_secret_access_key': 'XXX' }} -AWS_CREDENTIALS = {} - -# s3_buckets should be a dict looks like: -# { bucket_name: credential_identifie } -S3_BUCKETS = {} - - -def load_json(file_name): - return config_helper.load_json(file_name, APP_NAME) - - -def get_from_dict(dictionary, key, default=""): - value = dictionary.get(key) - if value is None: - value = default - return value - - -creds = load_json("creds.json") -key_list = ["db_username", "db_password", "db_host", "db_database"] - -DB = "postgresql://%s:%s@%s:5432/%s" % tuple( - [get_from_dict(creds, k, "unknown-" + k) for k in key_list] -) -HMAC_ENCRYPTION_KEY = get_from_dict(creds, "hmac_key", "unknown-hmac_key") -HOSTNAME = get_from_dict(creds, "hostname", "unknown-hostname") -BASE_URL = "https://%s/user" % HOSTNAME - -OPENID_CONNECT["google"]["client_id"] = get_from_dict( - creds, "google_client_id", "unknown-google_client_id" -) -OPENID_CONNECT["google"]["client_secret"] = get_from_dict( - creds, "google_client_secret", "unknown-google_client_secret" -) -OPENID_CONNECT["google"]["redirect_url"] = ( - "https://" + HOSTNAME + "/user/login/google/login/" -) - -GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS = { - "dataflow-service-producer-prod.iam.gserviceaccount.com", - "cloudbuild.gserviceaccount.com", - "cloud-ml.google.com.iam.gserviceaccount.com", - "container-engine-robot.iam.gserviceaccount.com", - "dataflow-service-producer-prod.iam.gserviceaccount.com", - "sourcerepo-service-accounts.iam.gserviceaccount.com", - "dataproc-accounts.iam.gserviceaccount.com", - "gae-api-prod.google.com.iam.gserviceaccount.com", - "genomics-api.google.com.iam.gserviceaccount.com", - "containerregistry.iam.gserviceaccount.com", - "container-analysis.iam.gserviceaccount.com", - "cloudservices.gserviceaccount.com", - "stackdriver-service.iam.gserviceaccount.com", - "appspot.gserviceaccount.com", - "partnercontent.gserviceaccount.com", - "trifacta-gcloud-prod.iam.gserviceaccount.com", - "gcf-admin-robot.iam.gserviceaccount.com", - "compute-system.iam.gserviceaccount.com", - "gcp-sa-websecurityscanner.iam.gserviceaccount.com", - "storage-transfer-service.iam.gserviceaccount.com", -} - -CIRRUS_CFG = {} -data = load_json("fence_credentials.json") -if data: - AWS_CREDENTIALS = data["AWS_CREDENTIALS"] - S3_BUCKETS = data["S3_BUCKETS"] - DEFAULT_LOGIN_URL = data["DEFAULT_LOGIN_URL"] - OPENID_CONNECT.update(data["OPENID_CONNECT"]) - OIDC_ISSUER = data["OIDC_ISSUER"] - ENABLED_IDENTITY_PROVIDERS = data["ENABLED_IDENTITY_PROVIDERS"] - APP_NAME = data["APP_NAME"] - HTTP_PROXY = data["HTTP_PROXY"] - dbGaP = data.get("dbGaP", DEFAULT_DBGAP) - CIRRUS_CFG["GOOGLE_API_KEY"] = get_from_dict(data, "GOOGLE_API_KEY") - CIRRUS_CFG["GOOGLE_PROJECT_ID"] = get_from_dict(data, "GOOGLE_PROJECT_ID") - CIRRUS_CFG["GOOGLE_ADMIN_EMAIL"] = get_from_dict(data, "GOOGLE_ADMIN_EMAIL") - CIRRUS_CFG["GOOGLE_IDENTITY_DOMAIN"] = get_from_dict(data, "GOOGLE_IDENTITY_DOMAIN") - CIRRUS_CFG["GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL"] = get_from_dict( - data, "GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL" - ) - - STORAGE_CREDENTIALS = get_from_dict(data, "STORAGE_CREDENTIALS", {}) - GOOGLE_GROUP_PREFIX = get_from_dict(data, "GOOGLE_GROUP_PREFIX", "gen3") - SUPPORT_EMAIL_FOR_ERRORS = get_from_dict(data, "SUPPORT_EMAIL_FOR_ERRORS", None) - WHITE_LISTED_SERVICE_ACCOUNT_EMAILS = get_from_dict( - data, "WHITE_LISTED_SERVICE_ACCOUNT_EMAILS", [] - ) - WHITE_LISTED_GOOGLE_PARENT_ORGS = get_from_dict( - data, "WHITE_LISTED_GOOGLE_PARENT_ORGS", [] - ) - GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS.update( - data.get("GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS", []) - ) - GUN_MAIL = data.get("GUN_MAIL") - REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION = data.get( - "REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION" - ) - # use for intergration tests to skip the login page - MOCK_GOOGLE_AUTH = data.get("MOCK_GOOGLE_AUTH", False) - -CIRRUS_CFG[ - "GOOGLE_APPLICATION_CREDENTIALS" -] = "/var/www/fence/fence_google_app_creds_secret.json" -CIRRUS_CFG[ - "GOOGLE_STORAGE_CREDS" -] = "/var/www/fence/fence_google_storage_creds_secret.json" - -DEFAULT_LOGIN_URL_REDIRECT_PARAM = "redirect" - -INDEXD = "http://indexd-service/" - -ARBORIST = "http://arborist-service/" diff --git a/helm/fence/templates/fence-creds.yaml b/helm/fence/templates/fence-creds.yaml deleted file mode 100644 index 24cfb7adc..000000000 --- a/helm/fence/templates/fence-creds.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: fence-creds -type: Opaque -stringData: - creds.json: |- - { - "db_host": "{{ include "gen3.service-postgres" (dict "key" "host" "service" $.Chart.Name "context" $) }}", - "db_username": "{{include "gen3.service-postgres" (dict "key" "username" "service" $.Chart.Name "context" $) }}", - "db_password": "{{include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) }}", - "db_database": "{{ include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $)}}", - "hostname": "{{ .Values.global.hostname }}", - "indexd_password": "", - "google_client_secret": "YOUR.GOOGLE.SECRET", - "google_client_id": "YOUR.GOOGLE.CLIENT", - "hmac_key": "" - } - diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml index 6facf6a7f..c6b6c535f 100644 --- a/helm/fence/templates/usersync-cron.yaml +++ b/helm/fence/templates/usersync-cron.yaml @@ -56,9 +56,6 @@ spec: configMap: name: "manifest-fence" optional: true - - name: creds-volume - secret: - secretName: "fence-creds" - name: projects configMap: name: "projects" @@ -210,10 +207,6 @@ spec: readOnly: true mountPath: "/var/www/fence/yaml_merge.py" subPath: yaml_merge.py - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json - name: "projects" mountPath: "/var/www/fence/projects.yaml" subPath: "projects.yaml" diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml index dcea975cd..f29313f19 100644 --- a/helm/fence/values.yaml +++ b/helm/fence/values.yaml @@ -389,9 +389,6 @@ env: # -- (list) Volumes to attach to the container. volumes: - - name: creds-volume - secret: - secretName: "fence-creds" - name: logo-volume configMap: name: "logo-config" @@ -421,10 +418,6 @@ volumes: # -- (list) Volumes to mount to the container. volumeMounts: - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json - name: "logo-volume" readOnly: true mountPath: "/fence/fence/static/img/logo.svg" diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 546aab711..35922402d 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -84,7 +84,7 @@ dependencies: repository: "file://../metadata" condition: metadata.enabled - name: peregrine - version: 0.1.32 + version: 0.1.33 repository: "file://../peregrine" condition: peregrine.enabled - name: portal @@ -100,7 +100,7 @@ dependencies: repository: "file://../revproxy" condition: revproxy.enabled - name: sheepdog - version: 0.1.32 + version: 0.1.33 repository: "file://../sheepdog" condition: sheepdog.enabled - name: ssjdispatcher diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 115c183b5..2a28224dc 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -44,11 +44,11 @@ Helm chart to deploy Gen3 Data Commons | file://../neuvector | neuvector | 0.1.2 | | file://../ohif-viewer | ohif-viewer | 0.1.4 | | file://../orthanc | orthanc | 0.1.5 | -| file://../peregrine | peregrine | 0.1.32 | +| file://../peregrine | peregrine | 0.1.33 | | file://../portal | portal | 0.1.46 | | file://../requestor | requestor | 0.1.25 | | file://../revproxy | revproxy | 0.1.46 | -| file://../sheepdog | sheepdog | 0.1.32 | +| file://../sheepdog | sheepdog | 0.1.33 | | file://../sower | sower | 0.1.35 | | file://../ssjdispatcher | ssjdispatcher | 0.1.33 | | file://../wts | wts | 0.1.31 | diff --git a/helm/peregrine/Chart.yaml b/helm/peregrine/Chart.yaml index 4e986bf98..5d3dfac3c 100644 --- a/helm/peregrine/Chart.yaml +++ b/helm/peregrine/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.32 +version: 0.1.33 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/peregrine/README.md b/helm/peregrine/README.md index 0eec1e215..a263ef46a 100644 --- a/helm/peregrine/README.md +++ b/helm/peregrine/README.md @@ -1,6 +1,6 @@ # peregrine -![Version: 0.1.32](https://img.shields.io/badge/Version-0.1.32-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.33](https://img.shields.io/badge/Version-0.1.33-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Peregrine service diff --git a/helm/peregrine/peregrine-secret/config_helper.py b/helm/peregrine/peregrine-secret/config_helper.py deleted file mode 100644 index 6b303beac..000000000 --- a/helm/peregrine/peregrine-secret/config_helper.py +++ /dev/null @@ -1,376 +0,0 @@ -import json -import os -import copy -import argparse -import re -import types - -# -# make it easy to change this for testing -XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") - - -def default_search_folders(app_name): - """ - Return the list of folders to search for configuration files - """ - return [ - "%s/cdis/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/cdis/%s" % app_name, - "%s/gen3/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/gen3/%s" % app_name, - "/var/www/%s" % app_name, - "/etc/gen3/%s" % app_name, - ] - - -def find_paths(file_name, app_name, search_folders=None): - """ - Search the given folders for file_name - search_folders defaults to default_search_folders if not specified - return the first path to file_name found - """ - search_folders = search_folders or default_search_folders(app_name) - possible_files = [os.path.join(folder, file_name) for folder in search_folders] - return [path for path in possible_files if os.path.exists(path)] - - -def load_json(file_name, app_name, search_folders=None): - """ - json.load(file_name) after finding file_name in search_folders - - return the loaded json data or None if file not found - """ - actual_files = find_paths(file_name, app_name, search_folders) - if not actual_files: - return None - with open(actual_files[0], "r") as reader: - return json.load(reader) - - -def inject_creds_into_fence_config(creds_file_path, config_file_path): - creds_file = open(creds_file_path, "r") - creds = json.load(creds_file) - creds_file.close() - - # get secret values from creds.json file - db_host = _get_nested_value(creds, "db_host") - db_username = _get_nested_value(creds, "db_username") - db_password = _get_nested_value(creds, "db_password") - db_database = _get_nested_value(creds, "db_database") - hostname = _get_nested_value(creds, "hostname") - indexd_password = _get_nested_value(creds, "indexd_password") - google_client_secret = _get_nested_value(creds, "google_client_secret") - google_client_id = _get_nested_value(creds, "google_client_id") - hmac_key = _get_nested_value(creds, "hmac_key") - db_path = "postgresql://{}:{}@{}:5432/{}".format( - db_username, db_password, db_host, db_database - ) - - config_file = open(config_file_path, "r").read() - - print(" DB injected with value(s) from creds.json") - config_file = _replace(config_file, "DB", db_path) - - print(" BASE_URL injected with value(s) from creds.json") - config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - - print(" ENCRYPTION_KEY injected with value(s) from creds.json") - config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) - - print( - " OPENID_CONNECT/google/client_secret injected with value(s) " - "from creds.json" - ) - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_secret", google_client_secret - ) - - print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_id", google_client_id - ) - - open(config_file_path, "w+").write(config_file) - - -def set_prod_defaults(config_file_path): - config_file = open(config_file_path, "r").read() - - print( - " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " - "var/www/fence/fence_google_app_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", - "/var/www/fence/fence_google_app_creds_secret.json", - ) - - print( - " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " - "var/www/fence/fence_google_storage_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", - "/var/www/fence/fence_google_storage_creds_secret.json", - ) - - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - - print(" ARBORIST set as http://arborist-service/") - config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") - - print(" HTTP_PROXY/host set as cloud-proxy.internal.io") - config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") - - print(" HTTP_PROXY/port set as 3128") - config_file = _replace(config_file, "HTTP_PROXY/port", 3128) - - print(" DEBUG set to false") - config_file = _replace(config_file, "DEBUG", False) - - print(" MOCK_AUTH set to false") - config_file = _replace(config_file, "MOCK_AUTH", False) - - print(" MOCK_GOOGLE_AUTH set to false") - config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) - - print(" AUTHLIB_INSECURE_TRANSPORT set to true") - config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) - - print(" SESSION_COOKIE_SECURE set to true") - config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) - - print(" ENABLE_CSRF_PROTECTION set to true") - config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) - - open(config_file_path, "w+").write(config_file) - - -def inject_other_files_into_fence_config(other_files, config_file_path): - additional_cfgs = _get_all_additional_configs(other_files) - - config_file = open(config_file_path, "r").read() - - for key, value in additional_cfgs.iteritems(): - print(" {} set to {}".format(key, value)) - config_file = _nested_replace(config_file, key, value) - - open(config_file_path, "w+").write(config_file) - - -def _get_all_additional_configs(other_files): - """ - Attempt to parse given list of files and extract configuration variables and values - """ - additional_configs = dict() - for file_path in other_files: - try: - file_ext = file_path.strip().split(".")[-1] - if file_ext == "json": - json_file = open(file_path, "r") - configs = json.load(json_file) - json_file.close() - elif file_ext == "py": - configs = from_pyfile(file_path) - else: - print( - "Cannot load config vars from a file with extention: {}".format( - file_ext - ) - ) - except Exception as exc: - # if there's any issue reading the file, exit - print( - "Error reading {}. Cannot get configuration. Skipping this file. " - "Details: {}".format(other_files, str(exc)) - ) - continue - - if configs: - additional_configs.update(configs) - - return additional_configs - - -def _nested_replace(config_file, key, value, replacement_path=None): - replacement_path = replacement_path or key - try: - for inner_key, inner_value in value.iteritems(): - temp_path = replacement_path - temp_path = temp_path + "/" + inner_key - config_file = _nested_replace( - config_file, inner_key, inner_value, temp_path - ) - except AttributeError: - # not a dict so replace - if value is not None: - config_file = _replace(config_file, replacement_path, value) - - return config_file - - -def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): - """ - Replace a nested value in a YAML file string with the given value without - losing comments. Uses a regex to do the replacement. - - Args: - yaml_config (str): a string representing a full configuration file - path_to_key (str): nested/path/to/key. The value of this key will be - replaced - replacement_value (str): Replacement value for the key from - path_to_key - """ - nested_path_to_replace = path_to_key.split("/") - - # our regex looks for a specific number of spaces to ensure correct - # level of nesting. It matches to the end of the line - search_string = ( - " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" - ) - matches = re.search(search_string, yaml_config[start:]) - - # early return if we haven't found anything - if not matches: - return yaml_config - - # if we're on the last item in the path, we need to get the value and - # replace it in the original file - if len(nested_path_to_replace) == 1: - # replace the current key:value with the new replacement value - match_start = start + matches.start(0) + len(" " * nested_level) - match_end = start + matches.end(0) - yaml_config = ( - yaml_config[:match_start] - + "{}: {}\n".format( - nested_path_to_replace[0], - _get_yaml_replacement_value(replacement_value, nested_level), - ) - + yaml_config[match_end:] - ) - - return yaml_config - - # set new start point to past current match and move on to next match - start = matches.end(0) - nested_level += 1 - del nested_path_to_replace[0] - - return _replace( - yaml_config, - "/".join(nested_path_to_replace), - replacement_value, - start, - nested_level, - ) - - -def from_pyfile(filename, silent=False): - """ - Modeled after flask's ability to load in python files: - https://github.com/pallets/flask/blob/master/flask/config.py - - Some alterations were made but logic is essentially the same - """ - filename = os.path.abspath(filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except IOError as e: - print("Unable to load configuration file ({})".format(e.strerror)) - if silent: - return False - raise - return _from_object(d) - - -def _from_object(obj): - configs = {} - for key in dir(obj): - if key.isupper(): - configs[key] = getattr(obj, key) - return configs - - -def _get_yaml_replacement_value(value, nested_level=0): - if isinstance(value, str): - return "'" + value + "'" - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, list) or isinstance(value, set): - output = "" - for item in value: - # spaces for nested level then spaces and hyphen for each list item - output += ( - "\n" - + " " * nested_level - + " - " - + _get_yaml_replacement_value(item) - + "" - ) - return output - else: - return value - - -def _get_nested_value(dictionary, nested_path): - """ - Return a value from a dictionary given a path-like nesting of keys. - - Will default to an empty string if value cannot be found. - - Args: - dictionary (dict): a dictionary - nested_path (str): nested/path/to/key - - Returns: - ?: Value from dict - """ - replacement_value_path = nested_path.split("/") - replacement_value = copy.deepcopy(dictionary) - - for item in replacement_value_path: - replacement_value = replacement_value.get(item, {}) - - if replacement_value == {}: - replacement_value = "" - - return replacement_value - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", - "--creds_file_to_inject", - default="creds.json", - help="creds file to inject into the configuration yaml", - ) - parser.add_argument( - "--other_files_to_inject", - nargs="+", - help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " - "inject into the configuration yaml", - ) - parser.add_argument( - "-c", "--config_file", default="config.yaml", help="configuration yaml" - ) - args = parser.parse_args() - - inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) - set_prod_defaults(args.config_file) - - if args.other_files_to_inject: - inject_other_files_into_fence_config( - args.other_files_to_inject, args.config_file - ) diff --git a/helm/peregrine/peregrine-secret/settings.py b/helm/peregrine/peregrine-secret/settings.py index 1a623a907..125b3f63b 100644 --- a/helm/peregrine/peregrine-secret/settings.py +++ b/helm/peregrine/peregrine-secret/settings.py @@ -5,13 +5,8 @@ from peregrine.api import app, app_init from os import environ -# import config_helper APP_NAME='peregrine' -# def load_json(file_name): -# return config_helper.load_json(file_name, APP_NAME) - -# conf_data = load_json('creds.json') config = app.config # config["AUTH"] = 'https://auth.service.consul:5000/v3/' diff --git a/helm/sheepdog/Chart.yaml b/helm/sheepdog/Chart.yaml index 135d822f0..fedf6f7aa 100644 --- a/helm/sheepdog/Chart.yaml +++ b/helm/sheepdog/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.32 +version: 0.1.33 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/sheepdog/README.md b/helm/sheepdog/README.md index c6c58fd55..c87f08922 100644 --- a/helm/sheepdog/README.md +++ b/helm/sheepdog/README.md @@ -1,6 +1,6 @@ # sheepdog -![Version: 0.1.32](https://img.shields.io/badge/Version-0.1.32-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.33](https://img.shields.io/badge/Version-0.1.33-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Sheepdog Service diff --git a/helm/sheepdog/sheepdog-secret/config_helper.py b/helm/sheepdog/sheepdog-secret/config_helper.py deleted file mode 100644 index ab1805496..000000000 --- a/helm/sheepdog/sheepdog-secret/config_helper.py +++ /dev/null @@ -1,376 +0,0 @@ -import json -import os -import copy -import argparse -import re -import types - -# -# make it easy to change this for testing -XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") - - -def default_search_folders(app_name): - """ - Return the list of folders to search for configuration files - """ - return [ - "%s/cdis/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/cdis/%s" % app_name, - "%s/gen3/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/gen3/%s" % app_name, - "/var/www/%s" % app_name, - "/etc/gen3/%s" % app_name, - ] - - -def find_paths(file_name, app_name, search_folders=None): - """ - Search the given folders for file_name - search_folders defaults to default_search_folders if not specified - return the first path to file_name found - """ - search_folders = search_folders or default_search_folders(app_name) - possible_files = [os.path.join(folder, file_name) for folder in search_folders] - return [path for path in possible_files if os.path.exists(path)] - - -def load_json(file_name, app_name, search_folders=None): - """ - json.load(file_name) after finding file_name in search_folders - - return the loaded json data or None if file not found - """ - actual_files = find_paths(file_name, app_name, search_folders) - if not actual_files: - return None - with open(actual_files[0], "r") as reader: - return json.load(reader) - - -def inject_creds_into_fence_config(creds_file_path, config_file_path): - creds_file = open(creds_file_path, "r") - creds = json.load(creds_file) - creds_file.close() - - # get secret values from creds.json file - db_host = _get_nested_value(creds, "db_host") - db_username = _get_nested_value(creds, "db_username") - db_password = _get_nested_value(creds, "db_password") - db_database = _get_nested_value(creds, "db_database") - hostname = _get_nested_value(creds, "hostname") - indexd_password = environ.get('INDEXD_PASS') - google_client_secret = _get_nested_value(creds, "google_client_secret") - google_client_id = _get_nested_value(creds, "google_client_id") - hmac_key = _get_nested_value(creds, "hmac_key") - db_path = "postgresql://{}:{}@{}:5432/{}".format( - db_username, db_password, db_host, db_database - ) - - config_file = open(config_file_path, "r").read() - - print(" DB injected with value(s) from creds.json") - config_file = _replace(config_file, "DB", db_path) - - print(" BASE_URL injected with value(s) from creds.json") - config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - - print(" ENCRYPTION_KEY injected with value(s) from creds.json") - config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) - - print( - " OPENID_CONNECT/google/client_secret injected with value(s) " - "from creds.json" - ) - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_secret", google_client_secret - ) - - print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_id", google_client_id - ) - - open(config_file_path, "w+").write(config_file) - - -def set_prod_defaults(config_file_path): - config_file = open(config_file_path, "r").read() - - print( - " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " - "var/www/fence/fence_google_app_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", - "/var/www/fence/fence_google_app_creds_secret.json", - ) - - print( - " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " - "var/www/fence/fence_google_storage_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", - "/var/www/fence/fence_google_storage_creds_secret.json", - ) - - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - - print(" ARBORIST set as http://arborist-service/") - config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") - - print(" HTTP_PROXY/host set as cloud-proxy.internal.io") - config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") - - print(" HTTP_PROXY/port set as 3128") - config_file = _replace(config_file, "HTTP_PROXY/port", 3128) - - print(" DEBUG set to false") - config_file = _replace(config_file, "DEBUG", False) - - print(" MOCK_AUTH set to false") - config_file = _replace(config_file, "MOCK_AUTH", False) - - print(" MOCK_GOOGLE_AUTH set to false") - config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) - - print(" AUTHLIB_INSECURE_TRANSPORT set to true") - config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) - - print(" SESSION_COOKIE_SECURE set to true") - config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) - - print(" ENABLE_CSRF_PROTECTION set to true") - config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) - - open(config_file_path, "w+").write(config_file) - - -def inject_other_files_into_fence_config(other_files, config_file_path): - additional_cfgs = _get_all_additional_configs(other_files) - - config_file = open(config_file_path, "r").read() - - for key, value in additional_cfgs.iteritems(): - print(" {} set to {}".format(key, value)) - config_file = _nested_replace(config_file, key, value) - - open(config_file_path, "w+").write(config_file) - - -def _get_all_additional_configs(other_files): - """ - Attempt to parse given list of files and extract configuration variables and values - """ - additional_configs = dict() - for file_path in other_files: - try: - file_ext = file_path.strip().split(".")[-1] - if file_ext == "json": - json_file = open(file_path, "r") - configs = json.load(json_file) - json_file.close() - elif file_ext == "py": - configs = from_pyfile(file_path) - else: - print( - "Cannot load config vars from a file with extention: {}".format( - file_ext - ) - ) - except Exception as exc: - # if there's any issue reading the file, exit - print( - "Error reading {}. Cannot get configuration. Skipping this file. " - "Details: {}".format(other_files, str(exc)) - ) - continue - - if configs: - additional_configs.update(configs) - - return additional_configs - - -def _nested_replace(config_file, key, value, replacement_path=None): - replacement_path = replacement_path or key - try: - for inner_key, inner_value in value.iteritems(): - temp_path = replacement_path - temp_path = temp_path + "/" + inner_key - config_file = _nested_replace( - config_file, inner_key, inner_value, temp_path - ) - except AttributeError: - # not a dict so replace - if value is not None: - config_file = _replace(config_file, replacement_path, value) - - return config_file - - -def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): - """ - Replace a nested value in a YAML file string with the given value without - losing comments. Uses a regex to do the replacement. - - Args: - yaml_config (str): a string representing a full configuration file - path_to_key (str): nested/path/to/key. The value of this key will be - replaced - replacement_value (str): Replacement value for the key from - path_to_key - """ - nested_path_to_replace = path_to_key.split("/") - - # our regex looks for a specific number of spaces to ensure correct - # level of nesting. It matches to the end of the line - search_string = ( - " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" - ) - matches = re.search(search_string, yaml_config[start:]) - - # early return if we haven't found anything - if not matches: - return yaml_config - - # if we're on the last item in the path, we need to get the value and - # replace it in the original file - if len(nested_path_to_replace) == 1: - # replace the current key:value with the new replacement value - match_start = start + matches.start(0) + len(" " * nested_level) - match_end = start + matches.end(0) - yaml_config = ( - yaml_config[:match_start] - + "{}: {}\n".format( - nested_path_to_replace[0], - _get_yaml_replacement_value(replacement_value, nested_level), - ) - + yaml_config[match_end:] - ) - - return yaml_config - - # set new start point to past current match and move on to next match - start = matches.end(0) - nested_level += 1 - del nested_path_to_replace[0] - - return _replace( - yaml_config, - "/".join(nested_path_to_replace), - replacement_value, - start, - nested_level, - ) - - -def from_pyfile(filename, silent=False): - """ - Modeled after flask's ability to load in python files: - https://github.com/pallets/flask/blob/master/flask/config.py - - Some alterations were made but logic is essentially the same - """ - filename = os.path.abspath(filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except IOError as e: - print("Unable to load configuration file ({})".format(e.strerror)) - if silent: - return False - raise - return _from_object(d) - - -def _from_object(obj): - configs = {} - for key in dir(obj): - if key.isupper(): - configs[key] = getattr(obj, key) - return configs - - -def _get_yaml_replacement_value(value, nested_level=0): - if isinstance(value, str): - return "'" + value + "'" - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, list) or isinstance(value, set): - output = "" - for item in value: - # spaces for nested level then spaces and hyphen for each list item - output += ( - "\n" - + " " * nested_level - + " - " - + _get_yaml_replacement_value(item) - + "" - ) - return output - else: - return value - - -def _get_nested_value(dictionary, nested_path): - """ - Return a value from a dictionary given a path-like nesting of keys. - - Will default to an empty string if value cannot be found. - - Args: - dictionary (dict): a dictionary - nested_path (str): nested/path/to/key - - Returns: - ?: Value from dict - """ - replacement_value_path = nested_path.split("/") - replacement_value = copy.deepcopy(dictionary) - - for item in replacement_value_path: - replacement_value = replacement_value.get(item, {}) - - if replacement_value == {}: - replacement_value = "" - - return replacement_value - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", - "--creds_file_to_inject", - default="creds.json", - help="creds file to inject into the configuration yaml", - ) - parser.add_argument( - "--other_files_to_inject", - nargs="+", - help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " - "inject into the configuration yaml", - ) - parser.add_argument( - "-c", "--config_file", default="config.yaml", help="configuration yaml" - ) - args = parser.parse_args() - - inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) - set_prod_defaults(args.config_file) - - if args.other_files_to_inject: - inject_other_files_into_fence_config( - args.other_files_to_inject, args.config_file - ) \ No newline at end of file diff --git a/helm/sheepdog/sheepdog-secret/settings.py b/helm/sheepdog/sheepdog-secret/settings.py index ac896e523..6c76f6d42 100644 --- a/helm/sheepdog/sheepdog-secret/settings.py +++ b/helm/sheepdog/sheepdog-secret/settings.py @@ -5,13 +5,8 @@ from sheepdog.api import app, app_init from os import environ -# import config_helper APP_NAME='sheepdog' -# def load_json(file_name): -# return config_helper.load_json(file_name, APP_NAME) - -# conf_data = load_json('creds.json') config = app.config diff --git a/wip/gen3-test-data-job/templates/_jobs.tpl b/wip/gen3-test-data-job/templates/_jobs.tpl index 260a2ff47..155dbbb66 100644 --- a/wip/gen3-test-data-job/templates/_jobs.tpl +++ b/wip/gen3-test-data-job/templates/_jobs.tpl @@ -21,14 +21,6 @@ spec: name: "fence-yaml-merge" - name: shared-data emptyDir: {} -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: creds-volume - secret: - secretName: "fence-creds" -# ----------------------------------------------------------------------------- - name: config-volume secret: secretName: "fence-config" @@ -100,15 +92,6 @@ spec: key: fence-config-public.yaml optional: true volumeMounts: -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json -# ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true mountPath: "/var/www/fence/fence-config-secret.yaml" diff --git a/wip/gen3-test-data-job/templates/jobs.yaml b/wip/gen3-test-data-job/templates/jobs.yaml index 47318f449..4dd11a497 100644 --- a/wip/gen3-test-data-job/templates/jobs.yaml +++ b/wip/gen3-test-data-job/templates/jobs.yaml @@ -16,14 +16,6 @@ spec: name: "fence-yaml-merge" - name: shared-data emptyDir: {} -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: creds-volume - secret: - secretName: "fence-creds" -# ----------------------------------------------------------------------------- - name: config-volume secret: secretName: "fence-config" @@ -95,16 +87,6 @@ spec: key: fence-config-public.yaml optional: true volumeMounts: -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json -# ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true mountPath: "/var/www/fence/fence-config-secret.yaml" From 599aab047c4353b1db91252e8c4302142df9841b Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Wed, 19 Nov 2025 09:43:04 -0600 Subject: [PATCH 3/5] Update common chart version --- helm/common/Chart.yaml | 2 +- helm/common/README.md | 2 +- helm/gen3/Chart.yaml | 2 +- helm/gen3/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/common/Chart.yaml b/helm/common/Chart.yaml index 3452d7491..cdc7a1269 100644 --- a/helm/common/Chart.yaml +++ b/helm/common/Chart.yaml @@ -15,7 +15,7 @@ type: library # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.28 +version: 0.1.29 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/common/README.md b/helm/common/README.md index e35999e1d..fb4ccab0a 100644 --- a/helm/common/README.md +++ b/helm/common/README.md @@ -1,6 +1,6 @@ # common -![Version: 0.1.28](https://img.shields.io/badge/Version-0.1.28-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.29](https://img.shields.io/badge/Version-0.1.29-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for provisioning databases in gen3 diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index c86958786..5fe985fd1 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -37,7 +37,7 @@ dependencies: repository: "file://../cohort-middleware" condition: cohort-middleware.enabled - name: common - version: 0.1.28 + version: 0.1.29 repository: file://../common - name: dashboard version: 0.1.12 diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 6859c4446..476320572 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -26,7 +26,7 @@ Helm chart to deploy Gen3 Data Commons | file://../aws-es-proxy | aws-es-proxy | 0.1.34 | | file://../cedar | cedar | 0.1.17 | | file://../cohort-middleware | cohort-middleware | 0.1.15 | -| file://../common | common | 0.1.28 | +| file://../common | common | 0.1.29 | | file://../dashboard | dashboard | 0.1.12 | | file://../datareplicate | datareplicate | 0.0.36 | | file://../dicom-server | dicom-server | 0.1.22 | From d796d0343acb34f66feab49f2853c687688cca22 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Wed, 19 Nov 2025 09:55:35 -0600 Subject: [PATCH 4/5] Update chart versions to fix lint errors --- helm/fence/Chart.yaml | 2 +- helm/fence/README.md | 2 +- helm/peregrine/Chart.yaml | 2 +- helm/peregrine/README.md | 2 +- helm/sheepdog/Chart.yaml | 2 +- helm/sheepdog/README.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index 5f0c7e7f0..faa2adb81 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -24,7 +24,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.26 + version: 0.1.28 repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/fence/README.md b/helm/fence/README.md index 7321bf8f7..ca466858f 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -8,7 +8,7 @@ A Helm chart for gen3 Fence | Repository | Name | Version | |------------|------|---------| -| file://../common | common | 0.1.26 | +| file://../common | common | 0.1.28 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | ## Values diff --git a/helm/peregrine/Chart.yaml b/helm/peregrine/Chart.yaml index 568297d56..a339ced3b 100644 --- a/helm/peregrine/Chart.yaml +++ b/helm/peregrine/Chart.yaml @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.26 + version: 0.1.28 repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/peregrine/README.md b/helm/peregrine/README.md index e1c07ea0b..fbedf7a78 100644 --- a/helm/peregrine/README.md +++ b/helm/peregrine/README.md @@ -8,7 +8,7 @@ A Helm chart for gen3 Peregrine service | Repository | Name | Version | |------------|------|---------| -| file://../common | common | 0.1.26 | +| file://../common | common | 0.1.28 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | ## Values diff --git a/helm/sheepdog/Chart.yaml b/helm/sheepdog/Chart.yaml index 93c3d4757..4b8988d58 100644 --- a/helm/sheepdog/Chart.yaml +++ b/helm/sheepdog/Chart.yaml @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.26 + version: 0.1.28 repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/sheepdog/README.md b/helm/sheepdog/README.md index 219b6f05f..3f298b2a4 100644 --- a/helm/sheepdog/README.md +++ b/helm/sheepdog/README.md @@ -8,7 +8,7 @@ A Helm chart for gen3 Sheepdog Service | Repository | Name | Version | |------------|------|---------| -| file://../common | common | 0.1.26 | +| file://../common | common | 0.1.28 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | ## Values From 7e8d954d251f9de1182404559c75a19b2ff03fd3 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Wed, 19 Nov 2025 10:01:44 -0600 Subject: [PATCH 5/5] Update versions to fix lint errors --- helm/common/Chart.yaml | 2 +- helm/common/README.md | 2 +- helm/fence/Chart.yaml | 2 +- helm/fence/README.md | 2 +- helm/gen3/Chart.yaml | 2 +- helm/gen3/README.md | 2 +- helm/peregrine/Chart.yaml | 2 +- helm/peregrine/README.md | 2 +- helm/sheepdog/Chart.yaml | 2 +- helm/sheepdog/README.md | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/helm/common/Chart.yaml b/helm/common/Chart.yaml index cdc7a1269..7c9b67272 100644 --- a/helm/common/Chart.yaml +++ b/helm/common/Chart.yaml @@ -15,7 +15,7 @@ type: library # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.29 +version: 0.1.30 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/common/README.md b/helm/common/README.md index fb4ccab0a..a2a605853 100644 --- a/helm/common/README.md +++ b/helm/common/README.md @@ -1,6 +1,6 @@ # common -![Version: 0.1.29](https://img.shields.io/badge/Version-0.1.29-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.30](https://img.shields.io/badge/Version-0.1.30-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for provisioning databases in gen3 diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index faa2adb81..bacd44dea 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -24,7 +24,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.28 + version: 0.1.30 repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/fence/README.md b/helm/fence/README.md index ca466858f..ed5b224bb 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -8,7 +8,7 @@ A Helm chart for gen3 Fence | Repository | Name | Version | |------------|------|---------| -| file://../common | common | 0.1.28 | +| file://../common | common | 0.1.30 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | ## Values diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 5fe985fd1..82587deab 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -37,7 +37,7 @@ dependencies: repository: "file://../cohort-middleware" condition: cohort-middleware.enabled - name: common - version: 0.1.29 + version: 0.1.30 repository: file://../common - name: dashboard version: 0.1.12 diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 476320572..58040ab76 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -26,7 +26,7 @@ Helm chart to deploy Gen3 Data Commons | file://../aws-es-proxy | aws-es-proxy | 0.1.34 | | file://../cedar | cedar | 0.1.17 | | file://../cohort-middleware | cohort-middleware | 0.1.15 | -| file://../common | common | 0.1.29 | +| file://../common | common | 0.1.30 | | file://../dashboard | dashboard | 0.1.12 | | file://../datareplicate | datareplicate | 0.0.36 | | file://../dicom-server | dicom-server | 0.1.22 | diff --git a/helm/peregrine/Chart.yaml b/helm/peregrine/Chart.yaml index a339ced3b..e89354bb1 100644 --- a/helm/peregrine/Chart.yaml +++ b/helm/peregrine/Chart.yaml @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.28 + version: 0.1.30 repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/peregrine/README.md b/helm/peregrine/README.md index fbedf7a78..67e5465a1 100644 --- a/helm/peregrine/README.md +++ b/helm/peregrine/README.md @@ -8,7 +8,7 @@ A Helm chart for gen3 Peregrine service | Repository | Name | Version | |------------|------|---------| -| file://../common | common | 0.1.28 | +| file://../common | common | 0.1.30 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | ## Values diff --git a/helm/sheepdog/Chart.yaml b/helm/sheepdog/Chart.yaml index 4b8988d58..2c9939e32 100644 --- a/helm/sheepdog/Chart.yaml +++ b/helm/sheepdog/Chart.yaml @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.28 + version: 0.1.30 repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/sheepdog/README.md b/helm/sheepdog/README.md index 3f298b2a4..23152cf26 100644 --- a/helm/sheepdog/README.md +++ b/helm/sheepdog/README.md @@ -8,7 +8,7 @@ A Helm chart for gen3 Sheepdog Service | Repository | Name | Version | |------------|------|---------| -| file://../common | common | 0.1.28 | +| file://../common | common | 0.1.30 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | ## Values