diff --git a/helm/common/Chart.yaml b/helm/common/Chart.yaml index 7a0f8b571..3452d7491 100644 --- a/helm/common/Chart.yaml +++ b/helm/common/Chart.yaml @@ -15,7 +15,7 @@ type: library # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.27 +version: 0.1.28 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/common/README.md b/helm/common/README.md index 7c09e3e7d..e35999e1d 100644 --- a/helm/common/README.md +++ b/helm/common/README.md @@ -1,6 +1,6 @@ # common -![Version: 0.1.27](https://img.shields.io/badge/Version-0.1.27-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.28](https://img.shields.io/badge/Version-0.1.28-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for provisioning databases in gen3 diff --git a/helm/common/templates/_generate_test_data.tpl b/helm/common/templates/_generate_test_data.tpl index 2a2760efe..f5bcc7a6c 100644 --- a/helm/common/templates/_generate_test_data.tpl +++ b/helm/common/templates/_generate_test_data.tpl @@ -21,23 +21,6 @@ spec: name: "fence-yaml-merge" - name: shared-data emptyDir: {} -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: old-config-volume - secret: - secretName: "fence-secret" - - name: creds-volume - secret: - secretName: "fence-creds" - - name: config-helper - configMap: - name: config-helper - - name: json-secret-volume - secret: - secretName: "fence-json-secret" -# ----------------------------------------------------------------------------- - name: config-volume secret: secretName: "fence-config" @@ -109,27 +92,6 @@ spec: key: fence-config-public.yaml optional: true volumeMounts: -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: "old-config-volume" - readOnly: true - mountPath: "/var/www/fence/local_settings.py" - subPath: local_settings.py - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json - - name: "config-helper" - readOnly: true - mountPath: "/var/www/fence/config_helper.py" - subPath: config_helper.py - - name: "json-secret-volume" - readOnly: true - mountPath: "/var/www/fence/fence_credentials.json" - subPath: fence_credentials.json -# ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true mountPath: "/var/www/fence/fence-config-secret.yaml" diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index b38b118a2..208754a7a 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.64 +version: 0.1.65 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -24,7 +24,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.26 + version: 0.1.27 repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/fence/README.md b/helm/fence/README.md index 9e2c721c7..8f9a5a813 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -1,6 +1,6 @@ # fence -![Version: 0.1.64](https://img.shields.io/badge/Version-0.1.64-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.65](https://img.shields.io/badge/Version-0.1.65-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Fence @@ -8,7 +8,7 @@ A Helm chart for gen3 Fence | Repository | Name | Version | |------------|------|---------| -| file://../common | common | 0.1.26 | +| file://../common | common | 0.1.27 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | ## Values @@ -218,5 +218,5 @@ A Helm chart for gen3 Fence | usersync.syncFromDbgap | bool | `false` | Whether to sync data from dbGaP. | | usersync.userYamlS3Path | string | `"s3://cdis-gen3-users/helm-test/user.yaml"` | Path to the user.yaml file in S3. | | usersync.usersync | bool | `false` | Whether to run Fence usersync or not. | -| volumeMounts | list | `[{"mountPath":"/var/www/fence/local_settings.py","name":"old-config-volume","readOnly":true,"subPath":"local_settings.py"},{"mountPath":"/var/www/fence/fence_credentials.json","name":"json-secret-volume","readOnly":true,"subPath":"fence_credentials.json"},{"mountPath":"/var/www/fence/creds.json","name":"creds-volume","readOnly":true,"subPath":"creds.json"},{"mountPath":"/var/www/fence/config_helper.py","name":"config-helper","readOnly":true,"subPath":"config_helper.py"},{"mountPath":"/fence/fence/static/img/logo.svg","name":"logo-volume","readOnly":true,"subPath":"logo.svg"},{"mountPath":"/fence/fence/static/privacy_policy.md","name":"privacy-policy","readOnly":true,"subPath":"privacy_policy.md"},{"mountPath":"/var/www/fence/fence-config-secret.yaml","name":"config-volume","readOnly":true,"subPath":"fence-config.yaml"},{"mountPath":"/var/www/fence/yaml_merge.py","name":"yaml-merge","readOnly":true,"subPath":"yaml_merge.py"},{"mountPath":"/var/www/fence/fence_google_app_creds_secret.json","name":"fence-google-app-creds-secret-volume","readOnly":true,"subPath":"fence_google_app_creds_secret.json"},{"mountPath":"/var/www/fence/fence_google_storage_creds_secret.json","name":"fence-google-storage-creds-secret-volume","readOnly":true,"subPath":"fence_google_storage_creds_secret.json"},{"mountPath":"/fence/keys/key/jwt_private_key.pem","name":"fence-jwt-keys","readOnly":true,"subPath":"jwt_private_key.pem"},{"mountPath":"/var/www/fence/fence-config-public.yaml","name":"config-volume-public","readOnly":true,"subPath":"fence-config-public.yaml"}]` | Volumes to mount to the container. | -| volumes | list | `[{"name":"old-config-volume","secret":{"secretName":"fence-secret"}},{"name":"json-secret-volume","secret":{"optional":true,"secretName":"fence-json-secret"}},{"name":"creds-volume","secret":{"secretName":"fence-creds"}},{"configMap":{"name":"config-helper","optional":true},"name":"config-helper"},{"configMap":{"name":"logo-config"},"name":"logo-volume"},{"name":"config-volume","secret":{"secretName":"fence-config"}},{"name":"fence-google-app-creds-secret-volume","secret":{"secretName":"fence-google-app-creds-secret"}},{"name":"fence-google-storage-creds-secret-volume","secret":{"secretName":"fence-google-storage-creds-secret"}},{"name":"fence-jwt-keys","secret":{"secretName":"fence-jwt-keys"}},{"configMap":{"name":"privacy-policy"},"name":"privacy-policy"},{"configMap":{"name":"fence-yaml-merge","optional":false},"name":"yaml-merge"},{"configMap":{"name":"manifest-fence","optional":true},"name":"config-volume-public"}]` | Volumes to attach to the container. | +| volumeMounts | list | `[{"mountPath":"/fence/fence/static/img/logo.svg","name":"logo-volume","readOnly":true,"subPath":"logo.svg"},{"mountPath":"/fence/fence/static/privacy_policy.md","name":"privacy-policy","readOnly":true,"subPath":"privacy_policy.md"},{"mountPath":"/var/www/fence/fence-config-secret.yaml","name":"config-volume","readOnly":true,"subPath":"fence-config.yaml"},{"mountPath":"/var/www/fence/yaml_merge.py","name":"yaml-merge","readOnly":true,"subPath":"yaml_merge.py"},{"mountPath":"/var/www/fence/fence_google_app_creds_secret.json","name":"fence-google-app-creds-secret-volume","readOnly":true,"subPath":"fence_google_app_creds_secret.json"},{"mountPath":"/var/www/fence/fence_google_storage_creds_secret.json","name":"fence-google-storage-creds-secret-volume","readOnly":true,"subPath":"fence_google_storage_creds_secret.json"},{"mountPath":"/fence/keys/key/jwt_private_key.pem","name":"fence-jwt-keys","readOnly":true,"subPath":"jwt_private_key.pem"},{"mountPath":"/var/www/fence/fence-config-public.yaml","name":"config-volume-public","readOnly":true,"subPath":"fence-config-public.yaml"}]` | Volumes to mount to the container. | +| volumes | list | `[{"configMap":{"name":"logo-config"},"name":"logo-volume"},{"name":"config-volume","secret":{"secretName":"fence-config"}},{"name":"fence-google-app-creds-secret-volume","secret":{"secretName":"fence-google-app-creds-secret"}},{"name":"fence-google-storage-creds-secret-volume","secret":{"secretName":"fence-google-storage-creds-secret"}},{"name":"fence-jwt-keys","secret":{"secretName":"fence-jwt-keys"}},{"configMap":{"name":"privacy-policy"},"name":"privacy-policy"},{"configMap":{"name":"fence-yaml-merge","optional":false},"name":"yaml-merge"},{"configMap":{"name":"manifest-fence","optional":true},"name":"config-volume-public"}]` | Volumes to attach to the container. | diff --git a/helm/fence/fence-secret/config_helper.py b/helm/fence/fence-secret/config_helper.py deleted file mode 100644 index 6b303beac..000000000 --- a/helm/fence/fence-secret/config_helper.py +++ /dev/null @@ -1,376 +0,0 @@ -import json -import os -import copy -import argparse -import re -import types - -# -# make it easy to change this for testing -XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") - - -def default_search_folders(app_name): - """ - Return the list of folders to search for configuration files - """ - return [ - "%s/cdis/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/cdis/%s" % app_name, - "%s/gen3/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/gen3/%s" % app_name, - "/var/www/%s" % app_name, - "/etc/gen3/%s" % app_name, - ] - - -def find_paths(file_name, app_name, search_folders=None): - """ - Search the given folders for file_name - search_folders defaults to default_search_folders if not specified - return the first path to file_name found - """ - search_folders = search_folders or default_search_folders(app_name) - possible_files = [os.path.join(folder, file_name) for folder in search_folders] - return [path for path in possible_files if os.path.exists(path)] - - -def load_json(file_name, app_name, search_folders=None): - """ - json.load(file_name) after finding file_name in search_folders - - return the loaded json data or None if file not found - """ - actual_files = find_paths(file_name, app_name, search_folders) - if not actual_files: - return None - with open(actual_files[0], "r") as reader: - return json.load(reader) - - -def inject_creds_into_fence_config(creds_file_path, config_file_path): - creds_file = open(creds_file_path, "r") - creds = json.load(creds_file) - creds_file.close() - - # get secret values from creds.json file - db_host = _get_nested_value(creds, "db_host") - db_username = _get_nested_value(creds, "db_username") - db_password = _get_nested_value(creds, "db_password") - db_database = _get_nested_value(creds, "db_database") - hostname = _get_nested_value(creds, "hostname") - indexd_password = _get_nested_value(creds, "indexd_password") - google_client_secret = _get_nested_value(creds, "google_client_secret") - google_client_id = _get_nested_value(creds, "google_client_id") - hmac_key = _get_nested_value(creds, "hmac_key") - db_path = "postgresql://{}:{}@{}:5432/{}".format( - db_username, db_password, db_host, db_database - ) - - config_file = open(config_file_path, "r").read() - - print(" DB injected with value(s) from creds.json") - config_file = _replace(config_file, "DB", db_path) - - print(" BASE_URL injected with value(s) from creds.json") - config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - - print(" ENCRYPTION_KEY injected with value(s) from creds.json") - config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) - - print( - " OPENID_CONNECT/google/client_secret injected with value(s) " - "from creds.json" - ) - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_secret", google_client_secret - ) - - print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_id", google_client_id - ) - - open(config_file_path, "w+").write(config_file) - - -def set_prod_defaults(config_file_path): - config_file = open(config_file_path, "r").read() - - print( - " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " - "var/www/fence/fence_google_app_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", - "/var/www/fence/fence_google_app_creds_secret.json", - ) - - print( - " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " - "var/www/fence/fence_google_storage_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", - "/var/www/fence/fence_google_storage_creds_secret.json", - ) - - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - - print(" ARBORIST set as http://arborist-service/") - config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") - - print(" HTTP_PROXY/host set as cloud-proxy.internal.io") - config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") - - print(" HTTP_PROXY/port set as 3128") - config_file = _replace(config_file, "HTTP_PROXY/port", 3128) - - print(" DEBUG set to false") - config_file = _replace(config_file, "DEBUG", False) - - print(" MOCK_AUTH set to false") - config_file = _replace(config_file, "MOCK_AUTH", False) - - print(" MOCK_GOOGLE_AUTH set to false") - config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) - - print(" AUTHLIB_INSECURE_TRANSPORT set to true") - config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) - - print(" SESSION_COOKIE_SECURE set to true") - config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) - - print(" ENABLE_CSRF_PROTECTION set to true") - config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) - - open(config_file_path, "w+").write(config_file) - - -def inject_other_files_into_fence_config(other_files, config_file_path): - additional_cfgs = _get_all_additional_configs(other_files) - - config_file = open(config_file_path, "r").read() - - for key, value in additional_cfgs.iteritems(): - print(" {} set to {}".format(key, value)) - config_file = _nested_replace(config_file, key, value) - - open(config_file_path, "w+").write(config_file) - - -def _get_all_additional_configs(other_files): - """ - Attempt to parse given list of files and extract configuration variables and values - """ - additional_configs = dict() - for file_path in other_files: - try: - file_ext = file_path.strip().split(".")[-1] - if file_ext == "json": - json_file = open(file_path, "r") - configs = json.load(json_file) - json_file.close() - elif file_ext == "py": - configs = from_pyfile(file_path) - else: - print( - "Cannot load config vars from a file with extention: {}".format( - file_ext - ) - ) - except Exception as exc: - # if there's any issue reading the file, exit - print( - "Error reading {}. Cannot get configuration. Skipping this file. " - "Details: {}".format(other_files, str(exc)) - ) - continue - - if configs: - additional_configs.update(configs) - - return additional_configs - - -def _nested_replace(config_file, key, value, replacement_path=None): - replacement_path = replacement_path or key - try: - for inner_key, inner_value in value.iteritems(): - temp_path = replacement_path - temp_path = temp_path + "/" + inner_key - config_file = _nested_replace( - config_file, inner_key, inner_value, temp_path - ) - except AttributeError: - # not a dict so replace - if value is not None: - config_file = _replace(config_file, replacement_path, value) - - return config_file - - -def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): - """ - Replace a nested value in a YAML file string with the given value without - losing comments. Uses a regex to do the replacement. - - Args: - yaml_config (str): a string representing a full configuration file - path_to_key (str): nested/path/to/key. The value of this key will be - replaced - replacement_value (str): Replacement value for the key from - path_to_key - """ - nested_path_to_replace = path_to_key.split("/") - - # our regex looks for a specific number of spaces to ensure correct - # level of nesting. It matches to the end of the line - search_string = ( - " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" - ) - matches = re.search(search_string, yaml_config[start:]) - - # early return if we haven't found anything - if not matches: - return yaml_config - - # if we're on the last item in the path, we need to get the value and - # replace it in the original file - if len(nested_path_to_replace) == 1: - # replace the current key:value with the new replacement value - match_start = start + matches.start(0) + len(" " * nested_level) - match_end = start + matches.end(0) - yaml_config = ( - yaml_config[:match_start] - + "{}: {}\n".format( - nested_path_to_replace[0], - _get_yaml_replacement_value(replacement_value, nested_level), - ) - + yaml_config[match_end:] - ) - - return yaml_config - - # set new start point to past current match and move on to next match - start = matches.end(0) - nested_level += 1 - del nested_path_to_replace[0] - - return _replace( - yaml_config, - "/".join(nested_path_to_replace), - replacement_value, - start, - nested_level, - ) - - -def from_pyfile(filename, silent=False): - """ - Modeled after flask's ability to load in python files: - https://github.com/pallets/flask/blob/master/flask/config.py - - Some alterations were made but logic is essentially the same - """ - filename = os.path.abspath(filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except IOError as e: - print("Unable to load configuration file ({})".format(e.strerror)) - if silent: - return False - raise - return _from_object(d) - - -def _from_object(obj): - configs = {} - for key in dir(obj): - if key.isupper(): - configs[key] = getattr(obj, key) - return configs - - -def _get_yaml_replacement_value(value, nested_level=0): - if isinstance(value, str): - return "'" + value + "'" - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, list) or isinstance(value, set): - output = "" - for item in value: - # spaces for nested level then spaces and hyphen for each list item - output += ( - "\n" - + " " * nested_level - + " - " - + _get_yaml_replacement_value(item) - + "" - ) - return output - else: - return value - - -def _get_nested_value(dictionary, nested_path): - """ - Return a value from a dictionary given a path-like nesting of keys. - - Will default to an empty string if value cannot be found. - - Args: - dictionary (dict): a dictionary - nested_path (str): nested/path/to/key - - Returns: - ?: Value from dict - """ - replacement_value_path = nested_path.split("/") - replacement_value = copy.deepcopy(dictionary) - - for item in replacement_value_path: - replacement_value = replacement_value.get(item, {}) - - if replacement_value == {}: - replacement_value = "" - - return replacement_value - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", - "--creds_file_to_inject", - default="creds.json", - help="creds file to inject into the configuration yaml", - ) - parser.add_argument( - "--other_files_to_inject", - nargs="+", - help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " - "inject into the configuration yaml", - ) - parser.add_argument( - "-c", "--config_file", default="config.yaml", help="configuration yaml" - ) - args = parser.parse_args() - - inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) - set_prod_defaults(args.config_file) - - if args.other_files_to_inject: - inject_other_files_into_fence_config( - args.other_files_to_inject, args.config_file - ) diff --git a/helm/fence/fence-secret/fence_settings.py b/helm/fence/fence-secret/fence_settings.py deleted file mode 100644 index eb2cf818d..000000000 --- a/helm/fence/fence-secret/fence_settings.py +++ /dev/null @@ -1,170 +0,0 @@ -from boto.s3.connection import OrdinaryCallingFormat -import config_helper - -APP_NAME = "fence" - -DB = "postgresql://{{db_username}}:{{db_password}}@{{db_host}}:5432/{{db_database}}" - -MOCK_AUTH = False -MOCK_STORAGE = True - -EMAIL_SERVER = "localhost" - -SEND_FROM = "phillis.tt@gmail.com" - -SEND_TO = "phillis.tt@gmail.com" - -CEPH = { - "aws_access_key_id": "", - "aws_secret_access_key": "", - "host": "", - "port": 443, - "is_secure": True, - "calling_format": OrdinaryCallingFormat(), -} - -AWS = {"aws_access_key_id": "", "aws_secret_access_key": ""} - -HMAC_ENCRYPTION_KEY = "{{hmac_key}}" - - -HOSTNAME = "{{hostname}}" -BASE_URL = "https://{{hostname}}/user" - -OPENID_CONNECT = { - "google": { - "client_id": "{{google_client_id}}", - "client_secret": "{{google_client_secret}}", - "redirect_url": "https://" + HOSTNAME + "/user/login/google/login/", - } -} - -HTTP_PROXY = {"host": "cloud-proxy.internal.io", "port": 3128} - -DEFAULT_DBGAP = { - "sftp": { - "host": "", - "username": "", - "password": "", - "port": 22, - "proxy": "", - "proxy_user": "", - }, - "decrypt_key": "", -} - -STORAGE_CREDENTIALS = {} -# aws_credentials should be a dict looks like: -# { identifier: { 'aws_access_key_id': 'XXX', 'aws_secret_access_key': 'XXX' }} -AWS_CREDENTIALS = {} - -# s3_buckets should be a dict looks like: -# { bucket_name: credential_identifie } -S3_BUCKETS = {} - - -def load_json(file_name): - return config_helper.load_json(file_name, APP_NAME) - - -def get_from_dict(dictionary, key, default=""): - value = dictionary.get(key) - if value is None: - value = default - return value - - -creds = load_json("creds.json") -key_list = ["db_username", "db_password", "db_host", "db_database"] - -DB = "postgresql://%s:%s@%s:5432/%s" % tuple( - [get_from_dict(creds, k, "unknown-" + k) for k in key_list] -) -HMAC_ENCRYPTION_KEY = get_from_dict(creds, "hmac_key", "unknown-hmac_key") -HOSTNAME = get_from_dict(creds, "hostname", "unknown-hostname") -BASE_URL = "https://%s/user" % HOSTNAME - -OPENID_CONNECT["google"]["client_id"] = get_from_dict( - creds, "google_client_id", "unknown-google_client_id" -) -OPENID_CONNECT["google"]["client_secret"] = get_from_dict( - creds, "google_client_secret", "unknown-google_client_secret" -) -OPENID_CONNECT["google"]["redirect_url"] = ( - "https://" + HOSTNAME + "/user/login/google/login/" -) - -GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS = { - "dataflow-service-producer-prod.iam.gserviceaccount.com", - "cloudbuild.gserviceaccount.com", - "cloud-ml.google.com.iam.gserviceaccount.com", - "container-engine-robot.iam.gserviceaccount.com", - "dataflow-service-producer-prod.iam.gserviceaccount.com", - "sourcerepo-service-accounts.iam.gserviceaccount.com", - "dataproc-accounts.iam.gserviceaccount.com", - "gae-api-prod.google.com.iam.gserviceaccount.com", - "genomics-api.google.com.iam.gserviceaccount.com", - "containerregistry.iam.gserviceaccount.com", - "container-analysis.iam.gserviceaccount.com", - "cloudservices.gserviceaccount.com", - "stackdriver-service.iam.gserviceaccount.com", - "appspot.gserviceaccount.com", - "partnercontent.gserviceaccount.com", - "trifacta-gcloud-prod.iam.gserviceaccount.com", - "gcf-admin-robot.iam.gserviceaccount.com", - "compute-system.iam.gserviceaccount.com", - "gcp-sa-websecurityscanner.iam.gserviceaccount.com", - "storage-transfer-service.iam.gserviceaccount.com", -} - -CIRRUS_CFG = {} -data = load_json("fence_credentials.json") -if data: - AWS_CREDENTIALS = data["AWS_CREDENTIALS"] - S3_BUCKETS = data["S3_BUCKETS"] - DEFAULT_LOGIN_URL = data["DEFAULT_LOGIN_URL"] - OPENID_CONNECT.update(data["OPENID_CONNECT"]) - OIDC_ISSUER = data["OIDC_ISSUER"] - ENABLED_IDENTITY_PROVIDERS = data["ENABLED_IDENTITY_PROVIDERS"] - APP_NAME = data["APP_NAME"] - HTTP_PROXY = data["HTTP_PROXY"] - dbGaP = data.get("dbGaP", DEFAULT_DBGAP) - CIRRUS_CFG["GOOGLE_API_KEY"] = get_from_dict(data, "GOOGLE_API_KEY") - CIRRUS_CFG["GOOGLE_PROJECT_ID"] = get_from_dict(data, "GOOGLE_PROJECT_ID") - CIRRUS_CFG["GOOGLE_ADMIN_EMAIL"] = get_from_dict(data, "GOOGLE_ADMIN_EMAIL") - CIRRUS_CFG["GOOGLE_IDENTITY_DOMAIN"] = get_from_dict(data, "GOOGLE_IDENTITY_DOMAIN") - CIRRUS_CFG["GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL"] = get_from_dict( - data, "GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL" - ) - - STORAGE_CREDENTIALS = get_from_dict(data, "STORAGE_CREDENTIALS", {}) - GOOGLE_GROUP_PREFIX = get_from_dict(data, "GOOGLE_GROUP_PREFIX", "gen3") - SUPPORT_EMAIL_FOR_ERRORS = get_from_dict(data, "SUPPORT_EMAIL_FOR_ERRORS", None) - WHITE_LISTED_SERVICE_ACCOUNT_EMAILS = get_from_dict( - data, "WHITE_LISTED_SERVICE_ACCOUNT_EMAILS", [] - ) - WHITE_LISTED_GOOGLE_PARENT_ORGS = get_from_dict( - data, "WHITE_LISTED_GOOGLE_PARENT_ORGS", [] - ) - GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS.update( - data.get("GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS", []) - ) - GUN_MAIL = data.get("GUN_MAIL") - REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION = data.get( - "REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION" - ) - # use for intergration tests to skip the login page - MOCK_GOOGLE_AUTH = data.get("MOCK_GOOGLE_AUTH", False) - -CIRRUS_CFG[ - "GOOGLE_APPLICATION_CREDENTIALS" -] = "/var/www/fence/fence_google_app_creds_secret.json" -CIRRUS_CFG[ - "GOOGLE_STORAGE_CREDS" -] = "/var/www/fence/fence_google_storage_creds_secret.json" - -DEFAULT_LOGIN_URL_REDIRECT_PARAM = "redirect" - -INDEXD = "http://indexd-service/" - -ARBORIST = "http://arborist-service/" diff --git a/helm/fence/templates/fence-creds.yaml b/helm/fence/templates/fence-creds.yaml deleted file mode 100644 index 24cfb7adc..000000000 --- a/helm/fence/templates/fence-creds.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: fence-creds -type: Opaque -stringData: - creds.json: |- - { - "db_host": "{{ include "gen3.service-postgres" (dict "key" "host" "service" $.Chart.Name "context" $) }}", - "db_username": "{{include "gen3.service-postgres" (dict "key" "username" "service" $.Chart.Name "context" $) }}", - "db_password": "{{include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) }}", - "db_database": "{{ include "gen3.service-postgres" (dict "key" "database" "service" $.Chart.Name "context" $)}}", - "hostname": "{{ .Values.global.hostname }}", - "indexd_password": "", - "google_client_secret": "YOUR.GOOGLE.SECRET", - "google_client_id": "YOUR.GOOGLE.CLIENT", - "hmac_key": "" - } - diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml index 6facf6a7f..c6b6c535f 100644 --- a/helm/fence/templates/usersync-cron.yaml +++ b/helm/fence/templates/usersync-cron.yaml @@ -56,9 +56,6 @@ spec: configMap: name: "manifest-fence" optional: true - - name: creds-volume - secret: - secretName: "fence-creds" - name: projects configMap: name: "projects" @@ -210,10 +207,6 @@ spec: readOnly: true mountPath: "/var/www/fence/yaml_merge.py" subPath: yaml_merge.py - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json - name: "projects" mountPath: "/var/www/fence/projects.yaml" subPath: "projects.yaml" diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml index a53b08cf8..e07d8a8be 100644 --- a/helm/fence/values.yaml +++ b/helm/fence/values.yaml @@ -391,20 +391,6 @@ env: # -- (list) Volumes to attach to the container. volumes: - - name: old-config-volume - secret: - secretName: "fence-secret" - - name: json-secret-volume - secret: - secretName: "fence-json-secret" - optional: true - - name: creds-volume - secret: - secretName: "fence-creds" - - name: config-helper - configMap: - name: config-helper - optional: true - name: logo-volume configMap: name: "logo-config" @@ -434,22 +420,6 @@ volumes: # -- (list) Volumes to mount to the container. volumeMounts: - - name: "old-config-volume" - readOnly: true - mountPath: "/var/www/fence/local_settings.py" - subPath: local_settings.py - - name: "json-secret-volume" - readOnly: true - mountPath: "/var/www/fence/fence_credentials.json" - subPath: fence_credentials.json - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json - - name: "config-helper" - readOnly: true - mountPath: "/var/www/fence/config_helper.py" - subPath: config_helper.py - name: "logo-volume" readOnly: true mountPath: "/fence/fence/static/img/logo.svg" diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 3eb962779..4e7818180 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -37,7 +37,7 @@ dependencies: repository: "file://../cohort-middleware" condition: cohort-middleware.enabled - name: common - version: 0.1.27 + version: 0.1.28 repository: file://../common - name: dashboard version: 0.1.12 @@ -56,7 +56,7 @@ dependencies: repository: "file://../frontend-framework" condition: frontend-framework.enabled - name: fence - version: 0.1.64 + version: 0.1.65 repository: "file://../fence" condition: fence.enabled - name: gen3-user-data-library diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 9ef55e794..562e3a14c 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -26,12 +26,12 @@ Helm chart to deploy Gen3 Data Commons | file://../aws-es-proxy | aws-es-proxy | 0.1.34 | | file://../cedar | cedar | 0.1.17 | | file://../cohort-middleware | cohort-middleware | 0.1.15 | -| file://../common | common | 0.1.27 | +| file://../common | common | 0.1.28 | | file://../dashboard | dashboard | 0.1.12 | | file://../datareplicate | datareplicate | 0.1.1 | | file://../dicom-server | dicom-server | 0.1.22 | | file://../etl | etl | 0.1.19 | -| file://../fence | fence | 0.1.64 | +| file://../fence | fence | 0.1.65 | | file://../frontend-framework | frontend-framework | 0.1.17 | | file://../gen3-analysis | gen3-analysis | 0.1.4 | | file://../gen3-network-policies | gen3-network-policies | 0.1.3 | diff --git a/helm/peregrine/peregrine-secret/config_helper.py b/helm/peregrine/peregrine-secret/config_helper.py deleted file mode 100644 index 6b303beac..000000000 --- a/helm/peregrine/peregrine-secret/config_helper.py +++ /dev/null @@ -1,376 +0,0 @@ -import json -import os -import copy -import argparse -import re -import types - -# -# make it easy to change this for testing -XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") - - -def default_search_folders(app_name): - """ - Return the list of folders to search for configuration files - """ - return [ - "%s/cdis/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/cdis/%s" % app_name, - "%s/gen3/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/gen3/%s" % app_name, - "/var/www/%s" % app_name, - "/etc/gen3/%s" % app_name, - ] - - -def find_paths(file_name, app_name, search_folders=None): - """ - Search the given folders for file_name - search_folders defaults to default_search_folders if not specified - return the first path to file_name found - """ - search_folders = search_folders or default_search_folders(app_name) - possible_files = [os.path.join(folder, file_name) for folder in search_folders] - return [path for path in possible_files if os.path.exists(path)] - - -def load_json(file_name, app_name, search_folders=None): - """ - json.load(file_name) after finding file_name in search_folders - - return the loaded json data or None if file not found - """ - actual_files = find_paths(file_name, app_name, search_folders) - if not actual_files: - return None - with open(actual_files[0], "r") as reader: - return json.load(reader) - - -def inject_creds_into_fence_config(creds_file_path, config_file_path): - creds_file = open(creds_file_path, "r") - creds = json.load(creds_file) - creds_file.close() - - # get secret values from creds.json file - db_host = _get_nested_value(creds, "db_host") - db_username = _get_nested_value(creds, "db_username") - db_password = _get_nested_value(creds, "db_password") - db_database = _get_nested_value(creds, "db_database") - hostname = _get_nested_value(creds, "hostname") - indexd_password = _get_nested_value(creds, "indexd_password") - google_client_secret = _get_nested_value(creds, "google_client_secret") - google_client_id = _get_nested_value(creds, "google_client_id") - hmac_key = _get_nested_value(creds, "hmac_key") - db_path = "postgresql://{}:{}@{}:5432/{}".format( - db_username, db_password, db_host, db_database - ) - - config_file = open(config_file_path, "r").read() - - print(" DB injected with value(s) from creds.json") - config_file = _replace(config_file, "DB", db_path) - - print(" BASE_URL injected with value(s) from creds.json") - config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - - print(" ENCRYPTION_KEY injected with value(s) from creds.json") - config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) - - print( - " OPENID_CONNECT/google/client_secret injected with value(s) " - "from creds.json" - ) - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_secret", google_client_secret - ) - - print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_id", google_client_id - ) - - open(config_file_path, "w+").write(config_file) - - -def set_prod_defaults(config_file_path): - config_file = open(config_file_path, "r").read() - - print( - " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " - "var/www/fence/fence_google_app_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", - "/var/www/fence/fence_google_app_creds_secret.json", - ) - - print( - " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " - "var/www/fence/fence_google_storage_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", - "/var/www/fence/fence_google_storage_creds_secret.json", - ) - - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - - print(" ARBORIST set as http://arborist-service/") - config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") - - print(" HTTP_PROXY/host set as cloud-proxy.internal.io") - config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") - - print(" HTTP_PROXY/port set as 3128") - config_file = _replace(config_file, "HTTP_PROXY/port", 3128) - - print(" DEBUG set to false") - config_file = _replace(config_file, "DEBUG", False) - - print(" MOCK_AUTH set to false") - config_file = _replace(config_file, "MOCK_AUTH", False) - - print(" MOCK_GOOGLE_AUTH set to false") - config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) - - print(" AUTHLIB_INSECURE_TRANSPORT set to true") - config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) - - print(" SESSION_COOKIE_SECURE set to true") - config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) - - print(" ENABLE_CSRF_PROTECTION set to true") - config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) - - open(config_file_path, "w+").write(config_file) - - -def inject_other_files_into_fence_config(other_files, config_file_path): - additional_cfgs = _get_all_additional_configs(other_files) - - config_file = open(config_file_path, "r").read() - - for key, value in additional_cfgs.iteritems(): - print(" {} set to {}".format(key, value)) - config_file = _nested_replace(config_file, key, value) - - open(config_file_path, "w+").write(config_file) - - -def _get_all_additional_configs(other_files): - """ - Attempt to parse given list of files and extract configuration variables and values - """ - additional_configs = dict() - for file_path in other_files: - try: - file_ext = file_path.strip().split(".")[-1] - if file_ext == "json": - json_file = open(file_path, "r") - configs = json.load(json_file) - json_file.close() - elif file_ext == "py": - configs = from_pyfile(file_path) - else: - print( - "Cannot load config vars from a file with extention: {}".format( - file_ext - ) - ) - except Exception as exc: - # if there's any issue reading the file, exit - print( - "Error reading {}. Cannot get configuration. Skipping this file. " - "Details: {}".format(other_files, str(exc)) - ) - continue - - if configs: - additional_configs.update(configs) - - return additional_configs - - -def _nested_replace(config_file, key, value, replacement_path=None): - replacement_path = replacement_path or key - try: - for inner_key, inner_value in value.iteritems(): - temp_path = replacement_path - temp_path = temp_path + "/" + inner_key - config_file = _nested_replace( - config_file, inner_key, inner_value, temp_path - ) - except AttributeError: - # not a dict so replace - if value is not None: - config_file = _replace(config_file, replacement_path, value) - - return config_file - - -def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): - """ - Replace a nested value in a YAML file string with the given value without - losing comments. Uses a regex to do the replacement. - - Args: - yaml_config (str): a string representing a full configuration file - path_to_key (str): nested/path/to/key. The value of this key will be - replaced - replacement_value (str): Replacement value for the key from - path_to_key - """ - nested_path_to_replace = path_to_key.split("/") - - # our regex looks for a specific number of spaces to ensure correct - # level of nesting. It matches to the end of the line - search_string = ( - " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" - ) - matches = re.search(search_string, yaml_config[start:]) - - # early return if we haven't found anything - if not matches: - return yaml_config - - # if we're on the last item in the path, we need to get the value and - # replace it in the original file - if len(nested_path_to_replace) == 1: - # replace the current key:value with the new replacement value - match_start = start + matches.start(0) + len(" " * nested_level) - match_end = start + matches.end(0) - yaml_config = ( - yaml_config[:match_start] - + "{}: {}\n".format( - nested_path_to_replace[0], - _get_yaml_replacement_value(replacement_value, nested_level), - ) - + yaml_config[match_end:] - ) - - return yaml_config - - # set new start point to past current match and move on to next match - start = matches.end(0) - nested_level += 1 - del nested_path_to_replace[0] - - return _replace( - yaml_config, - "/".join(nested_path_to_replace), - replacement_value, - start, - nested_level, - ) - - -def from_pyfile(filename, silent=False): - """ - Modeled after flask's ability to load in python files: - https://github.com/pallets/flask/blob/master/flask/config.py - - Some alterations were made but logic is essentially the same - """ - filename = os.path.abspath(filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except IOError as e: - print("Unable to load configuration file ({})".format(e.strerror)) - if silent: - return False - raise - return _from_object(d) - - -def _from_object(obj): - configs = {} - for key in dir(obj): - if key.isupper(): - configs[key] = getattr(obj, key) - return configs - - -def _get_yaml_replacement_value(value, nested_level=0): - if isinstance(value, str): - return "'" + value + "'" - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, list) or isinstance(value, set): - output = "" - for item in value: - # spaces for nested level then spaces and hyphen for each list item - output += ( - "\n" - + " " * nested_level - + " - " - + _get_yaml_replacement_value(item) - + "" - ) - return output - else: - return value - - -def _get_nested_value(dictionary, nested_path): - """ - Return a value from a dictionary given a path-like nesting of keys. - - Will default to an empty string if value cannot be found. - - Args: - dictionary (dict): a dictionary - nested_path (str): nested/path/to/key - - Returns: - ?: Value from dict - """ - replacement_value_path = nested_path.split("/") - replacement_value = copy.deepcopy(dictionary) - - for item in replacement_value_path: - replacement_value = replacement_value.get(item, {}) - - if replacement_value == {}: - replacement_value = "" - - return replacement_value - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", - "--creds_file_to_inject", - default="creds.json", - help="creds file to inject into the configuration yaml", - ) - parser.add_argument( - "--other_files_to_inject", - nargs="+", - help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " - "inject into the configuration yaml", - ) - parser.add_argument( - "-c", "--config_file", default="config.yaml", help="configuration yaml" - ) - args = parser.parse_args() - - inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) - set_prod_defaults(args.config_file) - - if args.other_files_to_inject: - inject_other_files_into_fence_config( - args.other_files_to_inject, args.config_file - ) diff --git a/helm/peregrine/peregrine-secret/settings.py b/helm/peregrine/peregrine-secret/settings.py index 1a623a907..125b3f63b 100644 --- a/helm/peregrine/peregrine-secret/settings.py +++ b/helm/peregrine/peregrine-secret/settings.py @@ -5,13 +5,8 @@ from peregrine.api import app, app_init from os import environ -# import config_helper APP_NAME='peregrine' -# def load_json(file_name): -# return config_helper.load_json(file_name, APP_NAME) - -# conf_data = load_json('creds.json') config = app.config # config["AUTH"] = 'https://auth.service.consul:5000/v3/' diff --git a/helm/sheepdog/sheepdog-secret/config_helper.py b/helm/sheepdog/sheepdog-secret/config_helper.py deleted file mode 100644 index ab1805496..000000000 --- a/helm/sheepdog/sheepdog-secret/config_helper.py +++ /dev/null @@ -1,376 +0,0 @@ -import json -import os -import copy -import argparse -import re -import types - -# -# make it easy to change this for testing -XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") - - -def default_search_folders(app_name): - """ - Return the list of folders to search for configuration files - """ - return [ - "%s/cdis/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/cdis/%s" % app_name, - "%s/gen3/%s" % (XDG_DATA_HOME, app_name), - "/usr/share/gen3/%s" % app_name, - "/var/www/%s" % app_name, - "/etc/gen3/%s" % app_name, - ] - - -def find_paths(file_name, app_name, search_folders=None): - """ - Search the given folders for file_name - search_folders defaults to default_search_folders if not specified - return the first path to file_name found - """ - search_folders = search_folders or default_search_folders(app_name) - possible_files = [os.path.join(folder, file_name) for folder in search_folders] - return [path for path in possible_files if os.path.exists(path)] - - -def load_json(file_name, app_name, search_folders=None): - """ - json.load(file_name) after finding file_name in search_folders - - return the loaded json data or None if file not found - """ - actual_files = find_paths(file_name, app_name, search_folders) - if not actual_files: - return None - with open(actual_files[0], "r") as reader: - return json.load(reader) - - -def inject_creds_into_fence_config(creds_file_path, config_file_path): - creds_file = open(creds_file_path, "r") - creds = json.load(creds_file) - creds_file.close() - - # get secret values from creds.json file - db_host = _get_nested_value(creds, "db_host") - db_username = _get_nested_value(creds, "db_username") - db_password = _get_nested_value(creds, "db_password") - db_database = _get_nested_value(creds, "db_database") - hostname = _get_nested_value(creds, "hostname") - indexd_password = environ.get('INDEXD_PASS') - google_client_secret = _get_nested_value(creds, "google_client_secret") - google_client_id = _get_nested_value(creds, "google_client_id") - hmac_key = _get_nested_value(creds, "hmac_key") - db_path = "postgresql://{}:{}@{}:5432/{}".format( - db_username, db_password, db_host, db_database - ) - - config_file = open(config_file_path, "r").read() - - print(" DB injected with value(s) from creds.json") - config_file = _replace(config_file, "DB", db_path) - - print(" BASE_URL injected with value(s) from creds.json") - config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) - - print(" INDEXD_PASSWORD injected with value(s) from creds.json") - config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) - config_file = _replace(config_file, "INDEXD_USERNAME", "fence") - - print(" ENCRYPTION_KEY injected with value(s) from creds.json") - config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) - - print( - " OPENID_CONNECT/google/client_secret injected with value(s) " - "from creds.json" - ) - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_secret", google_client_secret - ) - - print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") - config_file = _replace( - config_file, "OPENID_CONNECT/google/client_id", google_client_id - ) - - open(config_file_path, "w+").write(config_file) - - -def set_prod_defaults(config_file_path): - config_file = open(config_file_path, "r").read() - - print( - " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " - "var/www/fence/fence_google_app_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", - "/var/www/fence/fence_google_app_creds_secret.json", - ) - - print( - " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " - "var/www/fence/fence_google_storage_creds_secret.json" - ) - config_file = _replace( - config_file, - "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", - "/var/www/fence/fence_google_storage_creds_secret.json", - ) - - print(" INDEXD set as http://indexd-service/") - config_file = _replace(config_file, "INDEXD", "http://indexd-service/") - - print(" ARBORIST set as http://arborist-service/") - config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") - - print(" HTTP_PROXY/host set as cloud-proxy.internal.io") - config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") - - print(" HTTP_PROXY/port set as 3128") - config_file = _replace(config_file, "HTTP_PROXY/port", 3128) - - print(" DEBUG set to false") - config_file = _replace(config_file, "DEBUG", False) - - print(" MOCK_AUTH set to false") - config_file = _replace(config_file, "MOCK_AUTH", False) - - print(" MOCK_GOOGLE_AUTH set to false") - config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) - - print(" AUTHLIB_INSECURE_TRANSPORT set to true") - config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) - - print(" SESSION_COOKIE_SECURE set to true") - config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) - - print(" ENABLE_CSRF_PROTECTION set to true") - config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) - - open(config_file_path, "w+").write(config_file) - - -def inject_other_files_into_fence_config(other_files, config_file_path): - additional_cfgs = _get_all_additional_configs(other_files) - - config_file = open(config_file_path, "r").read() - - for key, value in additional_cfgs.iteritems(): - print(" {} set to {}".format(key, value)) - config_file = _nested_replace(config_file, key, value) - - open(config_file_path, "w+").write(config_file) - - -def _get_all_additional_configs(other_files): - """ - Attempt to parse given list of files and extract configuration variables and values - """ - additional_configs = dict() - for file_path in other_files: - try: - file_ext = file_path.strip().split(".")[-1] - if file_ext == "json": - json_file = open(file_path, "r") - configs = json.load(json_file) - json_file.close() - elif file_ext == "py": - configs = from_pyfile(file_path) - else: - print( - "Cannot load config vars from a file with extention: {}".format( - file_ext - ) - ) - except Exception as exc: - # if there's any issue reading the file, exit - print( - "Error reading {}. Cannot get configuration. Skipping this file. " - "Details: {}".format(other_files, str(exc)) - ) - continue - - if configs: - additional_configs.update(configs) - - return additional_configs - - -def _nested_replace(config_file, key, value, replacement_path=None): - replacement_path = replacement_path or key - try: - for inner_key, inner_value in value.iteritems(): - temp_path = replacement_path - temp_path = temp_path + "/" + inner_key - config_file = _nested_replace( - config_file, inner_key, inner_value, temp_path - ) - except AttributeError: - # not a dict so replace - if value is not None: - config_file = _replace(config_file, replacement_path, value) - - return config_file - - -def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): - """ - Replace a nested value in a YAML file string with the given value without - losing comments. Uses a regex to do the replacement. - - Args: - yaml_config (str): a string representing a full configuration file - path_to_key (str): nested/path/to/key. The value of this key will be - replaced - replacement_value (str): Replacement value for the key from - path_to_key - """ - nested_path_to_replace = path_to_key.split("/") - - # our regex looks for a specific number of spaces to ensure correct - # level of nesting. It matches to the end of the line - search_string = ( - " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" - ) - matches = re.search(search_string, yaml_config[start:]) - - # early return if we haven't found anything - if not matches: - return yaml_config - - # if we're on the last item in the path, we need to get the value and - # replace it in the original file - if len(nested_path_to_replace) == 1: - # replace the current key:value with the new replacement value - match_start = start + matches.start(0) + len(" " * nested_level) - match_end = start + matches.end(0) - yaml_config = ( - yaml_config[:match_start] - + "{}: {}\n".format( - nested_path_to_replace[0], - _get_yaml_replacement_value(replacement_value, nested_level), - ) - + yaml_config[match_end:] - ) - - return yaml_config - - # set new start point to past current match and move on to next match - start = matches.end(0) - nested_level += 1 - del nested_path_to_replace[0] - - return _replace( - yaml_config, - "/".join(nested_path_to_replace), - replacement_value, - start, - nested_level, - ) - - -def from_pyfile(filename, silent=False): - """ - Modeled after flask's ability to load in python files: - https://github.com/pallets/flask/blob/master/flask/config.py - - Some alterations were made but logic is essentially the same - """ - filename = os.path.abspath(filename) - d = types.ModuleType("config") - d.__file__ = filename - try: - with open(filename, mode="rb") as config_file: - exec(compile(config_file.read(), filename, "exec"), d.__dict__) - except IOError as e: - print("Unable to load configuration file ({})".format(e.strerror)) - if silent: - return False - raise - return _from_object(d) - - -def _from_object(obj): - configs = {} - for key in dir(obj): - if key.isupper(): - configs[key] = getattr(obj, key) - return configs - - -def _get_yaml_replacement_value(value, nested_level=0): - if isinstance(value, str): - return "'" + value + "'" - elif isinstance(value, bool): - return str(value).lower() - elif isinstance(value, list) or isinstance(value, set): - output = "" - for item in value: - # spaces for nested level then spaces and hyphen for each list item - output += ( - "\n" - + " " * nested_level - + " - " - + _get_yaml_replacement_value(item) - + "" - ) - return output - else: - return value - - -def _get_nested_value(dictionary, nested_path): - """ - Return a value from a dictionary given a path-like nesting of keys. - - Will default to an empty string if value cannot be found. - - Args: - dictionary (dict): a dictionary - nested_path (str): nested/path/to/key - - Returns: - ?: Value from dict - """ - replacement_value_path = nested_path.split("/") - replacement_value = copy.deepcopy(dictionary) - - for item in replacement_value_path: - replacement_value = replacement_value.get(item, {}) - - if replacement_value == {}: - replacement_value = "" - - return replacement_value - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-i", - "--creds_file_to_inject", - default="creds.json", - help="creds file to inject into the configuration yaml", - ) - parser.add_argument( - "--other_files_to_inject", - nargs="+", - help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " - "inject into the configuration yaml", - ) - parser.add_argument( - "-c", "--config_file", default="config.yaml", help="configuration yaml" - ) - args = parser.parse_args() - - inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) - set_prod_defaults(args.config_file) - - if args.other_files_to_inject: - inject_other_files_into_fence_config( - args.other_files_to_inject, args.config_file - ) \ No newline at end of file diff --git a/helm/sheepdog/sheepdog-secret/settings.py b/helm/sheepdog/sheepdog-secret/settings.py index ac896e523..6c76f6d42 100644 --- a/helm/sheepdog/sheepdog-secret/settings.py +++ b/helm/sheepdog/sheepdog-secret/settings.py @@ -5,13 +5,8 @@ from sheepdog.api import app, app_init from os import environ -# import config_helper APP_NAME='sheepdog' -# def load_json(file_name): -# return config_helper.load_json(file_name, APP_NAME) - -# conf_data = load_json('creds.json') config = app.config diff --git a/wip/gen3-test-data-job/templates/_jobs.tpl b/wip/gen3-test-data-job/templates/_jobs.tpl index 8629182b6..155dbbb66 100644 --- a/wip/gen3-test-data-job/templates/_jobs.tpl +++ b/wip/gen3-test-data-job/templates/_jobs.tpl @@ -21,23 +21,6 @@ spec: name: "fence-yaml-merge" - name: shared-data emptyDir: {} -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: old-config-volume - secret: - secretName: "fence-secret" - - name: creds-volume - secret: - secretName: "fence-creds" - - name: config-helper - configMap: - name: config-helper - - name: json-secret-volume - secret: - secretName: "fence-json-secret" -# ----------------------------------------------------------------------------- - name: config-volume secret: secretName: "fence-config" @@ -109,27 +92,6 @@ spec: key: fence-config-public.yaml optional: true volumeMounts: -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: "old-config-volume" - readOnly: true - mountPath: "/var/www/fence/local_settings.py" - subPath: local_settings.py - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json - - name: "config-helper" - readOnly: true - mountPath: "/var/www/fence/config_helper.py" - subPath: config_helper.py - - name: "json-secret-volume" - readOnly: true - mountPath: "/var/www/fence/fence_credentials.json" - subPath: fence_credentials.json -# ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true mountPath: "/var/www/fence/fence-config-secret.yaml" diff --git a/wip/gen3-test-data-job/templates/jobs.yaml b/wip/gen3-test-data-job/templates/jobs.yaml index 5b86fbf9a..4dd11a497 100644 --- a/wip/gen3-test-data-job/templates/jobs.yaml +++ b/wip/gen3-test-data-job/templates/jobs.yaml @@ -16,23 +16,6 @@ spec: name: "fence-yaml-merge" - name: shared-data emptyDir: {} -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: old-config-volume - secret: - secretName: "fence-secret" - - name: creds-volume - secret: - secretName: "fence-creds" - - name: config-helper - configMap: - name: config-helper - - name: json-secret-volume - secret: - secretName: "fence-json-secret" -# ----------------------------------------------------------------------------- - name: config-volume secret: secretName: "fence-config" @@ -104,27 +87,6 @@ spec: key: fence-config-public.yaml optional: true volumeMounts: -# ----------------------------------------------------------------------------- -# DEPRECATED! Remove when all commons are no longer using local_settings.py -# for fence. -# ----------------------------------------------------------------------------- - - name: "old-config-volume" - readOnly: true - mountPath: "/var/www/fence/local_settings.py" - subPath: local_settings.py - - name: "creds-volume" - readOnly: true - mountPath: "/var/www/fence/creds.json" - subPath: creds.json - - name: "config-helper" - readOnly: true - mountPath: "/var/www/fence/config_helper.py" - subPath: config_helper.py - - name: "json-secret-volume" - readOnly: true - mountPath: "/var/www/fence/fence_credentials.json" - subPath: fence_credentials.json -# ----------------------------------------------------------------------------- - name: "config-volume" readOnly: true mountPath: "/var/www/fence/fence-config-secret.yaml"