From 05660781a7b93b58967205b9ca5bd3aa8b8ab384 Mon Sep 17 00:00:00 2001 From: Chuck Levesque Date: Thu, 12 Sep 2024 12:41:27 -0400 Subject: [PATCH 1/5] Update CDE wrappers for PvC - cdpcli 0.9.105 Update CDE wrappers for PvC - cdpcli 0.9.105 Signed-off-by: Chuck Levesque --- .gitignore | 3 ++- src/cdpy/de.py | 65 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 57ffc17..da27470 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .pytest_cache .tox build -.coverage \ No newline at end of file +.coverage +src/cdpy.egg-info/ diff --git a/src/cdpy/de.py b/src/cdpy/de.py index 917061c..2d1539b 100644 --- a/src/cdpy/de.py +++ b/src/cdpy/de.py @@ -31,7 +31,10 @@ def list_vcs(self, cluster_id): ) def create_vc(self, name, cluster_id, cpu_requests, memory_requests, chart_value_overrides=None, - runtime_spot_component=None, spark_version=None, acl_users=None): + runtime_spot_component=None, spark_version=None, acl_users=None, + gpu_requests=None, guaranteed_cpu_requests=None, + guaranteed_memory_requests=None, guaranteed_gpu_requests=None, + smtp_configs=None, vc_tier='CORE' ): return self.sdk.call( svc='de', func='create_vc', ret_field='Vc', squelch=[ Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) @@ -43,7 +46,13 @@ def create_vc(self, name, cluster_id, cpu_requests, memory_requests, chart_value chartValueOverrides=chart_value_overrides, runtimeSpotComponent=runtime_spot_component, sparkVersion=spark_version, - aclUsers=acl_users + aclUsers=acl_users, + gpuRequests=gpu_requests, + guaranteedCpuRequests=guaranteed_cpu_requests, + guaranteedMemoryRequests=guaranteed_memory_requests, + guaranteedGpuRequests=guaranteed_gpu_requests, + smtpConfigs=smtp_configs, + vcTier=vc_tier ) def delete_vc(self, cluster_id, vc_id): @@ -73,11 +82,17 @@ def list_services(self, env=None, remove_deleted=False): ) return [s for s in services if env is None or s['environmentName'] == env] - def enable_service(self, name, env, instance_type, minimum_instances, maximum_instances, - initial_instances=None, minimum_spot_instances=None, maximum_spot_instances=None, - initial_spot_instances=None, chart_value_overrides=None, enable_public_endpoint=False, - enable_private_network=False, enable_workload_analytics=False, root_volume_size=None, - skip_validation=False, tags=None, use_ssd=None, loadbalancer_allowlist=None, whitelist_ips=None): + + def enable_service(self, name, env, instance_type, minimum_instances, + maximum_instances, initial_instances=None, + minimum_spot_instances=None, maximum_spot_instances=None, + initial_spot_instances=None, chart_value_overrides=None, + enable_public_endpoint=False, enable_private_network=False, + enable_workload_analytics=False, root_volume_size=None, + skip_validation=False, tags=None, use_ssd=None, + loadbalancer_allowlist=None, whitelist_ips=None, subnets=None, + cpu_requests=None, memory_requests=None, gpu_requests=None, + resource_pool=None, nfs_storage_class=None ): return self.sdk.call( svc='de', func='enable_service', ret_field='service', squelch=[ Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) @@ -101,6 +116,12 @@ def enable_service(self, name, env, instance_type, minimum_instances, maximum_in useSsd=use_ssd, whitelistIps=whitelist_ips, loadbalancerAllowlist=loadbalancer_allowlist + subnets=subnets, + cpuRequests=cpu_requests, + memoryRequests=memory_requests, + gpuRequests=gpu_requests, + resourcePool=resource_pool, + nfsStorageClass=nfs_storage_class ) def disable_service(self, cluster_id, force=False): @@ -137,3 +158,33 @@ def get_vc_id_by_name(self, name, cluster_id, remove_deleted=True): vc_id = vc['vcId'] break return vc_id + + def update_service(self, cluster_id, minimum_instances=None, maximum_instances=None, + minimum_spot_instances=None, maximum_spot_instances=None, + whitelist_ips=None, loadbalancer_allowlist=None, + cpu_requests=None, memory_requests=None, gpu_requests=None ): + return self.sdk.call( + svc='de', func='update_service', ret_field='service', squelch=[ + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + clusterId=cluster_id, + minimumInstances=minimum_instances, + maximumInstances=maximum_instances, + minimumSpotInstances=minimum_spot_instances, + maximumSpotInstances=maximum_spot_instances, + whitelistIps=whitelist_ips, + loadbalancerAllowlist=loadbalancer_allowlist, + cpuRequests=cpu_requests, + memoryRequests=memory_requests, + gpuRequests=gpu_requests + ) + + def update_vc(self, cluster_id, vc_id, acl_users=None ): + return self.sdk.call( + svc='de', func='update_vc', ret_field='Vc', squelch=[ + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + clusterId=cluster_id, + vcId=vc_id, + aclUsers=acl_users + ) From bd7b2cba37bfd0319ebbb3a921eac91fa6e70297 Mon Sep 17 00:00:00 2001 From: Chuck Levesque Date: Thu, 12 Sep 2024 12:41:28 -0400 Subject: [PATCH 2/5] Update CDE wrappers for PvC - cdpcli 0.9.105 Update CDE wrappers for PvC - cdpcli 0.9.105 Signed-off-by: Chuck Levesque --- src/cdpy/de.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdpy/de.py b/src/cdpy/de.py index 2d1539b..8058a42 100644 --- a/src/cdpy/de.py +++ b/src/cdpy/de.py @@ -73,7 +73,7 @@ def describe_service(self, cluster_id): clusterId=cluster_id, ) - def list_services(self, env=None, remove_deleted=False): + def list_services(self, remove_deleted=False): services = self.sdk.call( svc='de', func='list_services', ret_field='services', squelch=[ Squelch(value='NOT_FOUND', default=list()), From 0109c6bbf6f604d03abb2846266f3f6b265b3630 Mon Sep 17 00:00:00 2001 From: Chuck Levesque Date: Thu, 12 Sep 2024 12:41:28 -0400 Subject: [PATCH 3/5] Update wrappers for PvC - cdpcli 0.9.105 Update wrappers for PvC - cdpcli 0.9.105 - add drscp.py for PvC DRS Backup/Restore - add metrics.py for PvC Metrics Export - update environments.py for 3 new PvC functions Signed-off-by: Chuck Levesque --- src/cdpy/cdpy.py | 4 ++ src/cdpy/drscp.py | 93 ++++++++++++++++++++++++++++++++++++++++ src/cdpy/environments.py | 62 +++++++++++++++++++++++++++ src/cdpy/metrics.py | 52 ++++++++++++++++++++++ 4 files changed, 211 insertions(+) create mode 100644 src/cdpy/drscp.py create mode 100644 src/cdpy/metrics.py diff --git a/src/cdpy/cdpy.py b/src/cdpy/cdpy.py index 72ae7de..7102c78 100644 --- a/src/cdpy/cdpy.py +++ b/src/cdpy/cdpy.py @@ -14,6 +14,8 @@ from cdpy.opdb import CdpyOpdb from cdpy.dw import CdpyDw from cdpy.df import CdpyDf +from cdpy.drscp import CdpyDrscp +from cdpy.metrics import CdpyMetrics class Cdpy(CdpSdkBase): @@ -30,3 +32,5 @@ def __init__(self, *args, **kwargs): self.dw = CdpyDw(*args, **kwargs) self.df = CdpyDf(*args, **kwargs) self.de = CdpyDe(*args, **kwargs) + self.drscp = CdpyDrscp(*args, **kwargs) + self.metrics = CdpyMetrics(*args, **kwargs) diff --git a/src/cdpy/drscp.py b/src/cdpy/drscp.py new file mode 100644 index 0000000..ab25f98 --- /dev/null +++ b/src/cdpy/drscp.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +from cdpy.common import CdpSdkBase, Squelch, CdpcliWrapper + +ENTITLEMENT_DISABLED='DRS not enabled on CDP Tenant' + + +class CdpyDrscp(CdpSdkBase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def create_backup(self, backup_name=None, item_name=None ): + return self.sdk.call( + svc='drscp', func='create_backup', ret_field='backupCrn', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + backupName=backup_name, + itemName=item_name + ) + + def delete_backup(self, backup_crn): + return self.sdk.call( + svc='drscp', func='delete_backup', ret_field='deleteBackupCrn', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + backupCrn=backup_crn + ) + + def describe_backup(self, backup_crn): + return self.sdk.call( + svc='drscp', func='describe_backup', ret_field='backup', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + backupCrn=backup_crn + ) + + def describe_restore(self, restore_crn): + return self.sdk.call( + svc='drscp', func='describe_restore', ret_field='restore', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + restoreCrn=restore_crn + ) + + def get_logs(self, crn ): + return self.sdk.call( + svc='drscp', func='get_logs', ret_field='logs', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + crn=crn + ) + + def list_backup_entities(self): + return self.sdk.call( + svc='drscp', func='list_backup_entities', ret_field='items', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ] + ) + + def list_backups(self, backup_name=None, job_states=None): + return self.sdk.call( + svc='drscp', func='list_backups', ret_field='backup', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + backupName=backup_name, + jobStates=job_states + ) + + def list_restores(self, job_states=None, backup_crn=None): + return self.sdk.call( + svc='drscp', func='list_restores', ret_field='restores', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + jobStates=job_states, + backupCrn=backup_crn + ) + + def restore_backup(self, backup_crn ): + return self.sdk.call( + svc='drscp', func='restore_backup', ret_field='restoreCrn', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + backupCrn=backup_crn + ) \ No newline at end of file diff --git a/src/cdpy/environments.py b/src/cdpy/environments.py index 415879c..52d2cc3 100644 --- a/src/cdpy/environments.py +++ b/src/cdpy/environments.py @@ -140,6 +140,35 @@ def create_gcp_environment(self, **kwargs): self.sdk.throw_error(resp) return resp + def create_private_environment(self, env_name, address, user, authentication_token, cluster_names, + kube_config=None, authentication_token_type="CLEARTEXT_PASSWORD", + namespace_prefix=None, domain=None, platform=None, docker_config_json=None, + docker_user_pass=None, description=None, storage_class=None): + resp = self.sdk.call( + svc='environments', func='create_private_environment', ret_field='environment', ret_error=True, + squelch=[ + Squelch(value='NOT_FOUND', default=list(), + warning='No Workspaces found in Tenant'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED, + default=list()) + ], + environmentName=env_name, + address=address, + user=user, + authenticationToken=authentication_token, + clusterNames=cluster_names, + kubeConfig=kube_config, + authenticationTokenType=authentication_token_type, + namespacePrefix=namespace_prefix, + domain=domain, + platform=platform, + dockerConfigJson=docker_config_json, + dockerUserPass=docker_user_pass, + description=description, + storageClass=storage_class + ) + + def stop_environment(self, name): return self.sdk.call( svc='environments', func='stop_environment', ret_field='environment', squelch=[ @@ -336,3 +365,36 @@ def resolve_environment_crn(self, env: Union[str, None]): return env_desc['crn'] if env_desc else None else: return None + + def check_database_connectivity(self, host, port, name, username, password): + return self.sdk.call( + svc='environments', func='check_database_connectivity', + ret_field='result', + host=host, + port=port, + name=name, + userName=username, + password=password + ) + + def check_environment_connectivity(self, address, user, authentication_token, + authentication_token_type=None, cluster_names=None): + return self.sdk.call( + svc='environments', func='check_environment_connectivity', + ret_field='clusters', + address=address, + user=user, + authenticationToken=authentication_token, + authenticationTokenType=authentication_token_type, + clusterNames=cluster_names + ) + + def check_kubernetes_connectivity(self, kube_config, format=None): + return self.sdk.call( + svc='environments', func='check_kubernetes_connectivity', + ret_field='status', + kubeConfig=kube_config, + format=format + ) + + diff --git a/src/cdpy/metrics.py b/src/cdpy/metrics.py new file mode 100644 index 0000000..a1e645b --- /dev/null +++ b/src/cdpy/metrics.py @@ -0,0 +1,52 @@ +from cdpy.common import CdpSdkBase, Squelch + + +class CdpyMetrics(CdpSdkBase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def create_remote_write_config(self, remote_write_config=None): + return self.sdk.call( + svc='metrics', func='create_remote_write_config', ret_field='remoteWriteConfig', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + remoteWriteConfig=remote_write_config + ) + + def delete_remote_write_config(self, id=None): + return self.sdk.call( + svc='metrics', func='delete_remote_write_config', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + id=id + ) + + def describe_remote_write_config(self, id=None): + return self.sdk.call( + svc='metrics', func='describe_remote_write_config', + ret_field='remoteWriteConfig', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + id=id + ) + + def list_remote_write_configs(self): + return self.sdk.call( + svc='metrics', func='list_remote_write_configs', + ret_field='remoteWriteConfigs', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ] + ) + + def update_remote_write_config(self, remote_write_config=None): + return self.sdk.call( + svc='metrics', func='update_remote_write_config', ret_field='remoteWriteConfig', squelch=[ + Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), + Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) + ], + remoteWriteConfig=remote_write_config + ) \ No newline at end of file From 96cc53592c8dbe579d2c30a28caf72556cdc429e Mon Sep 17 00:00:00 2001 From: Chuck Levesque Date: Thu, 12 Sep 2024 12:41:28 -0400 Subject: [PATCH 4/5] Update CDE wrappers for PvC - cdpcli 0.9.105 Update CDE wrappers for PvC - cdpcli 0.9.105 Signed-off-by: Chuck Levesque --- src/cdpy/de.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdpy/de.py b/src/cdpy/de.py index 8058a42..235216d 100644 --- a/src/cdpy/de.py +++ b/src/cdpy/de.py @@ -115,7 +115,7 @@ def enable_service(self, name, env, instance_type, minimum_instances, tags=tags, useSsd=use_ssd, whitelistIps=whitelist_ips, - loadbalancerAllowlist=loadbalancer_allowlist + loadbalancerAllowlist=loadbalancer_allowlist, subnets=subnets, cpuRequests=cpu_requests, memoryRequests=memory_requests, From 86835e0a3b53a2bb24004de49e9f09e534f29a65 Mon Sep 17 00:00:00 2001 From: Chuck Levesque Date: Thu, 12 Sep 2024 12:41:28 -0400 Subject: [PATCH 5/5] Update wrappers for PvC - cdpcli 0.9.105 Update wrappers for PvC - cdpcli 0.9.105 - add drscp.py for PvC DRS Backup/Restore - add metrics.py for PvC Metrics Export - update environments.py for 3 new PvC functions Signed-off-by: Chuck Levesque --- src/cdpy/de.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cdpy/de.py b/src/cdpy/de.py index 235216d..39a02b5 100644 --- a/src/cdpy/de.py +++ b/src/cdpy/de.py @@ -73,7 +73,7 @@ def describe_service(self, cluster_id): clusterId=cluster_id, ) - def list_services(self, remove_deleted=False): + def list_services(self, env=None, remove_deleted=False): services = self.sdk.call( svc='de', func='list_services', ret_field='services', squelch=[ Squelch(value='NOT_FOUND', default=list()), @@ -91,6 +91,7 @@ def enable_service(self, name, env, instance_type, minimum_instances, enable_workload_analytics=False, root_volume_size=None, skip_validation=False, tags=None, use_ssd=None, loadbalancer_allowlist=None, whitelist_ips=None, subnets=None, + network_outbound_type=None, cpu_requests=None, memory_requests=None, gpu_requests=None, resource_pool=None, nfs_storage_class=None ): return self.sdk.call( @@ -117,6 +118,7 @@ def enable_service(self, name, env, instance_type, minimum_instances, whitelistIps=whitelist_ips, loadbalancerAllowlist=loadbalancer_allowlist, subnets=subnets, + networkOutboundType=network_outbound_type, cpuRequests=cpu_requests, memoryRequests=memory_requests, gpuRequests=gpu_requests,