From ae4d10d4affde0e8a24778908d42d91fd8196226 Mon Sep 17 00:00:00 2001 From: Alex Lopes Date: Sun, 11 Sep 2022 17:49:11 -0700 Subject: [PATCH 1/3] Adding option for a user to set the job clusters as single user to make the cluster UC friendly, also adding option for users to set a policy to all job clusters --- dbclient/JobsClient.py | 29 +++++++++++++++++++++++++++++ dbclient/parser.py | 6 ++++++ import_db.py | 18 ++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/dbclient/JobsClient.py b/dbclient/JobsClient.py index e4445820..71bb5d66 100644 --- a/dbclient/JobsClient.py +++ b/dbclient/JobsClient.py @@ -227,3 +227,32 @@ def delete_all_jobs(self): job_list = self.get('/jobs/list').get('jobs', []) for job in job_list: self.post('/jobs/delete', {'job_id': job['job_id']}) + + # MTJ Jobs not supported + def single_user_all_jobs(self): + job_list = self.get_jobs_list() + for job_conf in job_list: + job_settings = job_conf['settings'] + job_clusters = job_settings.get('new_cluster', None) + + if job_clusters: + job_clusters['data_security_mode'] = 'LEGACY_SINGLE_USER_STANDARD' + job_settings['new_cluster'] = job_clusters + update_job_conf = {'job_id': job_conf['job_id'], + 'new_settings': job_settings} + update_job_resp = self.post('/jobs/reset', update_job_conf) + + # MTJ Jobs not supported + def set_policy_all_jobs(self, policy_id): + job_list = self.get_jobs_list() + for job_conf in job_list: + job_settings = job_conf['settings'] + job_clusters = job_settings.get('new_cluster', None) + + if job_clusters: + job_clusters['policy_id'] = policy_id + job_settings['new_cluster'] = job_clusters + update_job_conf = {'job_id': job_conf['job_id'], + 'new_settings': job_settings} + update_job_resp = self.post('/jobs/reset', update_job_conf) + diff --git a/dbclient/parser.py b/dbclient/parser.py index 8cbbb786..2f0ffa5a 100644 --- a/dbclient/parser.py +++ b/dbclient/parser.py @@ -391,6 +391,12 @@ def get_import_parser(): parser.add_argument('--delete-all-jobs', action='store_true', help='Delete all jobs') + parser.add_argument('--single-user-all-jobs', action='store_true', + help='Set all jobs as single user to allow UC enabled clusters') + + parser.add_argument('--set-policy-all-jobs', action='store', + help='Set all jobs with the provided policy') + parser.add_argument('--use-checkpoint', action='store_true', help='use checkpointing to restart from previous state') diff --git a/import_db.py b/import_db.py index 508830a0..a278bf1f 100644 --- a/import_db.py +++ b/import_db.py @@ -186,6 +186,24 @@ def main(): end = timer() print("Delete all jobs time: " + str(timedelta(seconds=end - start))) + if args.single_user_all_jobs: + print("Setting all jobs on new clusters as single user mode to enable UC {0}".format(now)) + start = timer() + jobs_c = JobsClient(client_config, checkpoint_service) + # log job configs + jobs_c.single_user_all_jobs() + end = timer() + print("Single user all jobs time: " + str(timedelta(seconds=end - start))) + + if args.set_policy_all_jobs: + print("Setting all jobs on new clusters with the provided policy {0}".format(now)) + start = timer() + jobs_c = JobsClient(client_config, checkpoint_service) + # log job configs + jobs_c.set_policy_all_jobs(args.set_policy_all_jobs) + end = timer() + print("Set policy all jobs time: " + str(timedelta(seconds=end - start))) + if args.single_user: user_email = args.single_user print(f"Import user {user_email} at {now}") From 1e4d22922c9e0566ba81e336ca3ac63f8b751632 Mon Sep 17 00:00:00 2001 From: Alex Lopes Date: Mon, 12 Sep 2022 10:42:25 -0700 Subject: [PATCH 2/3] =?UTF-8?q?=C3=84dding=20the=20option=20to=20set=20all?= =?UTF-8?q?=20clusters=20as=20shared=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbclient/JobsClient.py | 14 ++++++++++++++ dbclient/parser.py | 3 +++ import_db.py | 9 +++++++++ 3 files changed, 26 insertions(+) diff --git a/dbclient/JobsClient.py b/dbclient/JobsClient.py index 71bb5d66..99a3086e 100644 --- a/dbclient/JobsClient.py +++ b/dbclient/JobsClient.py @@ -242,6 +242,20 @@ def single_user_all_jobs(self): 'new_settings': job_settings} update_job_resp = self.post('/jobs/reset', update_job_conf) + # MTJ Jobs not supported + def shared_all_jobs(self): + job_list = self.get_jobs_list() + for job_conf in job_list: + job_settings = job_conf['settings'] + job_clusters = job_settings.get('new_cluster', None) + + if job_clusters: + job_clusters['data_security_mode'] = 'LEGACY_TABLE_ACL' + job_settings['new_cluster'] = job_clusters + update_job_conf = {'job_id': job_conf['job_id'], + 'new_settings': job_settings} + update_job_resp = self.post('/jobs/reset', update_job_conf) + # MTJ Jobs not supported def set_policy_all_jobs(self, policy_id): job_list = self.get_jobs_list() diff --git a/dbclient/parser.py b/dbclient/parser.py index 2f0ffa5a..11620d8e 100644 --- a/dbclient/parser.py +++ b/dbclient/parser.py @@ -394,6 +394,9 @@ def get_import_parser(): parser.add_argument('--single-user-all-jobs', action='store_true', help='Set all jobs as single user to allow UC enabled clusters') + parser.add_argument('--shared-all-jobs', action='store_true', + help='Set all jobs as shared to allow UC enabled clusters') + parser.add_argument('--set-policy-all-jobs', action='store', help='Set all jobs with the provided policy') diff --git a/import_db.py b/import_db.py index a278bf1f..a94a5d01 100644 --- a/import_db.py +++ b/import_db.py @@ -195,6 +195,15 @@ def main(): end = timer() print("Single user all jobs time: " + str(timedelta(seconds=end - start))) + if args.shared_all_jobs: + print("Setting all jobs on new clusters as shared mode to enable UC {0}".format(now)) + start = timer() + jobs_c = JobsClient(client_config, checkpoint_service) + # log job configs + jobs_c.shared_all_jobs() + end = timer() + print("Shared access mode all jobs time: " + str(timedelta(seconds=end - start))) + if args.set_policy_all_jobs: print("Setting all jobs on new clusters with the provided policy {0}".format(now)) start = timer() From 72f2bca9bcad711cdf02644adf6d624f329e1d39 Mon Sep 17 00:00:00 2001 From: Alex Lopes Date: Mon, 12 Sep 2022 11:39:07 -0700 Subject: [PATCH 3/3] Removing unused variable --- dbclient/JobsClient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbclient/JobsClient.py b/dbclient/JobsClient.py index 99a3086e..50debda8 100644 --- a/dbclient/JobsClient.py +++ b/dbclient/JobsClient.py @@ -240,7 +240,7 @@ def single_user_all_jobs(self): job_settings['new_cluster'] = job_clusters update_job_conf = {'job_id': job_conf['job_id'], 'new_settings': job_settings} - update_job_resp = self.post('/jobs/reset', update_job_conf) + self.post('/jobs/reset', update_job_conf) # MTJ Jobs not supported def shared_all_jobs(self): @@ -254,7 +254,7 @@ def shared_all_jobs(self): job_settings['new_cluster'] = job_clusters update_job_conf = {'job_id': job_conf['job_id'], 'new_settings': job_settings} - update_job_resp = self.post('/jobs/reset', update_job_conf) + self.post('/jobs/reset', update_job_conf) # MTJ Jobs not supported def set_policy_all_jobs(self, policy_id): @@ -268,5 +268,5 @@ def set_policy_all_jobs(self, policy_id): job_settings['new_cluster'] = job_clusters update_job_conf = {'job_id': job_conf['job_id'], 'new_settings': job_settings} - update_job_resp = self.post('/jobs/reset', update_job_conf) + self.post('/jobs/reset', update_job_conf)