From 19d1216634c13bd61051e5756482c20e7e7c406f Mon Sep 17 00:00:00 2001 From: suryapratapdesai Date: Tue, 22 Oct 2024 14:02:32 +0530 Subject: [PATCH] DSE-38906 Provision to take various ini files at runtime. This is most sought feature request by customers in cases of Shared Multi-user machines with different ini. files for each customer. --- cmlutils/project_entrypoint.py | 93 +++++++++++++++++++++++++--------- examples/batch_export.py | 6 ++- examples/batch_import.py | 17 +++++-- 3 files changed, 88 insertions(+), 28 deletions(-) diff --git a/cmlutils/project_entrypoint.py b/cmlutils/project_entrypoint.py index a5e0ce6..d385836 100644 --- a/cmlutils/project_entrypoint.py +++ b/cmlutils/project_entrypoint.py @@ -85,6 +85,13 @@ def project_cmd(): """ +@click.group(name="experimental") +def experimental_cmd(): + """ + Sub-entrypoint for WIP/Experimental commands + """ + + @project_cmd.command(name="export") @click.option( "--project_name", @@ -92,10 +99,17 @@ def project_cmd(): help="Name of the project to be migrated. Make sure the name matches with the section name in export-config.ini file", required=True, ) -def project_export_cmd(project_name): +@click.option( + "--ini_location", + "-l", + default="/.cmlutils/export-config.ini", + show_default=True, + help="filepath of the export-config.ini file. default is /.cmlutils/export-config.ini", +) +def project_export_cmd(project_name, ini_location): pexport = None config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/export-config.ini", project_name + os.path.expanduser("~") + ini_location, project_name ) username = config[USERNAME_KEY] @@ -207,17 +221,24 @@ def project_export_cmd(project_name): help="Name of the project to be migrated. Make sure the name matches with the section name in import-config.ini file", required=True, ) +@click.option( + "--ini_location", + "-l", + default="/.cmlutils/import-config.ini", + show_default=True, + help="filepath of the import-config.ini file. default is /.cmlutils/import-config.ini", +) @click.option( "--verify", "-v", - is_flag=True, + is_flag=False, help="Flag to automatically trigger migration validation after import.", ) -def project_import_cmd(project_name, verify): +def project_import_cmd(project_name, ini_location, verify=False): pimport = None import_diff_file_list = None config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/import-config.ini", project_name + os.path.expanduser("~") + ini_location, project_name ) username = config[USERNAME_KEY] @@ -299,11 +320,10 @@ def project_import_cmd(project_name, verify): ) start_time = time.time() if verify: - import_diff_file_list=pimport.transfer_project(log_filedir=log_filedir, verify=True) + import_diff_file_list = pimport.transfer_project(log_filedir=log_filedir, verify=True) else: pimport.transfer_project(log_filedir=log_filedir) - if uses_engine: proj_patch_metadata = {"default_project_engine_type": "legacy_engine"} pimport.convert_project_to_engine_based( @@ -340,7 +360,9 @@ def project_import_cmd(project_name, verify): pimport.terminate_ssh_session() # If verification is also needed after import if verify: - print("***************************************************** Started verifying migration for project: {} ***************************************************** ".format(project_name)) + print( + "***************************************************** Started verifying migration for project: {} ***************************************************** ".format( + project_name)) ( imported_project_data, imported_project_list, @@ -578,8 +600,9 @@ def project_import_cmd(project_name, verify): True if (job_diff or job_config_diff) else False, message="Job Verification", ) - result = [export_diff_file_list,import_diff_file_list,proj_diff, - proj_config_diff,app_diff,app_config_diff,model_diff,model_config_diff,job_diff, job_config_diff] + result = [export_diff_file_list, import_diff_file_list, proj_diff, + proj_config_diff, app_diff, app_config_diff, model_diff, model_config_diff, job_diff, + job_config_diff] migration_status = all(not sublist for sublist in result) validation_data["isMigrationSuccessful"] = migration_status update_verification_status( @@ -612,13 +635,29 @@ def project_import_cmd(project_name, verify): help="Name of project migrated. Make sure the name matches with the section name in import-config.ini and export-config.ini file", required=True, ) -def project_verify_cmd(project_name): +@click.option( + "--export_ini_location", + "-el", + default="/.cmlutils/export-config.ini", + show_default=True, + help="filepath of the export-config.ini file. default is /.cmlutils/export-config.ini", +) +@click.option( + "--import_ini_location", + "-il", + default="/.cmlutils/import-config.ini", + show_default=True, + help="filepath of the import-config.ini file. default is /.cmlutils/import-config.ini", +) +def project_verify_cmd(project_name, export_ini_location, import_ini_location): + pexport = None validation_data = dict() config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/export-config.ini", project_name + os.path.expanduser("~") + export_ini_location, project_name ) + export_username = config[USERNAME_KEY] export_url = config[URL_KEY] export_apiv1_key = config[API_V1_KEY] @@ -709,7 +748,7 @@ def project_verify_cmd(project_name): pexport.terminate_ssh_session() pimport = None import_config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/import-config.ini", project_name + os.path.expanduser("~") + import_ini_location, project_name ) import_username = import_config[USERNAME_KEY] @@ -742,8 +781,8 @@ def project_verify_cmd(project_name): for v in validators: validation_response = v.validate() if ( - validation_response.validation_status - == ValidationResponseStatus.FAILED + validation_response.validation_status + == ValidationResponseStatus.FAILED ): logging.error( "Validation error for project %s: %s", @@ -924,8 +963,9 @@ def project_verify_cmd(project_name): True if (job_diff or job_config_diff) else False, message="Job Verification", ) - result = [export_diff_file_list,import_diff_file_list,proj_diff, - proj_config_diff,app_diff,app_config_diff,model_diff,model_config_diff,job_diff, job_config_diff] + result = [export_diff_file_list, import_diff_file_list, proj_diff, + proj_config_diff, app_diff, app_config_diff, model_diff, model_config_diff, job_diff, + job_config_diff] migration_status = all(not sublist for sublist in result) update_verification_status( not migration_status, @@ -961,11 +1001,18 @@ def project_helpers_cmd(): """ +@click.option( + "--ini_location", + "-l", + default="/.cmlutils/import-config.ini", + show_default=True, + help="filepath of the import-config.ini file. default is /.cmlutils/import-config.ini", +) @project_helpers_cmd.command("populate_engine_runtimes_mapping") -def populate_engine_runtimes_mapping(): +def populate_engine_runtimes_mapping(ini_location): project_name = "DEFAULT" config = _read_config_file( - os.path.expanduser("~") + "/.cmlutils/import-config.ini", project_name + os.path.expanduser("~") + ini_location, project_name ) username = config[USERNAME_KEY] @@ -1020,10 +1067,10 @@ def populate_engine_runtimes_mapping(): # Please make sure utility is having necessary permissions to write/overwrite data try: with open( - os.path.expanduser("~") - + "/.cmlutils/" - + "legacy_engine_runtime_constants.json", - "w", + os.path.expanduser("~") + + "/.cmlutils/" + + "legacy_engine_runtime_constants.json", + "w", ) as legacy_engine_runtime_constants: dump(legacy_runtime_image_map, legacy_engine_runtime_constants) except: diff --git a/examples/batch_export.py b/examples/batch_export.py index c805f36..174e484 100644 --- a/examples/batch_export.py +++ b/examples/batch_export.py @@ -33,8 +33,12 @@ def _get_project_list(file_path: str): def main(): + + # Change the ini location as per the convenience + ini_location = "/.cmlutils/export-config.ini" + project_names = _get_project_list( - os.path.expanduser("~") + "/.cmlutils/export-config.ini" + os.path.expanduser("~") + ini_location ) print(project_names) project_iter = [] diff --git a/examples/batch_import.py b/examples/batch_import.py index d94b75f..92cf8f3 100644 --- a/examples/batch_import.py +++ b/examples/batch_import.py @@ -16,6 +16,8 @@ # This variable controls if user want to trigger migration validation automatically after import # NOTE: Migration validation is resource intensive task keep the BATCH_SIZE to optimal size + +# DO NOT ENABLE this flag VERIFY = False @@ -39,7 +41,10 @@ def get_absolute_path(path: str) -> str: def import_validate(project_name: str): - output_dir = _read_config_file((os.path.expanduser("~") + "/.cmlutils/import-config.ini"), + # Change the ini location as per the convenience + ini_location = "/.cmlutils/import-config.ini" + + output_dir = _read_config_file((os.path.expanduser("~") + ini_location), project_name) import_metrics_file_path = os.path.join(get_absolute_path(output_dir[OUTPUT_DIR_KEY]), project_name, IMPORT_METRIC_FILE) @@ -85,9 +90,12 @@ def _get_project_list(file_path: str): def main(): - failed_validation_list = list() + + # Change the ini location as per the convenience + ini_location = "/.cmlutils/import-config.ini" + project_names = _get_project_list( - os.path.expanduser("~") + "/.cmlutils/import-config.ini" + os.path.expanduser("~") + ini_location ) project_iter = [] @@ -100,8 +108,9 @@ def main(): # call a function on each item in a list pool.starmap(import_project, project_iter) - # validation summary if VERIFY=True + # Please DO NOT enable this feature. if VERIFY: + failed_validation_list = list() for project in project_names: result = import_validate(project) if not result: