From 8d0b27f14615541bf780b968a229c4b958812299 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Tue, 20 Jan 2026 11:51:41 -0800 Subject: [PATCH 1/5] proposal: get hash from recipe loader --- cellpack/autopack/loaders/recipe_loader.py | 19 ++++++++++++++++++- docker/server.py | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/cellpack/autopack/loaders/recipe_loader.py b/cellpack/autopack/loaders/recipe_loader.py index bbdb662a..e3fe54a3 100644 --- a/cellpack/autopack/loaders/recipe_loader.py +++ b/cellpack/autopack/loaders/recipe_loader.py @@ -7,7 +7,7 @@ import cellpack.autopack as autopack -from cellpack.autopack.DBRecipeHandler import DBRecipeLoader +from cellpack.autopack.DBRecipeHandler import DataDoc, DBRecipeLoader from cellpack.autopack.interface_objects import ( Representations, default_recipe_values, @@ -55,6 +55,23 @@ def __init__( autopack.CURRENT_RECIPE_PATH = os.path.dirname(self.file_path) self.recipe_data = self._read(use_docker=use_docker) + # calculate dedup_hash from the normalized recipe data + self.dedup_hash = DataDoc.generate_hash(self.serializable_recipe_data) + + @staticmethod + def get_dedup_hash(recipe_path, json_recipe=None, use_docker=False): + """ + Load recipe and return its dedup_hash. + This method loads and normalizes the recipe to ensure consistent hashing + regardless of source (local file, firebase, or JSON body). + """ + loader = RecipeLoader( + recipe_path, + save_converted_recipe=False, + use_docker=use_docker, + json_recipe=json_recipe, + ) + return loader.dedup_hash @classmethod def from_json(cls, json_recipe, save_converted_recipe=False, use_docker=False): diff --git a/docker/server.py b/docker/server.py index 74bb20f3..3e736488 100644 --- a/docker/server.py +++ b/docker/server.py @@ -2,6 +2,7 @@ from aiohttp import web from cellpack.autopack.DBRecipeHandler import DataDoc, DBUploader from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS +from cellpack.autopack.loaders.recipe_loader import RecipeLoader from cellpack.bin.pack import pack SERVER_PORT = 80 From 76a3fe0ec5e4c4bc77bce6b605d6bd6ff6c94489 Mon Sep 17 00:00:00 2001 From: Ruge Li Date: Fri, 23 Jan 2026 14:39:54 -0800 Subject: [PATCH 2/5] simplify get_ dedup_hash --- cellpack/autopack/loaders/recipe_loader.py | 19 +------------------ docker/server.py | 1 - 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/cellpack/autopack/loaders/recipe_loader.py b/cellpack/autopack/loaders/recipe_loader.py index e3fe54a3..bbdb662a 100644 --- a/cellpack/autopack/loaders/recipe_loader.py +++ b/cellpack/autopack/loaders/recipe_loader.py @@ -7,7 +7,7 @@ import cellpack.autopack as autopack -from cellpack.autopack.DBRecipeHandler import DataDoc, DBRecipeLoader +from cellpack.autopack.DBRecipeHandler import DBRecipeLoader from cellpack.autopack.interface_objects import ( Representations, default_recipe_values, @@ -55,23 +55,6 @@ def __init__( autopack.CURRENT_RECIPE_PATH = os.path.dirname(self.file_path) self.recipe_data = self._read(use_docker=use_docker) - # calculate dedup_hash from the normalized recipe data - self.dedup_hash = DataDoc.generate_hash(self.serializable_recipe_data) - - @staticmethod - def get_dedup_hash(recipe_path, json_recipe=None, use_docker=False): - """ - Load recipe and return its dedup_hash. - This method loads and normalizes the recipe to ensure consistent hashing - regardless of source (local file, firebase, or JSON body). - """ - loader = RecipeLoader( - recipe_path, - save_converted_recipe=False, - use_docker=use_docker, - json_recipe=json_recipe, - ) - return loader.dedup_hash @classmethod def from_json(cls, json_recipe, save_converted_recipe=False, use_docker=False): diff --git a/docker/server.py b/docker/server.py index 3e736488..74bb20f3 100644 --- a/docker/server.py +++ b/docker/server.py @@ -2,7 +2,6 @@ from aiohttp import web from cellpack.autopack.DBRecipeHandler import DataDoc, DBUploader from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS -from cellpack.autopack.loaders.recipe_loader import RecipeLoader from cellpack.bin.pack import pack SERVER_PORT = 80 From 6aae137a6854967f9ff82aaf47e721d4e309ccff Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 21 Jan 2026 09:12:21 -0800 Subject: [PATCH 3/5] only post simularium results file once for server job runs --- cellpack/autopack/DBRecipeHandler.py | 6 +++++- .../upy/simularium/simularium_helper.py | 21 ++++++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index 00fc8bb7..b0199b27 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -675,8 +675,11 @@ def upload_outputs_to_s3(self, output_folder, recipe_name, dedup_hash): f"{base_url}/{file_info['s3_key']}" for file_info in upload_result["uploaded_files"] ] + simularium_url = None + for url in public_urls: + if url.endswith(".simularium"): + simularium_url = url outputs_directory = f"https://us-west-2.console.aws.amazon.com/s3/buckets/{bucket_name}/{s3_prefix}/" - logging.info( f"Successfully uploaded {upload_result['total_files']} files to {outputs_directory}" ) @@ -694,6 +697,7 @@ def upload_outputs_to_s3(self, output_folder, recipe_name, dedup_hash): "total_size": upload_result["total_size"], "urls": public_urls, "outputs_directory": outputs_directory, + "simularium_url": simularium_url, } except Exception as e: logging.error(e) diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index 4f934e0e..5d7c514b 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -1399,24 +1399,15 @@ def post_and_open_file(self, file_name, open_results_in_browser, dedup_hash=None @staticmethod def store_result_file( - file_path, storage=None, batch_job_id=None, sub_folder="simularium" + file_path, storage=None, sub_folder="simularium" ): if storage == "aws": handler = DATABASE_IDS.handlers().get(storage) - # if batch_job_id is not None, then we are in a batch job and should use the temp bucket - # TODO: use cellpack-results bucket for batch jobs once we have the correct permissions - if batch_job_id: - initialized_handler = handler( - bucket_name="cellpack-demo", - sub_folder_name=sub_folder, - region_name="us-west-2", - ) - else: - initialized_handler = handler( - bucket_name="cellpack-results", - sub_folder_name=sub_folder, - region_name="us-west-2", - ) + initialized_handler = handler( + bucket_name="cellpack-results", + sub_folder_name=sub_folder, + region_name="us-west-2", + ) file_name, url = initialized_handler.save_file_and_get_url(file_path) if not file_name or not url: db_maintainer = DBMaintenance(initialized_handler) From f8f82f592680e9193a9641cf496f1926c472961e Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 28 Jan 2026 10:29:32 -0800 Subject: [PATCH 4/5] update code for rebase --- cellpack/autopack/DBRecipeHandler.py | 6 ++---- .../autopack/upy/simularium/simularium_helper.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index b0199b27..3eb691d5 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -529,7 +529,7 @@ def upload_config(self, config_data, source_path): self.db.update_doc("configs", id, config_data) return id - def upload_result_metadata(self, file_name, url, dedup_hash=None): + def upload_result_metadata(self, file_name, url): """ Upload the metadata of the result file to the database. """ @@ -543,11 +543,8 @@ def upload_result_metadata(self, file_name, url, dedup_hash=None): "user": username, "timestamp": timestamp, "url": url, - "dedup_hash": dedup_hash, }, ) - if dedup_hash: - self.upload_job_status(dedup_hash, "DONE", result_path=url) def upload_job_status( self, @@ -644,6 +641,7 @@ def upload_packing_results_workflow( self.upload_job_status( dedup_hash, "DONE", + result_path=upload_result.get("simularium_url"), outputs_directory=upload_result.get("outputs_directory"), ) diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index 5d7c514b..483ae414 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -1387,12 +1387,14 @@ def raycast_test(self, obj, start, end, length, **kw): def post_and_open_file(self, file_name, open_results_in_browser, dedup_hash=None): simularium_file = Path(f"{file_name}.simularium") - file_name, url = simulariumHelper.store_result_file( - simularium_file, storage="aws", batch_job_id=dedup_hash - ) + file_name, url = None, None + if dedup_hash is None: + file_name, url = simulariumHelper.store_result_file( + simularium_file, storage="aws" + ) if file_name and url: simulariumHelper.store_metadata( - file_name, url, db="firebase", dedup_hash=dedup_hash + file_name, url, db="firebase" ) if open_results_in_browser: simulariumHelper.open_in_simularium(url) @@ -1417,7 +1419,7 @@ def store_result_file( return file_name, url @staticmethod - def store_metadata(file_name, url, db=None, dedup_hash=None): + def store_metadata(file_name, url, db=None): if db == "firebase": handler = DATABASE_IDS.handlers().get(db) initialized_db = handler( @@ -1425,7 +1427,7 @@ def store_metadata(file_name, url, db=None, dedup_hash=None): ) # default to staging for metadata uploads if initialized_db._initialized: db_uploader = DBUploader(initialized_db) - db_uploader.upload_result_metadata(file_name, url, dedup_hash) + db_uploader.upload_result_metadata(file_name, url) else: db_maintainer = DBMaintenance(initialized_db) logging.warning( From f648a2fd2247e0617259337facbd40210ea5a75a Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 28 Jan 2026 12:59:30 -0800 Subject: [PATCH 5/5] code cleanup --- .../autopack/upy/simularium/simularium_helper.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index 483ae414..08179d85 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -1387,17 +1387,16 @@ def raycast_test(self, obj, start, end, length, **kw): def post_and_open_file(self, file_name, open_results_in_browser, dedup_hash=None): simularium_file = Path(f"{file_name}.simularium") - file_name, url = None, None if dedup_hash is None: file_name, url = simulariumHelper.store_result_file( simularium_file, storage="aws" ) - if file_name and url: - simulariumHelper.store_metadata( - file_name, url, db="firebase" - ) - if open_results_in_browser: - simulariumHelper.open_in_simularium(url) + if file_name and url: + simulariumHelper.store_metadata( + file_name, url, db="firebase" + ) + if open_results_in_browser: + simulariumHelper.open_in_simularium(url) @staticmethod def store_result_file(