diff --git a/.github/workflows/cleanup-firebase.yml b/.github/workflows/cleanup-firebase.yml deleted file mode 100644 index ec939d5b..00000000 --- a/.github/workflows/cleanup-firebase.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Cleanup Firebase Metadata - -on: - schedule: - - cron: "24 18 * * 1" # Runs at 18:24 UTC every Monday - -jobs: - cleanup: - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.11] - os: [ubuntu-latest, windows-latest, macOS-latest] - steps: - - uses: actions/checkout@v4.2.2 - - uses: ./.github/actions/dependencies - - name: Cleanup Firebase Metadata - env: - FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} - FIREBASE_EMAIL: ${{ secrets.FIREBASE_EMAIL }} - run: | - uv run python cellpack/bin/cleanup_tasks.py diff --git a/cellpack/autopack/DBRecipeHandler.py b/cellpack/autopack/DBRecipeHandler.py index 3eb691d5..8e3aec7f 100644 --- a/cellpack/autopack/DBRecipeHandler.py +++ b/cellpack/autopack/DBRecipeHandler.py @@ -1,7 +1,6 @@ import copy import logging import shutil -from datetime import datetime, timezone from enum import Enum from pathlib import Path @@ -10,7 +9,6 @@ import hashlib import json -import requests from cellpack.autopack.utils import deep_merge @@ -321,36 +319,6 @@ def __init__(self, settings): self.settings = settings -class ResultDoc: - def __init__(self, db): - self.db = db - - def handle_expired_results(self): - """ - Check if the results in the database are expired and delete them if the linked object expired. - """ - current_utc = datetime.now(timezone.utc) - results = self.db.get_all_docs("results") - if results: - for result in results: - result_data = self.db.doc_to_dict(result) - result_age = current_utc - result_data["timestamp"] - if result_age.days > 180 and not self.validate_existence( - result_data["url"] - ): - self.db.delete_doc("results", self.db.doc_id(result)) - logging.info("Results cleanup complete.") - else: - logging.info("No results found in the database.") - - def validate_existence(self, url): - """ - Validate the existence of an S3 object by checking if the URL is accessible. - Returns True if the URL is accessible. - """ - return requests.head(url).status_code == requests.codes.ok - - class DBUploader(object): """ Handles the uploading of data to the database. @@ -529,23 +497,6 @@ def upload_config(self, config_data, source_path): self.db.update_doc("configs", id, config_data) return id - def upload_result_metadata(self, file_name, url): - """ - Upload the metadata of the result file to the database. - """ - if self.db: - username = self.db.get_username() - timestamp = self.db.create_timestamp() - self.db.update_or_create( - "results", - file_name, - { - "user": username, - "timestamp": timestamp, - "url": url, - }, - ) - def upload_job_status( self, dedup_hash, @@ -887,23 +838,4 @@ def compile_db_recipe_data(db_recipe_data, obj_dict, grad_dict, comp_dict): return recipe_data -class DBMaintenance(object): - """ - Handles the maintenance of the database. - """ - - def __init__(self, db_handler): - self.db = db_handler - self.result_doc = ResultDoc(self.db) - - def cleanup_results(self): - """ - Check if the results in the database are expired and delete them if the linked object expired. - """ - self.result_doc.handle_expired_results() - - def readme_url(self): - """ - Return the URL to the README file for the database setup section. - """ - return "https://github.com/mesoscope/cellpack?tab=readme-ov-file#introduction-to-remote-databases" +DB_SETUP_README_URL = "https://github.com/mesoscope/cellpack?tab=readme-ov-file#introduction-to-remote-databases" diff --git a/cellpack/autopack/interface_objects/default_values.py b/cellpack/autopack/interface_objects/default_values.py index bbdbc452..969eaa6a 100644 --- a/cellpack/autopack/interface_objects/default_values.py +++ b/cellpack/autopack/interface_objects/default_values.py @@ -9,7 +9,6 @@ "objects", "gradients", "recipes", - "results", "configs", "recipes_edited", ] diff --git a/cellpack/autopack/upy/simularium/simularium_helper.py b/cellpack/autopack/upy/simularium/simularium_helper.py index 08179d85..87c616e1 100644 --- a/cellpack/autopack/upy/simularium/simularium_helper.py +++ b/cellpack/autopack/upy/simularium/simularium_helper.py @@ -22,7 +22,7 @@ from simulariumio.cellpack import HAND_TYPE, CellpackConverter from simulariumio.constants import DISPLAY_TYPE, VIZ_TYPE -from cellpack.autopack.DBRecipeHandler import DBMaintenance, DBUploader +from cellpack.autopack.DBRecipeHandler import DB_SETUP_README_URL from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS from cellpack.autopack.upy import hostHelper from cellpack.autopack.upy.simularium.plots import PlotData @@ -1388,20 +1388,12 @@ def raycast_test(self, obj, start, end, length, **kw): def post_and_open_file(self, file_name, open_results_in_browser, dedup_hash=None): simularium_file = Path(f"{file_name}.simularium") if dedup_hash is None: - file_name, url = simulariumHelper.store_result_file( - simularium_file, storage="aws" - ) - if file_name and url: - simulariumHelper.store_metadata( - file_name, url, db="firebase" - ) - if open_results_in_browser: - simulariumHelper.open_in_simularium(url) + url = simulariumHelper.store_result_file(simularium_file, storage="aws") + if url and open_results_in_browser: + simulariumHelper.open_in_simularium(url) @staticmethod - def store_result_file( - file_path, storage=None, sub_folder="simularium" - ): + def store_result_file(file_path, storage=None, sub_folder="simularium"): if storage == "aws": handler = DATABASE_IDS.handlers().get(storage) initialized_handler = handler( @@ -1409,30 +1401,13 @@ def store_result_file( sub_folder_name=sub_folder, region_name="us-west-2", ) - file_name, url = initialized_handler.save_file_and_get_url(file_path) - if not file_name or not url: - db_maintainer = DBMaintenance(initialized_handler) - logging.warning( - f"Skipping browser opening, upload credentials not configured. For setup instructions see: {db_maintainer.readme_url()}" - ) - return file_name, url - - @staticmethod - def store_metadata(file_name, url, db=None): - if db == "firebase": - handler = DATABASE_IDS.handlers().get(db) - initialized_db = handler( - default_db="staging" - ) # default to staging for metadata uploads - if initialized_db._initialized: - db_uploader = DBUploader(initialized_db) - db_uploader.upload_result_metadata(file_name, url) - else: - db_maintainer = DBMaintenance(initialized_db) + _, url = initialized_handler.save_file_and_get_url(file_path) + if not url: logging.warning( - f"Firebase credentials not found. For setup instructions see: {db_maintainer.readme_url()}. Or try cellPACK web interface: https://cellpack.allencell.org (no setup required)" + f"Skipping browser opening, upload credentials not configured. For setup instructions see: {DB_SETUP_README_URL}" ) - return + return url + return None @staticmethod def open_in_simularium(aws_url): diff --git a/cellpack/bin/cleanup_tasks.py b/cellpack/bin/cleanup_tasks.py deleted file mode 100644 index 08217aa0..00000000 --- a/cellpack/bin/cleanup_tasks.py +++ /dev/null @@ -1,20 +0,0 @@ -from cellpack.autopack.DBRecipeHandler import DBMaintenance -from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS - - -def run_cleanup(db_id=DATABASE_IDS.FIREBASE): - """ - Performs cleanup operations on expired database entries. - This function is executed as part of a scheduled task defined in .github/workflows/cleanup-firebase.yml - - Args: - db_id(str): The database id to use - """ - handler = DATABASE_IDS.handlers().get(db_id) - initialized_db = handler(default_db="staging") - db_maintainer = DBMaintenance(initialized_db) - db_maintainer.cleanup_results() - - -if __name__ == "__main__": - run_cleanup() diff --git a/cellpack/bin/upload.py b/cellpack/bin/upload.py index b038b4e4..59fc2104 100644 --- a/cellpack/bin/upload.py +++ b/cellpack/bin/upload.py @@ -3,7 +3,7 @@ import json from cellpack.autopack.FirebaseHandler import FirebaseHandler -from cellpack.autopack.DBRecipeHandler import DBUploader, DBMaintenance +from cellpack.autopack.DBRecipeHandler import DBUploader, DB_SETUP_README_URL from cellpack.autopack.upy.simularium.simularium_helper import simulariumHelper from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS from cellpack.autopack.loaders.config_loader import ConfigLoader @@ -90,7 +90,7 @@ def upload( id, _ = db_handler.upload_data("editable_fields", field) editable_fields_ids.append(id) if output_file: - _, result_url = simulariumHelper.store_result_file( + result_url = simulariumHelper.store_result_file( output_file, storage="aws", sub_folder="client" ) if studio: @@ -105,9 +105,8 @@ def upload( db_handler.upload_data("example_packings", recipe_metadata) else: - db_maintainer = DBMaintenance(db_handler) sys.exit( - f"The selected database is not initialized. Please set up Firebase credentials to upload recipes. Refer to the instructions at {db_maintainer.readme_url()} " + f"The selected database is not initialized. Please set up Firebase credentials to upload recipes. Refer to the instructions at {DB_SETUP_README_URL} " )