From f342a43827a22ad008e85604532614e1d0537f59 Mon Sep 17 00:00:00 2001 From: James Nightingale Date: Fri, 28 Mar 2025 18:51:52 +0000 Subject: [PATCH 1/6] nautilus does not auto delete its search internal --- autofit/non_linear/search/nest/nautilus/search.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/autofit/non_linear/search/nest/nautilus/search.py b/autofit/non_linear/search/nest/nautilus/search.py index 289f1a010..b9768ab99 100644 --- a/autofit/non_linear/search/nest/nautilus/search.py +++ b/autofit/non_linear/search/nest/nautilus/search.py @@ -159,10 +159,6 @@ def _fit(self, model: AbstractPriorModel, analysis): checkpoint_exists=checkpoint_exists, ) - if self.checkpoint_file is not None: - - os.remove(self.checkpoint_file) - return search_internal @property From eff64f4c9d7d7fb2fa5a209432b87d8011c3ef76 Mon Sep 17 00:00:00 2001 From: James Nightingale Date: Fri, 28 Mar 2025 18:52:34 +0000 Subject: [PATCH 2/6] same for dynest --- autofit/non_linear/search/nest/dynesty/search/abstract.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/autofit/non_linear/search/nest/dynesty/search/abstract.py b/autofit/non_linear/search/nest/dynesty/search/abstract.py index da91f33e1..715f2b716 100644 --- a/autofit/non_linear/search/nest/dynesty/search/abstract.py +++ b/autofit/non_linear/search/nest/dynesty/search/abstract.py @@ -201,11 +201,6 @@ def _fit( during_analysis=True, ) - try: - os.remove(self.checkpoint_file) - except TypeError: - pass - return search_internal def samples_info_from(self, search_internal=None): From 926a3f428dbdc2642dab1a2b75a3d87fcae72c31 Mon Sep 17 00:00:00 2001 From: James Nightingale Date: Fri, 28 Mar 2025 18:55:03 +0000 Subject: [PATCH 3/6] remove nuclear mode --- .../non_linear/grid/grid_search/__init__.py | 1 - autofit/non_linear/grid/grid_search/job.py | 1 - autofit/non_linear/paths/abstract.py | 52 ------------------- autofit/non_linear/search/abstract_search.py | 14 ++--- 4 files changed, 3 insertions(+), 65 deletions(-) diff --git a/autofit/non_linear/grid/grid_search/__init__.py b/autofit/non_linear/grid/grid_search/__init__.py index dc57d4474..0a33d0af0 100644 --- a/autofit/non_linear/grid/grid_search/__init__.py +++ b/autofit/non_linear/grid/grid_search/__init__.py @@ -273,7 +273,6 @@ def write_results(): def save_metadata(self): self.paths.save_unique_tag(is_grid_search=True) - self.paths.zip_remove_nuclear() def make_jobs(self, model, analysis, grid_priors, info: Optional[Dict] = None): grid_priors = model.sort_priors_alphabetically(set(grid_priors)) diff --git a/autofit/non_linear/grid/grid_search/job.py b/autofit/non_linear/grid/grid_search/job.py index de70edb88..892d8ad46 100644 --- a/autofit/non_linear/grid/grid_search/job.py +++ b/autofit/non_linear/grid/grid_search/job.py @@ -55,7 +55,6 @@ def perform(self): model=self.model, analysis=self.analysis, info=self.info, - bypass_nuclear_if_on=True, ) result_list_row = [ self.index, diff --git a/autofit/non_linear/paths/abstract.py b/autofit/non_linear/paths/abstract.py index 9016a8dd1..1fea99cbf 100644 --- a/autofit/non_linear/paths/abstract.py +++ b/autofit/non_linear/paths/abstract.py @@ -291,58 +291,6 @@ def _zip(self): except FileNotFoundError: pass - def zip_remove_nuclear(self): - """ - When multiple model-fits are performed using the same `path_prefix` and `name`, - the results are populated in the same folder with different unique identifiers. - - By accident, one may perform runs where additional results are placed - in these folders which are not wanted for the subsequent analysis. - - Removing these results from the directory can be cumbersome, as determining - the unwanted results based on their unique identifier requires visually inspecting - them. - - These unwanted results can also make manipulating the results via the database - problematic, as one may need to again filter based on unique identifier. - - When a run is performed in nuclear mode, all results in every folder are - deleted except the results corresponding to the unique identifier of that run. - - Therefore, provided the user is 100% certain that the run corresponds to the - results they want to keep, nuclear mode can be used to remove all unwanted results. - - For example, suppose a folder has 5 results, 4 of which are unwanted and 1 which is - wanted. If nuclear mode runs, and the model-fit is set up correctly such that the - identifier created corresponds to the wanted result, all 4 unwanted results - will be deleted. - - To enable nuclear mode, set the environment variable ``PYAUTOFIT_NUCLEAR_MODE=1``. - - Nuclear mode is dangerous, and must be used with CAUTION AND CARE! - """ - - if os.environ.get("PYAUTOFIT_NUCLEAR_MODE") == "1": - file_path = Path(os.path.split(self.output_path)[0]) - - file_list = os.listdir(file_path) - file_list = [file for file in file_list if self.identifier not in file] - - for file in file_list: - file_to_remove = file_path / file - - try: - os.remove(file_to_remove) - logger.info(f"NUCLEAR MODE -- Removed {file_to_remove}") - except (IsADirectoryError, FileNotFoundError): - pass - - try: - shutil.rmtree(file_to_remove) - logger.info(f"NUCLEAR MODE -- Removed {file_to_remove}") - except (NotADirectoryError, FileNotFoundError): - pass - def restore(self): """ Copy files from the ``.zip`` file to the samples folder. diff --git a/autofit/non_linear/search/abstract_search.py b/autofit/non_linear/search/abstract_search.py index 5ad770ece..44a42e4fa 100644 --- a/autofit/non_linear/search/abstract_search.py +++ b/autofit/non_linear/search/abstract_search.py @@ -534,7 +534,6 @@ def fit( model: AbstractPriorModel, analysis: Analysis, info: Optional[Dict] = None, - bypass_nuclear_if_on: bool = False, ) -> Union[Result, List[Result]]: """ Fit a model, M with some function f that takes instances of the @@ -557,9 +556,6 @@ class represented by model M and gives a score for their fitness. info Optional dictionary containing information about the fit that can be saved in the `files` folder (e.g. as `files/info.json`) and can be loaded via the database. - bypass_nuclear_if_on - If nuclear mode is on (environment variable "PYAUTOFIT_NUCLEAR_MODE=1") passing this as True will - bypass it. Returns ------- @@ -612,7 +608,6 @@ class represented by model M and gives a score for their fitness. ) self.post_fit_output( - bypass_nuclear_if_on=bypass_nuclear_if_on, search_internal=result.search_internal, ) @@ -847,7 +842,7 @@ def result_via_completed_fit( return result - def post_fit_output(self, search_internal, bypass_nuclear_if_on: bool): + def post_fit_output(self, search_internal): """ Cleans up the output folderds after a completed non-linear search. @@ -860,8 +855,8 @@ def post_fit_output(self, search_internal, bypass_nuclear_if_on: bool): Parameters ---------- - bypass_nuclear_if_on - Whether to use nuclear mode to delete a lot of files (see nuclear mode description). + search_internal + The internal search. """ if not conf.instance["output"]["search_internal"]: self.logger.info("Removing search internal folder.") @@ -874,9 +869,6 @@ def post_fit_output(self, search_internal, bypass_nuclear_if_on: bool): self.paths.zip_remove() - if not bypass_nuclear_if_on: - self.paths.zip_remove_nuclear() - @abstractmethod def _fit(self, model: AbstractPriorModel, analysis: Analysis): pass From d1df0e7cea6de1bcb867df5ff60061cfa23fb3e5 Mon Sep 17 00:00:00 2001 From: James Nightingale Date: Fri, 28 Mar 2025 19:21:17 +0000 Subject: [PATCH 4/6] put checkpoint deletes in output_search_internal --- autofit/non_linear/search/nest/dynesty/search/abstract.py | 6 ++++++ autofit/non_linear/search/nest/nautilus/search.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/autofit/non_linear/search/nest/dynesty/search/abstract.py b/autofit/non_linear/search/nest/dynesty/search/abstract.py index 715f2b716..545568874 100644 --- a/autofit/non_linear/search/nest/dynesty/search/abstract.py +++ b/autofit/non_linear/search/nest/dynesty/search/abstract.py @@ -480,6 +480,12 @@ def search_internal_from( ): raise NotImplementedError() + def output_search_internal(self, search_internal): + try: + os.remove(self.checkpoint_file) + except TypeError: + pass + def check_pool(self, uses_pool: bool, pool): if (uses_pool and pool is None) or (not uses_pool and pool is not None): raise exc.SearchException( diff --git a/autofit/non_linear/search/nest/nautilus/search.py b/autofit/non_linear/search/nest/nautilus/search.py index b9768ab99..fe9f525c2 100644 --- a/autofit/non_linear/search/nest/nautilus/search.py +++ b/autofit/non_linear/search/nest/nautilus/search.py @@ -464,6 +464,10 @@ def output_search_internal(self, search_internal): search_internal.pool_l = pool_l search_internal.pool_s = pool_s + if self.checkpoint_file is not None: + + os.remove(self.checkpoint_file) + def samples_info_from(self, search_internal=None): return { "log_evidence": search_internal.evidence(), From e5f4d62999834143c155fc7331d75d1835714ef1 Mon Sep 17 00:00:00 2001 From: James Nightingale Date: Fri, 28 Mar 2025 19:26:16 +0000 Subject: [PATCH 5/6] try / except to fix case where file already deleted after resume --- autofit/non_linear/search/nest/dynesty/search/abstract.py | 2 +- autofit/non_linear/search/nest/nautilus/search.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/autofit/non_linear/search/nest/dynesty/search/abstract.py b/autofit/non_linear/search/nest/dynesty/search/abstract.py index 545568874..06db8d7df 100644 --- a/autofit/non_linear/search/nest/dynesty/search/abstract.py +++ b/autofit/non_linear/search/nest/dynesty/search/abstract.py @@ -483,7 +483,7 @@ def search_internal_from( def output_search_internal(self, search_internal): try: os.remove(self.checkpoint_file) - except TypeError: + except (TypeError, FileNotFoundError): pass def check_pool(self, uses_pool: bool, pool): diff --git a/autofit/non_linear/search/nest/nautilus/search.py b/autofit/non_linear/search/nest/nautilus/search.py index fe9f525c2..9cce8d34e 100644 --- a/autofit/non_linear/search/nest/nautilus/search.py +++ b/autofit/non_linear/search/nest/nautilus/search.py @@ -464,9 +464,10 @@ def output_search_internal(self, search_internal): search_internal.pool_l = pool_l search_internal.pool_s = pool_s - if self.checkpoint_file is not None: - + try: os.remove(self.checkpoint_file) + except (TypeError, FileNotFoundError): + pass def samples_info_from(self, search_internal=None): return { From 24fdafa32c8fd6393f1ee0000f7a9ab550fd9bf3 Mon Sep 17 00:00:00 2001 From: James Nightingale Date: Fri, 28 Mar 2025 19:33:54 +0000 Subject: [PATCH 6/6] add output_search_internal to dynesty which works --- autofit/non_linear/search/nest/dynesty/search/abstract.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/autofit/non_linear/search/nest/dynesty/search/abstract.py b/autofit/non_linear/search/nest/dynesty/search/abstract.py index 06db8d7df..2d7e9ccc5 100644 --- a/autofit/non_linear/search/nest/dynesty/search/abstract.py +++ b/autofit/non_linear/search/nest/dynesty/search/abstract.py @@ -481,6 +481,11 @@ def search_internal_from( raise NotImplementedError() def output_search_internal(self, search_internal): + + self.paths.save_search_internal( + obj=search_internal, + ) + try: os.remove(self.checkpoint_file) except (TypeError, FileNotFoundError):