From bd80d3e5d6e14d47bf96fb455817c1a2f02afc43 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 21 Feb 2025 16:38:22 +0000 Subject: [PATCH 01/28] extending database model to support hdulists rather than just primary hdus --- autofit/database/model/array.py | 67 ++++++++++++++++++++++++++++++++- autofit/database/model/fit.py | 40 +++++++++++++++++++- 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/autofit/database/model/array.py b/autofit/database/model/array.py index fc665c383..51fffd0a7 100644 --- a/autofit/database/model/array.py +++ b/autofit/database/model/array.py @@ -69,6 +69,53 @@ def __call__(self, *args, **kwargs): return self.value +class Fits(Object): + """ + A serialised astropy.io.fits.HDUList + """ + + __tablename__ = "fits" + + id = sa.Column( + sa.Integer, + sa.ForeignKey("object.id"), + primary_key=True, + index=True, + ) + + __mapper_args__ = { + "polymorphic_identity": "fits", + } + + hdus = sa.orm.relationship( + "HDU", + back_populates="fits", + foreign_keys="HDU.fits_id", + ) + + fit_id = sa.Column(sa.String, sa.ForeignKey("fit.id")) + fit = sa.orm.relationship( + "Fit", + uselist=False, + foreign_keys=[fit_id], + back_populates="fits", + ) + + @property + def hdu_list(self): + from astropy.io import fits + + return fits.HDUList([hdu.hdu for hdu in self.hdus]) + + @hdu_list.setter + def hdu_list(self, hdu_list): + self.hdus = [HDU(hdu=hdu) for hdu in hdu_list] + + @property + def value(self): + return self.hdu_list + + class HDU(Array): """ A serialised astropy.io.fits.PrimaryHDU @@ -89,6 +136,8 @@ class HDU(Array): "polymorphic_identity": "hdu", } + is_primary = sa.Column(sa.Boolean) + fit = sa.orm.relationship( "Fit", uselist=False, @@ -96,6 +145,17 @@ class HDU(Array): back_populates="hdus", ) + fits_id = sa.Column( + sa.Integer, + sa.ForeignKey("fits.id"), + ) + fits = sa.orm.relationship( + "Fits", + uselist=False, + foreign_keys=[fits_id], + back_populates="hdus", + ) + @property def header(self): """ @@ -113,13 +173,18 @@ def header(self, header): def hdu(self): from astropy.io import fits - return fits.PrimaryHDU( + type_ = fits.PrimaryHDU if self.is_primary else fits.ImageHDU + + return type_( self.array, self.header, ) @hdu.setter def hdu(self, hdu): + from astropy.io import fits + + self.is_primary = isinstance(hdu, fits.PrimaryHDU) self.array = hdu.data self.header = hdu.header diff --git a/autofit/database/model/fit.py b/autofit/database/model/fit.py index 60f2c9c6d..473dd1530 100644 --- a/autofit/database/model/fit.py +++ b/autofit/database/model/fit.py @@ -11,7 +11,7 @@ from autofit.non_linear.samples import Samples from .model import Base, Object from ..sqlalchemy_ import sa -from .array import Array, HDU +from .array import Array, HDU, Fits from ...aggregator import fit_interface from ...non_linear.samples.efficient import EfficientSamples @@ -337,6 +337,11 @@ def model(self, model: AbstractPriorModel): lazy="joined", foreign_keys=[HDU.fit_id], ) + fits: Mapped[List[Fits]] = sa.orm.relationship( + "Fits", + lazy="joined", + foreign_keys=[Fits.fit_id], + ) def __getitem__(self, item: str): """ @@ -470,6 +475,39 @@ def get_hdu(self, key: str): return p.hdu raise KeyError(f"HDU {key} not found") + def set_fits(self, key: str, value): + """ + Add a fits object to the database. Overwrites any existing fits + with the same name. + + Parameters + ---------- + key + The name of the fits + value + A fits HDUList + """ + new = Fits(name=key, hdu_list=value) + self.fits = [p for p in self.fits if p.name != key] + [new] + + def get_fits(self, key: str): + """ + Retrieve a fits object from the database. + + Parameters + ---------- + key + The name of the fits + + Returns + ------- + A fits HDUList + """ + for p in self.fits: + if p.name == key: + return p.hdu_list + raise KeyError(f"Fits {key} not found") + def __contains__(self, item): for i in self.pickles + self.jsons + self.arrays + self.hdus: if i.name == item: From fd6e60471e7f2e86ec1733a74cc1d727a347bdc6 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 21 Feb 2025 16:44:59 +0000 Subject: [PATCH 02/28] add migration steps --- autofit/database/migration/steps.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/autofit/database/migration/steps.py b/autofit/database/migration/steps.py index ac992d35f..3dea27ed1 100644 --- a/autofit/database/migration/steps.py +++ b/autofit/database/migration/steps.py @@ -23,6 +23,15 @@ Step( "ALTER TABLE object RENAME COLUMN latent_variables_for_id TO latent_samples_for_id;", ), + Step( + "CREATE TABLE fits (id INTEGER NOT NULL, name VARCHAR, fit_id VARCHAR, PRIMARY KEY (id), FOREIGN KEY (fit_id) REFERENCES fit (id));" + ), + Step( + "ALTER TABLE hdu ADD COLUMN fits_id INTEGER;", + ), + Step( + "ALTER TABLE hdu ADD COLUMN is_primary BOOLEAN;", + ), ] migrator = Migrator(*steps) From 44f15718cde11a1f1a00dcc3a089cdf0c217241e Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 21 Feb 2025 16:51:20 +0000 Subject: [PATCH 03/28] update scraper and search outputs to use fits --- autofit/aggregator/file_output.py | 20 +++----------------- autofit/aggregator/search_output.py | 4 ++-- autofit/database/aggregator/scrape.py | 4 ++-- autofit/non_linear/paths/abstract.py | 2 +- autofit/non_linear/paths/database.py | 8 ++++---- autofit/non_linear/paths/directory.py | 8 ++++---- test_autofit/aggregator/test_scrape.py | 2 +- test_autofit/database/test_file_types.py | 7 ------- 8 files changed, 17 insertions(+), 38 deletions(-) diff --git a/autofit/aggregator/file_output.py b/autofit/aggregator/file_output.py index 0cdb4dd97..88b5e2058 100644 --- a/autofit/aggregator/file_output.py +++ b/autofit/aggregator/file_output.py @@ -20,7 +20,7 @@ def __new__(cls, name, path: Path): elif suffix == ".csv": return super().__new__(ArrayOutput) elif suffix == ".fits": - return super().__new__(HDUOutput) + return super().__new__(FITSOutput) raise ValueError(f"File {path} is not a valid output file") def __init__(self, name: str, path: Path): @@ -92,17 +92,7 @@ def value(self): return dill.load(f) -class HDUOutput(FileOutput): - def __init__(self, name: str, path: Path): - super().__init__(name, path) - self._file = None - - @property - def file(self): - if self._file is None: - self._file = open(self.path, "rb") - return self._file - +class FITSOutput(FileOutput): @property def value(self): """ @@ -110,8 +100,4 @@ def value(self): """ from astropy.io import fits - return fits.PrimaryHDU.readfrom(self.file) - - def __del__(self): - if self._file is not None: - self._file.close() + return fits.open(self.path) diff --git a/autofit/aggregator/search_output.py b/autofit/aggregator/search_output.py index 9c1a6e2bd..13a8f3818 100644 --- a/autofit/aggregator/search_output.py +++ b/autofit/aggregator/search_output.py @@ -108,7 +108,7 @@ def pickles(self): return self._outputs(".pickle") @cached_property - def hdus(self): + def fits(self): """ The fits files in the search output files directory """ @@ -170,7 +170,7 @@ def value(self, name: str): for item in self.jsons: if item.name == name: return item.value_using_reference(self._reference) - for item in self.pickles + self.arrays + self.hdus: + for item in self.pickles + self.arrays + self.fits: if item.name == name: return item.value diff --git a/autofit/database/aggregator/scrape.py b/autofit/database/aggregator/scrape.py index 726dfb052..2e3bf8e15 100644 --- a/autofit/database/aggregator/scrape.py +++ b/autofit/database/aggregator/scrape.py @@ -183,5 +183,5 @@ def _add_files(fit: m.Fit, item: SearchOutput): except ValueError: logger.debug(f"Failed to load array {array_output.name} for {fit.id}") - for hdu_output in item.hdus: - fit.set_hdu(hdu_output.name, hdu_output.value) + for fits in item.fits: + fit.set_fits(fits.name, fits.value) diff --git a/autofit/non_linear/paths/abstract.py b/autofit/non_linear/paths/abstract.py index f57fabede..9016a8dd1 100644 --- a/autofit/non_linear/paths/abstract.py +++ b/autofit/non_linear/paths/abstract.py @@ -397,7 +397,7 @@ def load_array(self, name) -> np.ndarray: pass @abstractmethod - def save_fits(self, name: str, hdu, prefix: str = ""): + def save_fits(self, name: str, fits, prefix: str = ""): pass @abstractmethod diff --git a/autofit/non_linear/paths/database.py b/autofit/non_linear/paths/database.py index 0a48989fb..c57f48a02 100644 --- a/autofit/non_linear/paths/database.py +++ b/autofit/non_linear/paths/database.py @@ -183,7 +183,7 @@ def load_array(self, name: str) -> np.ndarray: return self.fit.get_array(name) @conditional_output - def save_fits(self, name: str, hdu, prefix: str = ""): + def save_fits(self, name: str, fits, prefix: str = ""): """ Save a fits file in the database @@ -191,10 +191,10 @@ def save_fits(self, name: str, hdu, prefix: str = ""): ---------- name The name of the fits file - hdu - The hdu to save + fits + The fits file to save """ - self.fit.set_hdu(name, hdu) + self.fit.set_fits(name, fits) def load_fits(self, name: str, prefix: str = ""): """ diff --git a/autofit/non_linear/paths/directory.py b/autofit/non_linear/paths/directory.py index 9b1383cea..109a2f89f 100644 --- a/autofit/non_linear/paths/directory.py +++ b/autofit/non_linear/paths/directory.py @@ -103,7 +103,7 @@ def load_array(self, name: str): return np.loadtxt(self._path_for_csv(name), delimiter=",") @conditional_output - def save_fits(self, name: str, hdu, prefix: str = ""): + def save_fits(self, name: str, fits, prefix: str = ""): """ Save an HDU as a fits file in the fits directory of the search. @@ -111,12 +111,12 @@ def save_fits(self, name: str, hdu, prefix: str = ""): ---------- name The name of the fits file - hdu - The HDU to save + fits + The HDUList to save prefix A prefix to add to the path which is the name of the folder the file is saved in. """ - hdu.writeto(self._path_for_fits(name, prefix), overwrite=True) + fits.writeto(self._path_for_fits(name, prefix), overwrite=True) def load_fits(self, name: str, prefix: str = ""): """ diff --git a/test_autofit/aggregator/test_scrape.py b/test_autofit/aggregator/test_scrape.py index bea3b7615..5033e7739 100644 --- a/test_autofit/aggregator/test_scrape.py +++ b/test_autofit/aggregator/test_scrape.py @@ -22,7 +22,7 @@ def set_pickle(self, key, value): def set_array(self, key, value): pass - def set_hdu(self, key, value): + def set_fits(self, key, value): pass diff --git a/test_autofit/database/test_file_types.py b/test_autofit/database/test_file_types.py index a37cb1fbe..8a162e822 100644 --- a/test_autofit/database/test_file_types.py +++ b/test_autofit/database/test_file_types.py @@ -54,10 +54,3 @@ def test_hdu(hdu, hdu_array): loaded = db_hdu.hdu assert (loaded.data == hdu_array).all() assert loaded.header == hdu.header - - -def test_set_hdu(fit, hdu, hdu_array): - fit.set_hdu("test", hdu) - loaded = fit.get_hdu("test") - assert (loaded.data == hdu_array).all() - assert loaded.header == hdu.header From c9ad10cf61aab5ba51ab0ce9af3330834d5dfd81 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 21 Feb 2025 16:53:41 +0000 Subject: [PATCH 04/28] remove code and fix --- autofit/database/model/fit.py | 33 ---------------------------- autofit/non_linear/paths/database.py | 2 +- 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/autofit/database/model/fit.py b/autofit/database/model/fit.py index 473dd1530..93b8076c2 100644 --- a/autofit/database/model/fit.py +++ b/autofit/database/model/fit.py @@ -442,39 +442,6 @@ def get_array(self, key: str) -> np.ndarray: return p.array raise KeyError(f"Array {key} not found") - def set_hdu(self, key: str, value): - """ - Add an HDU to the database. Overwrites any existing HDU - with the same name. - - Parameters - ---------- - key - The name of the HDU - value - A fits HDUList - """ - new = HDU(name=key, hdu=value) - self.hdus = [p for p in self.hdus if p.name != key] + [new] - - def get_hdu(self, key: str): - """ - Retrieve an HDU from the database. - - Parameters - ---------- - key - The name of the HDU - - Returns - ------- - A fits HDUList - """ - for p in self.hdus: - if p.name == key: - return p.hdu - raise KeyError(f"HDU {key} not found") - def set_fits(self, key: str, value): """ Add a fits object to the database. Overwrites any existing fits diff --git a/autofit/non_linear/paths/database.py b/autofit/non_linear/paths/database.py index c57f48a02..5845a1442 100644 --- a/autofit/non_linear/paths/database.py +++ b/autofit/non_linear/paths/database.py @@ -209,7 +209,7 @@ def load_fits(self, name: str, prefix: str = ""): ------- The loaded hdu """ - return self.fit.get_hdu(name) + return self.fit.get_fits(name) @conditional_output def save_object(self, name: str, obj: object, prefix: str = ""): From b3281a54e6a6ae3919346b726207f03791afad11 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 08:51:08 +0000 Subject: [PATCH 05/28] slow test --- .../non_linear/grid/test_sensitivity/test_masked_sensitivity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_autofit/non_linear/grid/test_sensitivity/test_masked_sensitivity.py b/test_autofit/non_linear/grid/test_sensitivity/test_masked_sensitivity.py index ed90f1822..a5ae65729 100644 --- a/test_autofit/non_linear/grid/test_sensitivity/test_masked_sensitivity.py +++ b/test_autofit/non_linear/grid/test_sensitivity/test_masked_sensitivity.py @@ -71,7 +71,7 @@ def test_perturbed_physical_centres_list_from(masked_result): ] -def test_visualise(sensitivity): +def _test_visualise(sensitivity): def visualiser(sensitivity_result, **_): assert len(sensitivity_result.samples) == 8 From 1b35670e4482687ff0e15c30a4268db92883bd0b Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 09:03:38 +0000 Subject: [PATCH 06/28] fits files for testing --- .../aggregate_summary/fit_1/image/fit.fits | Bin 0 -> 23040 bytes .../aggregate_summary/fit_2/image/fit.fits | Bin 0 -> 23040 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test_autofit/aggregator/aggregate_summary/fit_1/image/fit.fits create mode 100644 test_autofit/aggregator/aggregate_summary/fit_2/image/fit.fits diff --git a/test_autofit/aggregator/aggregate_summary/fit_1/image/fit.fits b/test_autofit/aggregator/aggregate_summary/fit_1/image/fit.fits new file mode 100644 index 0000000000000000000000000000000000000000..ff6df34909e41941b75e398462741a8b872057f5 GIT binary patch literal 23040 zcmeI4c{r6_zsC_tsmuw9@-&f9A*tV%A%sk2d@{>SLP~>quFSIxiIQX<%QDZi%=5I( zGZ_-c`@YBXJnwtz)aiQvIp^&C&+oR@y1(Cbf7afYdw=(REjcksX$fH_rb|rUc*!%J zVbU@+(KR(UzRP4`$|NEtFUNG(Lc>H`!(96tenjr&a6 z8WtK%7Wd6`zOC|StCWDEnB3N%-<%)UmY#{Fv8IkWld0~uvc9p7$z6R@le>S4{N?EH zdYs?Zm$KRKzt4BJ`^IY9xXEq=#2*tf2G>vjAu z-T&jmf9NUx+w``sS4u!q_-}fW*M)>7)WjqOt_uJ96#owIe|%NA6L17@1aJg!1aJg! z1aJg?o&eqtUf?kIXg~2v2)vV-+I9H|2V`s<8utk7g6vmRZ;Z|{L*}jLT8{p!kdWT| zHk9HZ_{KAcku$CQK=y~-kY3cxsrF$flvzDJ;Vd2i6_u>5%}MJ}5sWQu(@uhNo=(2w zS4^Q~xWi!cSOXN19&!s%y9D_ZY?)S!yCJPYJ;LvT7Py(5b5Pk)`9s718~_DfA3MZ+ zv!JfJg@^m>2sF%|A|U7&g{Dj8FD|P0K$GPpDRHYEG(I9`^_xBi^-qJdBotDh`ngxk?y)nBcCpZekP*Y^WASpI7Cjgi zE-W8Nih;gQk4r_P2B2L=Qu9FfduWXHD$r*2gepaqP{KwLD5hsnZ_iqUB=$p&L>CqR z@K3*a#a91x88q&6iatjB7;WmIG9z<_*)hWLyXESzB*-02 z9o+?=wXYCh_q)S5Cvr zvpk?R_gsjBf&kP@$CPN8~ETXWG=*}N-F$hL=R*TtY*`T+#KV+1W5Sm?z z?l7$1hnmWO{9NZuh%4D=ZKvxFscbt=E%=i`c`;>M+t)s52;?wS@RNXc>-DQao%}G+ zdE>zYs|=X*JA7P=?>;OwecnH>;RPGd=uVaUtH5UQlwW?6KCIm7Zn+of2(z6Cuw;M%ow73VD@-to4NY{bYl9i8F{*4*4w(JWlG8Xj3e&D0 zh2-bI3;5p2I_t6rb6yRXT3zu$tW54Bpa(n(=!7lij-rD)l%F3+Z8gW=yOO? zf}mJ8s<7R92CA+l%URLVKqGrtH(u8gw6;SIjc+aVaPu;P3mNpUTotSlF@(V<<2vg^ zelWN{ORvJ)8~WDeHJ=}zhpwvCnbNt>hLW2yKc^R-ez+L(4e3h1Jt*@aeii_%puwt@pKJo}4imK7ml+4AevS0qkBfeVE*3snr1NS6iSsmDVF7q*jF_A6A!B_8K9v4A?#;)|0V zX;9a9;~kIB5Y&DZYe|Whg6ddycX5Gr$a7L<4C@h_RmuFDW02V}@kc%dSC*nvUmFDFevMp|4C|k$~9LjX1S+ zy6@@#YCCv*;A!?Hu7&WqqdD}u_#u+aXhI&chMF5%_fie~UZi#I8$i<$r%q z{#{{$r55F7y<0UQAw0UUw|y4yaX zH`&U8?o=jH@+CAPm#gOY6%-VZGqt-1jiMxSH71rld@Bq&Mc*p#)v-qIm2)>^SnZH} zapC(t;>ADm+wMpBoE||PZFtC9R-;o=Ndj@IVw_(UtAlyXMs47w4e+2*$IF}Z1?OmL zAyus?^pJx~o!Bu1*{f~FDHIsP6@AIV!h@0M8Bgrn3VvpEM~v=l-cB8)GxIT6#DQts z=U*#wFp5qOcj5%IfG)4uE=J_A$AkOA&U~;V3l&_Q&<0l@VWwk>55T*})1SJS2|VM) z?}-xGf=lqr!%O8%5ERa5yu7a*9F)3F>J1+S<5a9+kN_EStGD1=KhTe?avbOAD+iJK zRRwL@=0A9V5-~PhW%gSim!#>M01~B_Vn0 zIsmb+!d*)T%^;T4h;ZX>AVhK9S!l`kfEO>tQt&JI!7oK-HGJa}C>$Z(D=;CAbSzj! zOzD`AU3!1J+LNMyooAe5q|qZw?$c+OPHp>x1!v&|e}V}}IdVjgG|Q&v+mqo-xN% zHEDF`G@+c;K~L}{-`IGY76Sg-6frVFyVD+!BZ*q%gitI;C4ordLdc< z_WC0|rhs6Ks5A;Bqr4>UqJ=CS6?>n!4jp0?g}|Cx+#Bm91ktvdx$|}e5MiSI zEct3ZNb3H6##J2yTBQlrPpp!_>%c{ZzMV8%xuvmkaIzi*SKY4^QLO}9?UA4t(dO**G9aiM%f=RzK^o2Ki=9)k1aKyt)D&|2Y$1mVlepe+$p%r5Izn0rJn}Okpo|6}oUxO`C z^Vq=#dE_uGo)Izs5t%a$>u2u%FZnmp9^Ik9>dt~}`?v(f1VfPjn!&9w(E#LQeB_WJ zrUYGk?=;#-;f3tqY6V}m#Ddv5xjkZW^YA;S{JzfVdI&yh&lSiQg52yr-}22nh2)GT zhS(~2kTOicD_lmETjhLOy-AgBgW($nkYYra_}TvhExq z+tD_Q>{sO8w+?QAt*_~C&DG_|c|afp^!<@t;Ppu;7n;CA`%?XfO&4-w%i&Dt0 zyv6yrWCJoVl-8VSG(yiC7t;$LiXfBMB1Z&iIKHReqs8Ry!hY!{4*|p{-@tR{#=}&Jo?M?)Kb@FB?TnJ zl!b-Vem#oD<&)4fnh>B0Tc zHbbb?saV{mw+2;+uku=!UO=5oj04RVT~J4))8YOH)TsOSGZ7*r1E`lbSFZSk5gPo$ zmh++mi{b_1=Ci9~e&n~^j{?%U11K_MPz^I(%sq!7l&hq2OpqlKGV=p&oRV*WPtOtv z@ZV#gUIy>&C-uFkadFZ9psFlt$<UvZ)G^df} zF9$I}!xNg`dmX!>R?J9=opToY!}gOJq{KkYhvBbYOrnrQ;TwPM_IcDNL78Ina1PZg z1=Ho}@S?QNv!!yLgWLWPhiN<(=b;A`5j#j2ZnC4Qlrjy|ITiSrM4?h3a-0G*?(0q%6Awh`2_*v>w`b9(xu=V}*>bl10X<$`qp}|Y#X3RL z&_{Uisfk?n@rU!!JP{}4r1>0*9!Soq=q94RBV4hh3?yh?OFXY3ISaL_ik#X!F%E4y z{9*>DV*%M#FI-&RdJZ;K%z*hWB}@e8_7w{HLyhU4jrrK8s40WJcycZW4Sfn#psn*p z0n|--r_;1i33o$`h28@6C>#(wR>ulG1lQI++VMi(0h>&rwinR1=V8)*6=T%T9zz)v zG?|ZA+vG7p_5nk%BIi@chDD#M7A}<^yNkv~M0ey!VxiZbmR)6p8wRLzNH&QzpoYXj zcYPBJt<%3Zda6hbdx{uv<+(Iyd@{{{wf*|wI-LY(L*B! zA2oeAXizc{{~P14D(F+2HP0313<$PlexOtzg#1sWrFHQRK>zcmfh!aasI?+BOzVIJ zN>cjrHfHVw>K|R)*YRNl<>f?5YKgIJ`+cvX5}OYX_lIYqal)=o88IQK)8|Cxf#qIQ zN1h}+HoFtGG}Wz3KU#+}<23d56n50=wf<#^loU0qMv2pGeO_Wi#)Qu(3!>~3Cp#rh z;-M64Z?;26DZl6Uy>?WUWuG;wv4Ludln0Me1fbTDeE;h^FQXm{4Ka3zLOrE>_ue{l z4Yj^keEH5v7L{~;e8^gV5e?j*2yegMfC~Banp@ZzzNi1I?I=D|yKj8rCHl0`{6JCk zG4hMod0%5i@vC~c^Kb-k1b&?WE`R>>%Mvbs{y+8nv!JM$nw-oH0oknp`j;Pp#^uj{ zem-zVI085VI085VI085V{~++AJ8y`vFihdEp@IHrEH&#}1DKnc<&oi(hPM1(>Q9tG zup}-zAiy&U3z8a~p{yxTX;zjoHY@NW-5>SC+-2R4)b*V(*K)mVfUg<`hm|f4t`WkF zi-QqGmOY!wbWO4_`L@ ze&n~^4}A*DlQO~n@+;glRAZW_-wHH#ff!rnIA@Cxy3_?QejR-CZuSGC?vN=mK*WMZ~Odfg)WCR zUM%4`=p{-cxHFyw3(j2!FMDmkw4{kNV z3fuk=Ih?HX+~*+lTV!GTVv_*7u_;7AunUH06CDT}wsNk9lDig!xF9F z*NX6)u(75IFw($|@mPkROR>7Pp6GYO^fSOL&0A#^+R|&ur~~p%j8W%?O*(R}O`L{RGQXOM9!| z^uU;=#-rd!LYR_laeHC!4@*h!+S}B}|M2h2-mO2+;{hEfpL0bPvcTNih;D;-GcY#v zeA4%J4}7`g+U|Zi5x%%h)+^RCVentSzMPQ92y-S4W}{~b;ESSBql8K!tUtge_(&YV z5IlZ1{+nbJEMGiMujoh#{i6eq-QTps#0~OeRYD(NBT<90h9eLxMonIIvSMJkRZUar z+$_xU((v?-?S&bd7LB7iNid%u{m6hU5T=s2X7}j<%#mw++?Q94p*o}=Z`UY~VQW8q z^=7p#X5U#Fz2|al7{bPw9dGzYVW#3F>$ut(SW=C58@;p#vm>2dsc36|h-HgqmqEV= z@KuqVF#WJ1422XLF7-^pL}iz9@{Mtrf8I~9ce5U*?{(IHbSZhoK-Hb`r!lfwd4m3te(u7`=2HYh^nPpXmmu zs1_Gt;c@9LidYwzGisEaek2b4M74%~{v|M27Iw{sY-^v6CBwZgYAi4^@95mNXD_U_ z^V!k*yoJ6cS&mnQXJOe-MwgI_5tb04xxUIVnAH@ul-ZRAi|oABp%+5nv;JMa$L{j5 z;Td*vq9q1K7L%T-oSTP*z&SSR&Lc3}F@3~OMjw`J?D`XiyJ4j8m2s`4;2-P%@nxs? z5aBKiz7Hd@?%Na?(srRB-`eM4)~2Zx=yISPZ!JCdSEVBX}t_=|+SFu|8=W?#Vu3ul})7D|?2k!NFVARrb-mgA@89UP&L z>OKC!hl|_(+ZD!$eaz@*jbSNv!fVWPKaBd5+6X?2g-&{{{pWbRVK_#FzR7|dzIIW^ z%TOr5T=wkaLh&`2h{t3HyY|9dgO1*-t7coDzq<1buSj7jxvs}W*c^tu&SHgomSC(j zrw%_m3o_3~emTN5z3sbQp&+n5s%)4OCR=O010&wR7w6v@B|1znJ3pu&h?Upe+SlX6 z1>5)qSgcYBrud2nOB|V}WZa2hacqyKtG7L@sk0ijpLT`KfwbEZ_hexpXxY^GTnWsk zY@jV4?*`epb^w!V%{p>hMeY`}^>#7MOY!m8)Upyfb#MyO8J%09l+&CNo K9D#oyf&Tyi5){_} literal 0 HcmV?d00001 diff --git a/test_autofit/aggregator/aggregate_summary/fit_2/image/fit.fits b/test_autofit/aggregator/aggregate_summary/fit_2/image/fit.fits new file mode 100644 index 0000000000000000000000000000000000000000..ff6df34909e41941b75e398462741a8b872057f5 GIT binary patch literal 23040 zcmeI4c{r6_zsC_tsmuw9@-&f9A*tV%A%sk2d@{>SLP~>quFSIxiIQX<%QDZi%=5I( zGZ_-c`@YBXJnwtz)aiQvIp^&C&+oR@y1(Cbf7afYdw=(REjcksX$fH_rb|rUc*!%J zVbU@+(KR(UzRP4`$|NEtFUNG(Lc>H`!(96tenjr&a6 z8WtK%7Wd6`zOC|StCWDEnB3N%-<%)UmY#{Fv8IkWld0~uvc9p7$z6R@le>S4{N?EH zdYs?Zm$KRKzt4BJ`^IY9xXEq=#2*tf2G>vjAu z-T&jmf9NUx+w``sS4u!q_-}fW*M)>7)WjqOt_uJ96#owIe|%NA6L17@1aJg!1aJg! z1aJg?o&eqtUf?kIXg~2v2)vV-+I9H|2V`s<8utk7g6vmRZ;Z|{L*}jLT8{p!kdWT| zHk9HZ_{KAcku$CQK=y~-kY3cxsrF$flvzDJ;Vd2i6_u>5%}MJ}5sWQu(@uhNo=(2w zS4^Q~xWi!cSOXN19&!s%y9D_ZY?)S!yCJPYJ;LvT7Py(5b5Pk)`9s718~_DfA3MZ+ zv!JfJg@^m>2sF%|A|U7&g{Dj8FD|P0K$GPpDRHYEG(I9`^_xBi^-qJdBotDh`ngxk?y)nBcCpZekP*Y^WASpI7Cjgi zE-W8Nih;gQk4r_P2B2L=Qu9FfduWXHD$r*2gepaqP{KwLD5hsnZ_iqUB=$p&L>CqR z@K3*a#a91x88q&6iatjB7;WmIG9z<_*)hWLyXESzB*-02 z9o+?=wXYCh_q)S5Cvr zvpk?R_gsjBf&kP@$CPN8~ETXWG=*}N-F$hL=R*TtY*`T+#KV+1W5Sm?z z?l7$1hnmWO{9NZuh%4D=ZKvxFscbt=E%=i`c`;>M+t)s52;?wS@RNXc>-DQao%}G+ zdE>zYs|=X*JA7P=?>;OwecnH>;RPGd=uVaUtH5UQlwW?6KCIm7Zn+of2(z6Cuw;M%ow73VD@-to4NY{bYl9i8F{*4*4w(JWlG8Xj3e&D0 zh2-bI3;5p2I_t6rb6yRXT3zu$tW54Bpa(n(=!7lij-rD)l%F3+Z8gW=yOO? zf}mJ8s<7R92CA+l%URLVKqGrtH(u8gw6;SIjc+aVaPu;P3mNpUTotSlF@(V<<2vg^ zelWN{ORvJ)8~WDeHJ=}zhpwvCnbNt>hLW2yKc^R-ez+L(4e3h1Jt*@aeii_%puwt@pKJo}4imK7ml+4AevS0qkBfeVE*3snr1NS6iSsmDVF7q*jF_A6A!B_8K9v4A?#;)|0V zX;9a9;~kIB5Y&DZYe|Whg6ddycX5Gr$a7L<4C@h_RmuFDW02V}@kc%dSC*nvUmFDFevMp|4C|k$~9LjX1S+ zy6@@#YCCv*;A!?Hu7&WqqdD}u_#u+aXhI&chMF5%_fie~UZi#I8$i<$r%q z{#{{$r55F7y<0UQAw0UUw|y4yaX zH`&U8?o=jH@+CAPm#gOY6%-VZGqt-1jiMxSH71rld@Bq&Mc*p#)v-qIm2)>^SnZH} zapC(t;>ADm+wMpBoE||PZFtC9R-;o=Ndj@IVw_(UtAlyXMs47w4e+2*$IF}Z1?OmL zAyus?^pJx~o!Bu1*{f~FDHIsP6@AIV!h@0M8Bgrn3VvpEM~v=l-cB8)GxIT6#DQts z=U*#wFp5qOcj5%IfG)4uE=J_A$AkOA&U~;V3l&_Q&<0l@VWwk>55T*})1SJS2|VM) z?}-xGf=lqr!%O8%5ERa5yu7a*9F)3F>J1+S<5a9+kN_EStGD1=KhTe?avbOAD+iJK zRRwL@=0A9V5-~PhW%gSim!#>M01~B_Vn0 zIsmb+!d*)T%^;T4h;ZX>AVhK9S!l`kfEO>tQt&JI!7oK-HGJa}C>$Z(D=;CAbSzj! zOzD`AU3!1J+LNMyooAe5q|qZw?$c+OPHp>x1!v&|e}V}}IdVjgG|Q&v+mqo-xN% zHEDF`G@+c;K~L}{-`IGY76Sg-6frVFyVD+!BZ*q%gitI;C4ordLdc< z_WC0|rhs6Ks5A;Bqr4>UqJ=CS6?>n!4jp0?g}|Cx+#Bm91ktvdx$|}e5MiSI zEct3ZNb3H6##J2yTBQlrPpp!_>%c{ZzMV8%xuvmkaIzi*SKY4^QLO}9?UA4t(dO**G9aiM%f=RzK^o2Ki=9)k1aKyt)D&|2Y$1mVlepe+$p%r5Izn0rJn}Okpo|6}oUxO`C z^Vq=#dE_uGo)Izs5t%a$>u2u%FZnmp9^Ik9>dt~}`?v(f1VfPjn!&9w(E#LQeB_WJ zrUYGk?=;#-;f3tqY6V}m#Ddv5xjkZW^YA;S{JzfVdI&yh&lSiQg52yr-}22nh2)GT zhS(~2kTOicD_lmETjhLOy-AgBgW($nkYYra_}TvhExq z+tD_Q>{sO8w+?QAt*_~C&DG_|c|afp^!<@t;Ppu;7n;CA`%?XfO&4-w%i&Dt0 zyv6yrWCJoVl-8VSG(yiC7t;$LiXfBMB1Z&iIKHReqs8Ry!hY!{4*|p{-@tR{#=}&Jo?M?)Kb@FB?TnJ zl!b-Vem#oD<&)4fnh>B0Tc zHbbb?saV{mw+2;+uku=!UO=5oj04RVT~J4))8YOH)TsOSGZ7*r1E`lbSFZSk5gPo$ zmh++mi{b_1=Ci9~e&n~^j{?%U11K_MPz^I(%sq!7l&hq2OpqlKGV=p&oRV*WPtOtv z@ZV#gUIy>&C-uFkadFZ9psFlt$<UvZ)G^df} zF9$I}!xNg`dmX!>R?J9=opToY!}gOJq{KkYhvBbYOrnrQ;TwPM_IcDNL78Ina1PZg z1=Ho}@S?QNv!!yLgWLWPhiN<(=b;A`5j#j2ZnC4Qlrjy|ITiSrM4?h3a-0G*?(0q%6Awh`2_*v>w`b9(xu=V}*>bl10X<$`qp}|Y#X3RL z&_{Uisfk?n@rU!!JP{}4r1>0*9!Soq=q94RBV4hh3?yh?OFXY3ISaL_ik#X!F%E4y z{9*>DV*%M#FI-&RdJZ;K%z*hWB}@e8_7w{HLyhU4jrrK8s40WJcycZW4Sfn#psn*p z0n|--r_;1i33o$`h28@6C>#(wR>ulG1lQI++VMi(0h>&rwinR1=V8)*6=T%T9zz)v zG?|ZA+vG7p_5nk%BIi@chDD#M7A}<^yNkv~M0ey!VxiZbmR)6p8wRLzNH&QzpoYXj zcYPBJt<%3Zda6hbdx{uv<+(Iyd@{{{wf*|wI-LY(L*B! zA2oeAXizc{{~P14D(F+2HP0313<$PlexOtzg#1sWrFHQRK>zcmfh!aasI?+BOzVIJ zN>cjrHfHVw>K|R)*YRNl<>f?5YKgIJ`+cvX5}OYX_lIYqal)=o88IQK)8|Cxf#qIQ zN1h}+HoFtGG}Wz3KU#+}<23d56n50=wf<#^loU0qMv2pGeO_Wi#)Qu(3!>~3Cp#rh z;-M64Z?;26DZl6Uy>?WUWuG;wv4Ludln0Me1fbTDeE;h^FQXm{4Ka3zLOrE>_ue{l z4Yj^keEH5v7L{~;e8^gV5e?j*2yegMfC~Banp@ZzzNi1I?I=D|yKj8rCHl0`{6JCk zG4hMod0%5i@vC~c^Kb-k1b&?WE`R>>%Mvbs{y+8nv!JM$nw-oH0oknp`j;Pp#^uj{ zem-zVI085VI085VI085V{~++AJ8y`vFihdEp@IHrEH&#}1DKnc<&oi(hPM1(>Q9tG zup}-zAiy&U3z8a~p{yxTX;zjoHY@NW-5>SC+-2R4)b*V(*K)mVfUg<`hm|f4t`WkF zi-QqGmOY!wbWO4_`L@ ze&n~^4}A*DlQO~n@+;glRAZW_-wHH#ff!rnIA@Cxy3_?QejR-CZuSGC?vN=mK*WMZ~Odfg)WCR zUM%4`=p{-cxHFyw3(j2!FMDmkw4{kNV z3fuk=Ih?HX+~*+lTV!GTVv_*7u_;7AunUH06CDT}wsNk9lDig!xF9F z*NX6)u(75IFw($|@mPkROR>7Pp6GYO^fSOL&0A#^+R|&ur~~p%j8W%?O*(R}O`L{RGQXOM9!| z^uU;=#-rd!LYR_laeHC!4@*h!+S}B}|M2h2-mO2+;{hEfpL0bPvcTNih;D;-GcY#v zeA4%J4}7`g+U|Zi5x%%h)+^RCVentSzMPQ92y-S4W}{~b;ESSBql8K!tUtge_(&YV z5IlZ1{+nbJEMGiMujoh#{i6eq-QTps#0~OeRYD(NBT<90h9eLxMonIIvSMJkRZUar z+$_xU((v?-?S&bd7LB7iNid%u{m6hU5T=s2X7}j<%#mw++?Q94p*o}=Z`UY~VQW8q z^=7p#X5U#Fz2|al7{bPw9dGzYVW#3F>$ut(SW=C58@;p#vm>2dsc36|h-HgqmqEV= z@KuqVF#WJ1422XLF7-^pL}iz9@{Mtrf8I~9ce5U*?{(IHbSZhoK-Hb`r!lfwd4m3te(u7`=2HYh^nPpXmmu zs1_Gt;c@9LidYwzGisEaek2b4M74%~{v|M27Iw{sY-^v6CBwZgYAi4^@95mNXD_U_ z^V!k*yoJ6cS&mnQXJOe-MwgI_5tb04xxUIVnAH@ul-ZRAi|oABp%+5nv;JMa$L{j5 z;Td*vq9q1K7L%T-oSTP*z&SSR&Lc3}F@3~OMjw`J?D`XiyJ4j8m2s`4;2-P%@nxs? z5aBKiz7Hd@?%Na?(srRB-`eM4)~2Zx=yISPZ!JCdSEVBX}t_=|+SFu|8=W?#Vu3ul})7D|?2k!NFVARrb-mgA@89UP&L z>OKC!hl|_(+ZD!$eaz@*jbSNv!fVWPKaBd5+6X?2g-&{{{pWbRVK_#FzR7|dzIIW^ z%TOr5T=wkaLh&`2h{t3HyY|9dgO1*-t7coDzq<1buSj7jxvs}W*c^tu&SHgomSC(j zrw%_m3o_3~emTN5z3sbQp&+n5s%)4OCR=O010&wR7w6v@B|1znJ3pu&h?Upe+SlX6 z1>5)qSgcYBrud2nOB|V}WZa2hacqyKtG7L@sk0ijpLT`KfwbEZ_hexpXxY^GTnWsk zY@jV4?*`epb^w!V%{p>hMeY`}^>#7MOY!m8)Upyfb#MyO8J%09l+&CNo K9D#oyf&Tyi5){_} literal 0 HcmV?d00001 From 242b6be883d01812949506842a8bb8c02679383d Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 09:17:39 +0000 Subject: [PATCH 07/28] refactor --- autofit/aggregator/aggregate_fits.py | 3 +++ .../fit_1/metadata => summary_files/__init__.py} | 0 .../fit_1/files/latent/latent_summary.json | 0 .../aggregate_summary/fit_1/files/model.json | 0 .../aggregate_summary/fit_1/files/samples.json | 0 .../fit_1/files/samples_summary.json | 0 .../aggregate_summary/fit_1/image/fit.fits | Bin .../aggregate_summary/fit_1/image/subplot_fit.png | Bin .../aggregate_summary/fit_1}/metadata | 0 .../fit_2/files/latent/latent_summary.json | 0 .../aggregate_summary/fit_2/files/model.json | 0 .../aggregate_summary/fit_2/files/samples.json | 0 .../fit_2/files/samples_summary.json | 0 .../aggregate_summary/fit_2/image/fit.fits | Bin .../aggregate_summary/fit_2/image/subplot_fit.png | Bin .../summary_files/aggregate_summary/fit_2/metadata | 0 .../{ => summary_files}/test_aggregate_csv.py | 0 .../aggregator/summary_files/test_aggregate_fits.py | 2 ++ .../{ => summary_files}/test_aggregate_images.py | 0 19 files changed, 5 insertions(+) create mode 100644 autofit/aggregator/aggregate_fits.py rename test_autofit/aggregator/{aggregate_summary/fit_1/metadata => summary_files/__init__.py} (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_1/files/latent/latent_summary.json (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_1/files/model.json (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_1/files/samples.json (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_1/files/samples_summary.json (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_1/image/fit.fits (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_1/image/subplot_fit.png (100%) rename test_autofit/aggregator/{aggregate_summary/fit_2 => summary_files/aggregate_summary/fit_1}/metadata (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_2/files/latent/latent_summary.json (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_2/files/model.json (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_2/files/samples.json (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_2/files/samples_summary.json (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_2/image/fit.fits (100%) rename test_autofit/aggregator/{ => summary_files}/aggregate_summary/fit_2/image/subplot_fit.png (100%) create mode 100644 test_autofit/aggregator/summary_files/aggregate_summary/fit_2/metadata rename test_autofit/aggregator/{ => summary_files}/test_aggregate_csv.py (100%) create mode 100644 test_autofit/aggregator/summary_files/test_aggregate_fits.py rename test_autofit/aggregator/{ => summary_files}/test_aggregate_images.py (100%) diff --git a/autofit/aggregator/aggregate_fits.py b/autofit/aggregator/aggregate_fits.py new file mode 100644 index 000000000..5136abc12 --- /dev/null +++ b/autofit/aggregator/aggregate_fits.py @@ -0,0 +1,3 @@ +class AggregateFits: + def __init__(self): + pass diff --git a/test_autofit/aggregator/aggregate_summary/fit_1/metadata b/test_autofit/aggregator/summary_files/__init__.py similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_1/metadata rename to test_autofit/aggregator/summary_files/__init__.py diff --git a/test_autofit/aggregator/aggregate_summary/fit_1/files/latent/latent_summary.json b/test_autofit/aggregator/summary_files/aggregate_summary/fit_1/files/latent/latent_summary.json similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_1/files/latent/latent_summary.json rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_1/files/latent/latent_summary.json diff --git a/test_autofit/aggregator/aggregate_summary/fit_1/files/model.json b/test_autofit/aggregator/summary_files/aggregate_summary/fit_1/files/model.json similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_1/files/model.json rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_1/files/model.json diff --git a/test_autofit/aggregator/aggregate_summary/fit_1/files/samples.json b/test_autofit/aggregator/summary_files/aggregate_summary/fit_1/files/samples.json similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_1/files/samples.json rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_1/files/samples.json diff --git a/test_autofit/aggregator/aggregate_summary/fit_1/files/samples_summary.json b/test_autofit/aggregator/summary_files/aggregate_summary/fit_1/files/samples_summary.json similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_1/files/samples_summary.json rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_1/files/samples_summary.json diff --git a/test_autofit/aggregator/aggregate_summary/fit_1/image/fit.fits b/test_autofit/aggregator/summary_files/aggregate_summary/fit_1/image/fit.fits similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_1/image/fit.fits rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_1/image/fit.fits diff --git a/test_autofit/aggregator/aggregate_summary/fit_1/image/subplot_fit.png b/test_autofit/aggregator/summary_files/aggregate_summary/fit_1/image/subplot_fit.png similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_1/image/subplot_fit.png rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_1/image/subplot_fit.png diff --git a/test_autofit/aggregator/aggregate_summary/fit_2/metadata b/test_autofit/aggregator/summary_files/aggregate_summary/fit_1/metadata similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_2/metadata rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_1/metadata diff --git a/test_autofit/aggregator/aggregate_summary/fit_2/files/latent/latent_summary.json b/test_autofit/aggregator/summary_files/aggregate_summary/fit_2/files/latent/latent_summary.json similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_2/files/latent/latent_summary.json rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_2/files/latent/latent_summary.json diff --git a/test_autofit/aggregator/aggregate_summary/fit_2/files/model.json b/test_autofit/aggregator/summary_files/aggregate_summary/fit_2/files/model.json similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_2/files/model.json rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_2/files/model.json diff --git a/test_autofit/aggregator/aggregate_summary/fit_2/files/samples.json b/test_autofit/aggregator/summary_files/aggregate_summary/fit_2/files/samples.json similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_2/files/samples.json rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_2/files/samples.json diff --git a/test_autofit/aggregator/aggregate_summary/fit_2/files/samples_summary.json b/test_autofit/aggregator/summary_files/aggregate_summary/fit_2/files/samples_summary.json similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_2/files/samples_summary.json rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_2/files/samples_summary.json diff --git a/test_autofit/aggregator/aggregate_summary/fit_2/image/fit.fits b/test_autofit/aggregator/summary_files/aggregate_summary/fit_2/image/fit.fits similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_2/image/fit.fits rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_2/image/fit.fits diff --git a/test_autofit/aggregator/aggregate_summary/fit_2/image/subplot_fit.png b/test_autofit/aggregator/summary_files/aggregate_summary/fit_2/image/subplot_fit.png similarity index 100% rename from test_autofit/aggregator/aggregate_summary/fit_2/image/subplot_fit.png rename to test_autofit/aggregator/summary_files/aggregate_summary/fit_2/image/subplot_fit.png diff --git a/test_autofit/aggregator/summary_files/aggregate_summary/fit_2/metadata b/test_autofit/aggregator/summary_files/aggregate_summary/fit_2/metadata new file mode 100644 index 000000000..e69de29bb diff --git a/test_autofit/aggregator/test_aggregate_csv.py b/test_autofit/aggregator/summary_files/test_aggregate_csv.py similarity index 100% rename from test_autofit/aggregator/test_aggregate_csv.py rename to test_autofit/aggregator/summary_files/test_aggregate_csv.py diff --git a/test_autofit/aggregator/summary_files/test_aggregate_fits.py b/test_autofit/aggregator/summary_files/test_aggregate_fits.py new file mode 100644 index 000000000..a71bceb0c --- /dev/null +++ b/test_autofit/aggregator/summary_files/test_aggregate_fits.py @@ -0,0 +1,2 @@ +def test_aggregate(): + pass diff --git a/test_autofit/aggregator/test_aggregate_images.py b/test_autofit/aggregator/summary_files/test_aggregate_images.py similarity index 100% rename from test_autofit/aggregator/test_aggregate_images.py rename to test_autofit/aggregator/summary_files/test_aggregate_images.py From f1ebd103ccaf4afb34c3d95f58ead8a3d4ef495c Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 09:18:55 +0000 Subject: [PATCH 08/28] conftest --- test_autofit/aggregator/summary_files/conftest.py | 9 +++++++++ .../aggregator/summary_files/test_aggregate_csv.py | 5 +---- .../aggregator/summary_files/test_aggregate_images.py | 6 ------ 3 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 test_autofit/aggregator/summary_files/conftest.py diff --git a/test_autofit/aggregator/summary_files/conftest.py b/test_autofit/aggregator/summary_files/conftest.py new file mode 100644 index 000000000..49a73bf2b --- /dev/null +++ b/test_autofit/aggregator/summary_files/conftest.py @@ -0,0 +1,9 @@ +import pytest +from pathlib import Path +from autofit.aggregator import Aggregator + + +@pytest.fixture +def aggregator(): + directory = Path(__file__).parent / "aggregate_summary" + return Aggregator.from_directory(directory) diff --git a/test_autofit/aggregator/summary_files/test_aggregate_csv.py b/test_autofit/aggregator/summary_files/test_aggregate_csv.py index 2469971d8..237045bc1 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_csv.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_csv.py @@ -1,6 +1,5 @@ import csv -from autofit.aggregator import Aggregator from pathlib import Path from autofit.aggregator.aggregate_csv import AggregateCSV @@ -16,9 +15,7 @@ def output_path(): @pytest.fixture -def summary(): - directory = Path(__file__).parent / "aggregate_summary" - aggregator = Aggregator.from_directory(directory) +def summary(aggregator): return AggregateCSV(aggregator) diff --git a/test_autofit/aggregator/summary_files/test_aggregate_images.py b/test_autofit/aggregator/summary_files/test_aggregate_images.py index df5b05ae9..b84470657 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_images.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_images.py @@ -7,12 +7,6 @@ from autofit.aggregator.aggregate_images import AggregateImages, SubplotFit -@pytest.fixture -def aggregator(): - directory = Path(__file__).parent / "aggregate_summary" - return Aggregator.from_directory(directory) - - @pytest.fixture def aggregate(aggregator): return AggregateImages(aggregator) From cae6880a71bf5a8973ce3bdd6259415dd26f67db Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 09:35:50 +0000 Subject: [PATCH 09/28] refactor --- autofit/__init__.py | 5 +++-- autofit/aggregator/summary/__init__.py | 0 autofit/aggregator/{ => summary}/aggregate_csv.py | 0 autofit/aggregator/{ => summary}/aggregate_fits.py | 2 +- autofit/aggregator/{ => summary}/aggregate_images.py | 0 5 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 autofit/aggregator/summary/__init__.py rename autofit/aggregator/{ => summary}/aggregate_csv.py (100%) rename autofit/aggregator/{ => summary}/aggregate_fits.py (63%) rename autofit/aggregator/{ => summary}/aggregate_images.py (100%) diff --git a/autofit/__init__.py b/autofit/__init__.py index c3173e2f8..2a36f59a3 100644 --- a/autofit/__init__.py +++ b/autofit/__init__.py @@ -25,8 +25,9 @@ from .non_linear.samples import load_from_table from .non_linear.samples import SamplesStored from .database.aggregator import Aggregator -from .aggregator.aggregate_csv import AggregateCSV -from .aggregator.aggregate_images import AggregateImages +from .aggregator.summary.aggregate_csv import AggregateCSV +from .aggregator.summary.aggregate_images import AggregateImages +from .aggregator.summary.aggregate_fits import AggregateFits from .database.aggregator import Query from autofit.aggregator.fit_interface import Fit from .aggregator.search_output import SearchOutput diff --git a/autofit/aggregator/summary/__init__.py b/autofit/aggregator/summary/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/autofit/aggregator/aggregate_csv.py b/autofit/aggregator/summary/aggregate_csv.py similarity index 100% rename from autofit/aggregator/aggregate_csv.py rename to autofit/aggregator/summary/aggregate_csv.py diff --git a/autofit/aggregator/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py similarity index 63% rename from autofit/aggregator/aggregate_fits.py rename to autofit/aggregator/summary/aggregate_fits.py index 5136abc12..5ab5e41e2 100644 --- a/autofit/aggregator/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -1,3 +1,3 @@ -class AggregateFits: +class AggregateFITS: def __init__(self): pass diff --git a/autofit/aggregator/aggregate_images.py b/autofit/aggregator/summary/aggregate_images.py similarity index 100% rename from autofit/aggregator/aggregate_images.py rename to autofit/aggregator/summary/aggregate_images.py From 6023d8442b2f28da4d5d3c38d1f276d38f07ec63 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 09:39:43 +0000 Subject: [PATCH 10/28] extract function to obtain filename --- autofit/aggregator/summary/aggregate_images.py | 17 ++++++----------- autofit/aggregator/summary/subplot_filename.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 autofit/aggregator/summary/subplot_filename.py diff --git a/autofit/aggregator/summary/aggregate_images.py b/autofit/aggregator/summary/aggregate_images.py index c08d29a3f..d59ea72dd 100644 --- a/autofit/aggregator/summary/aggregate_images.py +++ b/autofit/aggregator/summary/aggregate_images.py @@ -8,6 +8,7 @@ from autofit.aggregator.search_output import SearchOutput from autofit.aggregator.aggregator import Aggregator +from autofit.aggregator.summary.subplot_filename import subplot_filename class SubplotFit(Enum): @@ -231,18 +232,12 @@ class name but using snake_case. The image for the subplot. """ subplot_type = subplot_.__class__ - name = ( - re.sub( - r"([A-Z])", - r"_\1", - subplot_type.__name__, - ) - .lower() - .lstrip("_") - ) - if subplot_type not in _images: - _images[subplot_type] = SubplotFitImage(result.image(name)) + _images[subplot_type] = SubplotFitImage( + result.image( + subplot_filename(subplot_), + ) + ) return _images[subplot_type] matrix = [] diff --git a/autofit/aggregator/summary/subplot_filename.py b/autofit/aggregator/summary/subplot_filename.py new file mode 100644 index 000000000..aa6ec5f5a --- /dev/null +++ b/autofit/aggregator/summary/subplot_filename.py @@ -0,0 +1,15 @@ +import re +from enum import Enum + + +def subplot_filename(subplot: Enum) -> str: + subplot_type = subplot.__class__ + return ( + re.sub( + r"([A-Z])", + r"_\1", + subplot_type.__name__, + ) + .lower() + .lstrip("_") + ) From 9b1b7b2a644169adf5227958e44a64f7908a6a44 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 09:59:30 +0000 Subject: [PATCH 11/28] fixes --- autofit/__init__.py | 2 +- autofit/aggregator/search_output.py | 12 ++++++++---- autofit/aggregator/summary/aggregate_fits.py | 11 +++++++++-- .../aggregator/summary_files/test_aggregate_csv.py | 2 +- .../aggregator/summary_files/test_aggregate_fits.py | 8 ++++++-- .../summary_files/test_aggregate_images.py | 3 +-- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/autofit/__init__.py b/autofit/__init__.py index 2a36f59a3..0032570cb 100644 --- a/autofit/__init__.py +++ b/autofit/__init__.py @@ -27,7 +27,7 @@ from .database.aggregator import Aggregator from .aggregator.summary.aggregate_csv import AggregateCSV from .aggregator.summary.aggregate_images import AggregateImages -from .aggregator.summary.aggregate_fits import AggregateFits +from .aggregator.summary.aggregate_fits import AggregateFITS from .database.aggregator import Query from autofit.aggregator.fit_interface import Fit from .aggregator.search_output import SearchOutput diff --git a/autofit/aggregator/search_output.py b/autofit/aggregator/search_output.py index 13a8f3818..cc44178c6 100644 --- a/autofit/aggregator/search_output.py +++ b/autofit/aggregator/search_output.py @@ -78,11 +78,15 @@ def files_path(self): return self.directory / "files" def _outputs(self, suffix): + return self._outputs_in_directory("files", suffix) + self._outputs_in_directory( + "image", suffix + ) + + def _outputs_in_directory(self, name: str, suffix: str): + files_path = self.directory / name outputs = [] - for file_path in self.files_path.rglob(f"*{suffix}"): - name = ".".join( - file_path.relative_to(self.files_path).with_suffix("").parts - ) + for file_path in files_path.rglob(f"*{suffix}"): + name = ".".join(file_path.relative_to(files_path).with_suffix("").parts) outputs.append(FileOutput(name, file_path)) return outputs diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index 5ab5e41e2..91d522371 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -1,3 +1,10 @@ +from autofit.aggregator import Aggregator + + class AggregateFITS: - def __init__(self): - pass + def __init__(self, aggregator: Aggregator): + self.aggregator = aggregator + + def extract_fits(self): + for result in self.aggregator: + print(result.fits) diff --git a/test_autofit/aggregator/summary_files/test_aggregate_csv.py b/test_autofit/aggregator/summary_files/test_aggregate_csv.py index 237045bc1..cfab3fc98 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_csv.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_csv.py @@ -2,7 +2,7 @@ from pathlib import Path -from autofit.aggregator.aggregate_csv import AggregateCSV +from autofit.aggregator.summary.aggregate_csv import AggregateCSV import pytest diff --git a/test_autofit/aggregator/summary_files/test_aggregate_fits.py b/test_autofit/aggregator/summary_files/test_aggregate_fits.py index a71bceb0c..f97a39039 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_fits.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_fits.py @@ -1,2 +1,6 @@ -def test_aggregate(): - pass +import autofit as af + + +def test_aggregate(aggregator): + summary = af.AggregateFITS(aggregator) + summary.extract_fits() diff --git a/test_autofit/aggregator/summary_files/test_aggregate_images.py b/test_autofit/aggregator/summary_files/test_aggregate_images.py index b84470657..e4bd51b6c 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_images.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_images.py @@ -3,8 +3,7 @@ from PIL import Image -from autofit.aggregator import Aggregator -from autofit.aggregator.aggregate_images import AggregateImages, SubplotFit +from autofit.aggregator.summary.aggregate_images import AggregateImages, SubplotFit @pytest.fixture From dd821486a71cedce4447a4674f26e0f36b6870a7 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 10:10:36 +0000 Subject: [PATCH 12/28] basic summarisation of fits --- autofit/aggregator/summary/aggregate_fits.py | 27 +++++++++++++++++-- .../summary_files/test_aggregate_fits.py | 7 ++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index 91d522371..9f76e394a 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -1,10 +1,33 @@ +from enum import Enum + +from astropy.io import fits + from autofit.aggregator import Aggregator +from autofit.aggregator.summary.subplot_filename import subplot_filename + + +class Fit(Enum): + ModelImage = "MODEL_IMAGE" + ResidualMap = "RESIDUAL_MAP" + NormalizedResidualMap = "NORMALIZED_RESIDUAL_MAP" + ChiSquaredMap = "CHI_SQUARED_MAP" class AggregateFITS: def __init__(self, aggregator: Aggregator): self.aggregator = aggregator - def extract_fits(self): + def extract_fits(self, *hdus): + output = [fits.PrimaryHDU()] for result in self.aggregator: - print(result.fits) + for hdu in hdus: + source = result.value(subplot_filename(hdu)) + source_hdu = source[source.index_of(hdu.value)] + output.append( + fits.ImageHDU( + data=source_hdu.data, + header=source_hdu.header, + ) + ) + + return fits.HDUList(output) diff --git a/test_autofit/aggregator/summary_files/test_aggregate_fits.py b/test_autofit/aggregator/summary_files/test_aggregate_fits.py index f97a39039..3163f47e3 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_fits.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_fits.py @@ -1,6 +1,11 @@ import autofit as af +from autofit.aggregator.summary.aggregate_fits import Fit def test_aggregate(aggregator): summary = af.AggregateFITS(aggregator) - summary.extract_fits() + result = summary.extract_fits( + Fit.ModelImage, + Fit.ResidualMap, + ) + assert len(result) == 5 From 8d50c5d18b0dfa9197b46f0d39b0841b9870d684 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 10:12:59 +0000 Subject: [PATCH 13/28] improve API clarity --- autofit/__init__.py | 2 +- autofit/aggregator/summary/aggregate_fits.py | 17 +++++++++++++++-- .../aggregator/summary/aggregate_images.py | 19 ++++++++++++++++--- .../aggregator/summary/subplot_filename.py | 15 --------------- .../summary_files/test_aggregate_fits.py | 5 ++--- 5 files changed, 34 insertions(+), 24 deletions(-) delete mode 100644 autofit/aggregator/summary/subplot_filename.py diff --git a/autofit/__init__.py b/autofit/__init__.py index 0032570cb..7af5fe0af 100644 --- a/autofit/__init__.py +++ b/autofit/__init__.py @@ -27,7 +27,7 @@ from .database.aggregator import Aggregator from .aggregator.summary.aggregate_csv import AggregateCSV from .aggregator.summary.aggregate_images import AggregateImages -from .aggregator.summary.aggregate_fits import AggregateFITS +from .aggregator.summary.aggregate_fits import AggregateFITS, FitFITS from .database.aggregator import Query from autofit.aggregator.fit_interface import Fit from .aggregator.search_output import SearchOutput diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index 9f76e394a..c0819e6a2 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -1,12 +1,25 @@ +import re from enum import Enum from astropy.io import fits from autofit.aggregator import Aggregator -from autofit.aggregator.summary.subplot_filename import subplot_filename -class Fit(Enum): +def subplot_filename(subplot: Enum) -> str: + subplot_type = subplot.__class__ + return ( + re.sub( + r"([A-Z])", + r"_\1", + subplot_type.__name__.replace("FITS", ""), + ) + .lower() + .lstrip("_") + ) + + +class FitFITS(Enum): ModelImage = "MODEL_IMAGE" ResidualMap = "RESIDUAL_MAP" NormalizedResidualMap = "NORMALIZED_RESIDUAL_MAP" diff --git a/autofit/aggregator/summary/aggregate_images.py b/autofit/aggregator/summary/aggregate_images.py index d59ea72dd..640337f2f 100644 --- a/autofit/aggregator/summary/aggregate_images.py +++ b/autofit/aggregator/summary/aggregate_images.py @@ -1,6 +1,4 @@ -import re import sys -from enum import Enum from typing import Optional, List, Union, Callable, Type from pathlib import Path @@ -8,7 +6,22 @@ from autofit.aggregator.search_output import SearchOutput from autofit.aggregator.aggregator import Aggregator -from autofit.aggregator.summary.subplot_filename import subplot_filename + +import re +from enum import Enum + + +def subplot_filename(subplot: Enum) -> str: + subplot_type = subplot.__class__ + return ( + re.sub( + r"([A-Z])", + r"_\1", + subplot_type.__name__, + ) + .lower() + .lstrip("_") + ) class SubplotFit(Enum): diff --git a/autofit/aggregator/summary/subplot_filename.py b/autofit/aggregator/summary/subplot_filename.py deleted file mode 100644 index aa6ec5f5a..000000000 --- a/autofit/aggregator/summary/subplot_filename.py +++ /dev/null @@ -1,15 +0,0 @@ -import re -from enum import Enum - - -def subplot_filename(subplot: Enum) -> str: - subplot_type = subplot.__class__ - return ( - re.sub( - r"([A-Z])", - r"_\1", - subplot_type.__name__, - ) - .lower() - .lstrip("_") - ) diff --git a/test_autofit/aggregator/summary_files/test_aggregate_fits.py b/test_autofit/aggregator/summary_files/test_aggregate_fits.py index 3163f47e3..6811f8e9d 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_fits.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_fits.py @@ -1,11 +1,10 @@ import autofit as af -from autofit.aggregator.summary.aggregate_fits import Fit def test_aggregate(aggregator): summary = af.AggregateFITS(aggregator) result = summary.extract_fits( - Fit.ModelImage, - Fit.ResidualMap, + af.FitFITS.ModelImage, + af.FitFITS.ResidualMap, ) assert len(result) == 5 From 6355e11e1b6f8a5405155ffa875c905fe9fad3bd Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 10:29:15 +0000 Subject: [PATCH 14/28] output fits to file --- autofit/aggregator/summary/aggregate_fits.py | 47 +++++++++++++++---- .../aggregator/summary/aggregate_images.py | 2 +- .../summary_files/test_aggregate_fits.py | 21 ++++++++- 3 files changed, 57 insertions(+), 13 deletions(-) diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index c0819e6a2..5f04e75b5 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -1,7 +1,11 @@ import re from enum import Enum +from typing import List from astropy.io import fits +from pathlib import Path + +from astropy.io.fits.hdu.image import _ImageBaseHDU from autofit.aggregator import Aggregator @@ -30,17 +34,40 @@ class AggregateFITS: def __init__(self, aggregator: Aggregator): self.aggregator = aggregator - def extract_fits(self, *hdus): + def _hdus(self, result, *hdus) -> List[fits.ImageHDU]: + row = [] + for hdu in hdus: + source = result.value(subplot_filename(hdu)) + source_hdu = source[source.index_of(hdu.value)] + row.append( + fits.ImageHDU( + data=source_hdu.data, + header=source_hdu.header, + ) + ) + return row + + def extract_fits(self, *hdus: Enum): output = [fits.PrimaryHDU()] for result in self.aggregator: - for hdu in hdus: - source = result.value(subplot_filename(hdu)) - source_hdu = source[source.index_of(hdu.value)] - output.append( - fits.ImageHDU( - data=source_hdu.data, - header=source_hdu.header, - ) - ) + output.extend(self._hdus(result, *hdus)) return fits.HDUList(output) + + def output_to_folder( + self, + folder: Path, + *hdus: Enum, + name: str = "name", + ): + folder.mkdir(parents=True, exist_ok=True) + + for result in self.aggregator: + name = f"{getattr(result, name)}.fits" + fits.HDUList( + [fits.PrimaryHDU()] + + self._hdus( + result, + *hdus, + ) + ).writeto(folder / name) diff --git a/autofit/aggregator/summary/aggregate_images.py b/autofit/aggregator/summary/aggregate_images.py index 640337f2f..cce7cca6f 100644 --- a/autofit/aggregator/summary/aggregate_images.py +++ b/autofit/aggregator/summary/aggregate_images.py @@ -186,7 +186,7 @@ def output_to_folder( name The attribute of each fit to use as the name of the output file. """ - folder.mkdir(exist_ok=True) + folder.mkdir(exist_ok=True, parents=True) for i, result in enumerate(self._aggregator): image = self._matrix_to_image( diff --git a/test_autofit/aggregator/summary_files/test_aggregate_fits.py b/test_autofit/aggregator/summary_files/test_aggregate_fits.py index 6811f8e9d..a9c8a70b0 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_fits.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_fits.py @@ -1,10 +1,27 @@ +import pytest + import autofit as af -def test_aggregate(aggregator): - summary = af.AggregateFITS(aggregator) +@pytest.fixture(name="summary") +def make_summary(aggregator): + return af.AggregateFITS(aggregator) + + +def test_aggregate(summary): result = summary.extract_fits( af.FitFITS.ModelImage, af.FitFITS.ResidualMap, ) assert len(result) == 5 + + +def test_output_to_file(summary, output_directory): + folder = output_directory / "fits" + summary.output_to_folder( + folder, + af.FitFITS.ModelImage, + af.FitFITS.ResidualMap, + name="id", + ) + assert len((list(folder.glob("*")))) == 2 From 31fccc3f453d1fd9981115fc4c611ad1d9811696 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 10:38:06 +0000 Subject: [PATCH 15/28] docs --- autofit/aggregator/summary/aggregate_fits.py | 65 ++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index 5f04e75b5..b4b9982fe 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -5,8 +5,7 @@ from astropy.io import fits from pathlib import Path -from astropy.io.fits.hdu.image import _ImageBaseHDU - +from autofit import SearchOutput from autofit.aggregator import Aggregator @@ -24,6 +23,10 @@ def subplot_filename(subplot: Enum) -> str: class FitFITS(Enum): + """ + The HDUs that can be extracted from the fit.fits file. + """ + ModelImage = "MODEL_IMAGE" ResidualMap = "RESIDUAL_MAP" NormalizedResidualMap = "NORMALIZED_RESIDUAL_MAP" @@ -32,9 +35,35 @@ class FitFITS(Enum): class AggregateFITS: def __init__(self, aggregator: Aggregator): + """ + A class for extracting fits files from the aggregator. + + Parameters + ---------- + aggregator + The aggregator containing the fits files. + """ self.aggregator = aggregator - def _hdus(self, result, *hdus) -> List[fits.ImageHDU]: + @staticmethod + def _hdus( + result: SearchOutput, + *hdus: Enum, + ) -> List[fits.ImageHDU]: + """ + Extract the HDUs from a given fits for a given search. + + Parameters + ---------- + result + The search output. + hdus + The HDUs to extract. + + Returns + ------- + The extracted HDUs. + """ row = [] for hdu in hdus: source = result.value(subplot_filename(hdu)) @@ -47,7 +76,21 @@ def _hdus(self, result, *hdus) -> List[fits.ImageHDU]: ) return row - def extract_fits(self, *hdus: Enum): + def extract_fits(self, *hdus: Enum) -> List[fits.HDUList]: + """ + Extract the HDUs from the fits files for every search in the aggregator. + + Return the result as a list of HDULists. The first HDU in each list is an empty PrimaryHDU. + + Parameters + ---------- + hdus + The HDUs to extract. + + Returns + ------- + The extracted HDUs. + """ output = [fits.PrimaryHDU()] for result in self.aggregator: output.extend(self._hdus(result, *hdus)) @@ -60,6 +103,20 @@ def output_to_folder( *hdus: Enum, name: str = "name", ): + """ + Output the fits files for every search in the aggregator to a folder. + + Only include HDUs specific in the hdus argument. + + Parameters + ---------- + folder + The folder to output the fits files to. + hdus + The HDUs to output. + name + The name of the fits file. This is the attribute of the search output that is used to name the file. + """ folder.mkdir(parents=True, exist_ok=True) for result in self.aggregator: From 641bf1c9aecb9e7e500f0f8fba7b6f4ef91876da Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 5 Mar 2025 10:45:21 +0000 Subject: [PATCH 16/28] fix --- autofit/aggregator/summary/aggregate_fits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index b4b9982fe..c98584ea3 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -5,7 +5,7 @@ from astropy.io import fits from pathlib import Path -from autofit import SearchOutput +from autofit.aggregator.search_output import SearchOutput from autofit.aggregator import Aggregator From 55eb731937f7843221ebac088fb727a74fddfaeb Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 09:12:25 +0000 Subject: [PATCH 17/28] write to using file not name --- autofit/aggregator/summary/aggregate_fits.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index c98584ea3..1726a2d90 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -121,10 +121,12 @@ def output_to_folder( for result in self.aggregator: name = f"{getattr(result, name)}.fits" - fits.HDUList( + hdu_list = fits.HDUList( [fits.PrimaryHDU()] + self._hdus( result, *hdus, ) - ).writeto(folder / name) + ) + with open(folder / name, "wb") as file: + hdu_list.writeto(file) From 5cfb486963b2437a2be1884fc70f3152cd936724 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 10:13:14 +0000 Subject: [PATCH 18/28] generalise to enum --- autofit/aggregator/summary/aggregate_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autofit/aggregator/summary/aggregate_images.py b/autofit/aggregator/summary/aggregate_images.py index cce7cca6f..3aab8419b 100644 --- a/autofit/aggregator/summary/aggregate_images.py +++ b/autofit/aggregator/summary/aggregate_images.py @@ -256,7 +256,7 @@ class name but using snake_case. matrix = [] row = [] for subplot in subplots: - if isinstance(subplot, SubplotFit): + if isinstance(subplot, Enum): row.append( get_image(subplot).image_at_coordinates( *subplot.value, From 56721dbd9150991dde5570af4518142bd8cb29e7 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 10:15:36 +0000 Subject: [PATCH 19/28] extra asssertion --- autofit/aggregator/summary/aggregate_images.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/autofit/aggregator/summary/aggregate_images.py b/autofit/aggregator/summary/aggregate_images.py index 3aab8419b..faac61cff 100644 --- a/autofit/aggregator/summary/aggregate_images.py +++ b/autofit/aggregator/summary/aggregate_images.py @@ -263,6 +263,12 @@ class name but using snake_case. ) ) elif isinstance(subplot, list): + if not isinstance(subplot[i], Image.Image): + raise TypeError( + "The subplots must be of type Subplot or a list of " + "images or a function that takes a SearchOutput as an " + "argument." + ) row.append(subplot[i]) else: try: From b182f21ff6e62a7fb1cbc67c59c3c8815b13a17b Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 10:20:47 +0000 Subject: [PATCH 20/28] fixes --- autofit/aggregator/aggregate_images.py | 44 +++++++++++++++++--------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/autofit/aggregator/aggregate_images.py b/autofit/aggregator/aggregate_images.py index c08d29a3f..faac61cff 100644 --- a/autofit/aggregator/aggregate_images.py +++ b/autofit/aggregator/aggregate_images.py @@ -1,6 +1,4 @@ -import re import sys -from enum import Enum from typing import Optional, List, Union, Callable, Type from pathlib import Path @@ -9,6 +7,22 @@ from autofit.aggregator.search_output import SearchOutput from autofit.aggregator.aggregator import Aggregator +import re +from enum import Enum + + +def subplot_filename(subplot: Enum) -> str: + subplot_type = subplot.__class__ + return ( + re.sub( + r"([A-Z])", + r"_\1", + subplot_type.__name__, + ) + .lower() + .lstrip("_") + ) + class SubplotFit(Enum): """ @@ -172,7 +186,7 @@ def output_to_folder( name The attribute of each fit to use as the name of the output file. """ - folder.mkdir(exist_ok=True) + folder.mkdir(exist_ok=True, parents=True) for i, result in enumerate(self._aggregator): image = self._matrix_to_image( @@ -231,30 +245,30 @@ class name but using snake_case. The image for the subplot. """ subplot_type = subplot_.__class__ - name = ( - re.sub( - r"([A-Z])", - r"_\1", - subplot_type.__name__, - ) - .lower() - .lstrip("_") - ) - if subplot_type not in _images: - _images[subplot_type] = SubplotFitImage(result.image(name)) + _images[subplot_type] = SubplotFitImage( + result.image( + subplot_filename(subplot_), + ) + ) return _images[subplot_type] matrix = [] row = [] for subplot in subplots: - if isinstance(subplot, SubplotFit): + if isinstance(subplot, Enum): row.append( get_image(subplot).image_at_coordinates( *subplot.value, ) ) elif isinstance(subplot, list): + if not isinstance(subplot[i], Image.Image): + raise TypeError( + "The subplots must be of type Subplot or a list of " + "images or a function that takes a SearchOutput as an " + "argument." + ) row.append(subplot[i]) else: try: From 922b18f7aa0acb4ec5e207c988974d386ef7e016 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 10:34:47 +0000 Subject: [PATCH 21/28] test custom enums work --- .../aggregator/test_aggregate_images.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test_autofit/aggregator/test_aggregate_images.py b/test_autofit/aggregator/test_aggregate_images.py index df5b05ae9..33cb37056 100644 --- a/test_autofit/aggregator/test_aggregate_images.py +++ b/test_autofit/aggregator/test_aggregate_images.py @@ -1,3 +1,5 @@ +from enum import Enum + import pytest from pathlib import Path @@ -117,3 +119,19 @@ def make_image(output): ) assert result.size == (193, 120) + + +def test_custom_subplot_fit(aggregate): + class SubplotFit(Enum): + """ + The subplots that can be extracted from the subplot_fit image. + + The values correspond to the position of the subplot in the 4x3 grid. + """ + + Data = (0, 0) + + result = aggregate.extract_image( + SubplotFit.Data, + ) + assert result.size == (61, 120) From a04623cd8432734d407c1c48721bd5d5ea8cb578 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 10:37:10 +0000 Subject: [PATCH 22/28] raise a ValueError for an empty aggregator --- autofit/aggregator/aggregate_images.py | 3 +++ test_autofit/aggregator/test_aggregate_images.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/autofit/aggregator/aggregate_images.py b/autofit/aggregator/aggregate_images.py index faac61cff..e04529df2 100644 --- a/autofit/aggregator/aggregate_images.py +++ b/autofit/aggregator/aggregate_images.py @@ -114,6 +114,9 @@ def __init__( aggregator The aggregator containing the fit results. """ + if len(aggregator) == 0: + raise ValueError("The aggregator is empty.") + self._aggregator = aggregator self._source_images = None diff --git a/test_autofit/aggregator/test_aggregate_images.py b/test_autofit/aggregator/test_aggregate_images.py index 33cb37056..135a7b3b3 100644 --- a/test_autofit/aggregator/test_aggregate_images.py +++ b/test_autofit/aggregator/test_aggregate_images.py @@ -135,3 +135,11 @@ class SubplotFit(Enum): SubplotFit.Data, ) assert result.size == (61, 120) + + +def test_bad_aggregator(): + directory = Path(__file__).parent / "aggregate_summaries" + aggregator = Aggregator.from_directory(directory) + + with pytest.raises(ValueError): + AggregateImages(aggregator) From 032809396d11043f69fa02b31209eb357a1758bf Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 10:38:49 +0000 Subject: [PATCH 23/28] add assertion --- autofit/aggregator/summary/aggregate_csv.py | 3 +++ autofit/aggregator/summary/aggregate_fits.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/autofit/aggregator/summary/aggregate_csv.py b/autofit/aggregator/summary/aggregate_csv.py index 1af9b0054..22d94d0a5 100644 --- a/autofit/aggregator/summary/aggregate_csv.py +++ b/autofit/aggregator/summary/aggregate_csv.py @@ -190,6 +190,9 @@ def __init__(self, aggregator: Aggregator): ---------- aggregator """ + if len(aggregator) == 0: + raise ValueError("The aggregator is empty.") + self._aggregator = aggregator self._columns = [] diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index 1726a2d90..b92b5b25e 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -43,6 +43,9 @@ def __init__(self, aggregator: Aggregator): aggregator The aggregator containing the fits files. """ + if len(aggregator) == 0: + raise ValueError("The aggregator is empty.") + self.aggregator = aggregator @staticmethod From d200948d3e6530ba84c98e2631e2469ca6a1c4af Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 12:22:54 +0000 Subject: [PATCH 24/28] include fits when getting value from fit --- autofit/database/model/fit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autofit/database/model/fit.py b/autofit/database/model/fit.py index 93b8076c2..af946baab 100644 --- a/autofit/database/model/fit.py +++ b/autofit/database/model/fit.py @@ -359,7 +359,7 @@ def __getitem__(self, item: str): ------- An unpickled object """ - for p in self.jsons + self.arrays + self.hdus + self.pickles: + for p in self.jsons + self.arrays + self.hdus + self.pickles + self.fits: if p.name == item: value = p.value if item == "samples_summary": From 1a695468df6958e35ef09fe99ac8d81af7b6e61d Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 12:41:27 +0000 Subject: [PATCH 25/28] try bumping actions/cache version --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4fb654d4f..75e0a78a2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip From f61f3e9ed2987e305146dcec7070252331bab369 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 12:50:43 +0000 Subject: [PATCH 26/28] remove default + fixes --- autofit/aggregator/summary/aggregate_fits.py | 2 +- autofit/aggregator/summary/aggregate_images.py | 2 +- test_autofit/aggregator/summary_files/test_aggregate_images.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index b92b5b25e..8bda96a6e 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -104,7 +104,7 @@ def output_to_folder( self, folder: Path, *hdus: Enum, - name: str = "name", + name: str, ): """ Output the fits files for every search in the aggregator to a folder. diff --git a/autofit/aggregator/summary/aggregate_images.py b/autofit/aggregator/summary/aggregate_images.py index e04529df2..82a7fa1fb 100644 --- a/autofit/aggregator/summary/aggregate_images.py +++ b/autofit/aggregator/summary/aggregate_images.py @@ -166,7 +166,7 @@ def output_to_folder( folder: Path, *subplots: Union[SubplotFit, List[Image.Image], Callable], subplot_width: Optional[int] = sys.maxsize, - name: str = "name", + name: str, ): """ Output one subplot image for each fit in the aggregator. diff --git a/test_autofit/aggregator/summary_files/test_aggregate_images.py b/test_autofit/aggregator/summary_files/test_aggregate_images.py index 7d44a0a54..d2bbadeb3 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_images.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_images.py @@ -5,6 +5,7 @@ from PIL import Image +from autofit.aggregator import Aggregator from autofit.aggregator.summary.aggregate_images import AggregateImages, SubplotFit @@ -62,6 +63,7 @@ def test_output_to_folder(aggregate, output_directory): SubplotFit.Data, SubplotFit.SourcePlaneZoomed, SubplotFit.SourceModelImage, + name="name", ) assert list(Path(output_directory).glob("*.png")) From a27af1758f3760c3f466c55e2fbbc6519b03e4cf Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 12:58:48 +0000 Subject: [PATCH 27/28] optional list for naming aggregated images --- autofit/aggregator/summary/aggregate_images.py | 11 +++++++++-- .../summary_files/test_aggregate_images.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/autofit/aggregator/summary/aggregate_images.py b/autofit/aggregator/summary/aggregate_images.py index 82a7fa1fb..37d5b2ab2 100644 --- a/autofit/aggregator/summary/aggregate_images.py +++ b/autofit/aggregator/summary/aggregate_images.py @@ -166,7 +166,7 @@ def output_to_folder( folder: Path, *subplots: Union[SubplotFit, List[Image.Image], Callable], subplot_width: Optional[int] = sys.maxsize, - name: str, + name: Union[str, List[str]], ): """ Output one subplot image for each fit in the aggregator. @@ -188,6 +188,7 @@ def output_to_folder( images to wrap. name The attribute of each fit to use as the name of the output file. + OR a list of names, one for each fit. """ folder.mkdir(exist_ok=True, parents=True) @@ -200,7 +201,13 @@ def output_to_folder( subplot_width=subplot_width, ) ) - image.save(folder / f"{getattr(result, name)}.png") + + if isinstance(name, str): + output_name = getattr(result, name) + else: + output_name = name[i] + + image.save(folder / f"{output_name}.png") @staticmethod def _matrix_for_result( diff --git a/test_autofit/aggregator/summary_files/test_aggregate_images.py b/test_autofit/aggregator/summary_files/test_aggregate_images.py index d2bbadeb3..dce18d8ea 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_images.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_images.py @@ -68,6 +68,20 @@ def test_output_to_folder(aggregate, output_directory): assert list(Path(output_directory).glob("*.png")) +def test_list_of_names(aggregate, output_directory): + aggregate.output_to_folder( + output_directory, + SubplotFit.Data, + SubplotFit.SourcePlaneZoomed, + SubplotFit.SourceModelImage, + name=["one", "two"], + ) + assert [path.name for path in Path(output_directory).glob("*.png")] == [ + "two.png", + "one.png", + ] + + def test_output_to_folder_name( aggregate, output_directory, From 09a8825c6b5baa174101af6949359bdbff03c6f9 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 14 Mar 2025 13:02:47 +0000 Subject: [PATCH 28/28] also for fits --- autofit/aggregator/summary/aggregate_fits.py | 15 ++++++++++----- .../summary_files/test_aggregate_fits.py | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index 8bda96a6e..1039221bf 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -1,6 +1,6 @@ import re from enum import Enum -from typing import List +from typing import List, Union from astropy.io import fits from pathlib import Path @@ -104,7 +104,7 @@ def output_to_folder( self, folder: Path, *hdus: Enum, - name: str, + name: Union[str, List[str]], ): """ Output the fits files for every search in the aggregator to a folder. @@ -119,11 +119,16 @@ def output_to_folder( The HDUs to output. name The name of the fits file. This is the attribute of the search output that is used to name the file. + OR a list of names for each HDU. """ folder.mkdir(parents=True, exist_ok=True) - for result in self.aggregator: - name = f"{getattr(result, name)}.fits" + for i, result in enumerate(self.aggregator): + if isinstance(name, str): + output_name = getattr(result, name) + else: + output_name = name[i] + hdu_list = fits.HDUList( [fits.PrimaryHDU()] + self._hdus( @@ -131,5 +136,5 @@ def output_to_folder( *hdus, ) ) - with open(folder / name, "wb") as file: + with open(folder / f"{output_name}.fits", "wb") as file: hdu_list.writeto(file) diff --git a/test_autofit/aggregator/summary_files/test_aggregate_fits.py b/test_autofit/aggregator/summary_files/test_aggregate_fits.py index a9c8a70b0..da2e73426 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_fits.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_fits.py @@ -1,6 +1,7 @@ import pytest import autofit as af +from pathlib import Path @pytest.fixture(name="summary") @@ -25,3 +26,16 @@ def test_output_to_file(summary, output_directory): name="id", ) assert len((list(folder.glob("*")))) == 2 + + +def test_list_of_names(summary, output_directory): + summary.output_to_folder( + output_directory, + af.FitFITS.ModelImage, + af.FitFITS.ResidualMap, + name=["one", "two"], + ) + assert [path.name for path in Path(output_directory).glob("*.fits")] == [ + "one.fits", + "two.fits", + ]