From 75052235d6df1af53f3f3d461c48988e0fd2f120 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 16 Oct 2025 10:15:33 -0700 Subject: [PATCH 01/24] Add `edf` and `gb` adapters for `tiled<0.1.0-b16` --- src/als_tiled/bl733/adapters/edf.py | 96 +++++++++++++++++ src/als_tiled/bl733/adapters/gb.py | 160 ++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 src/als_tiled/bl733/adapters/edf.py create mode 100644 src/als_tiled/bl733/adapters/gb.py diff --git a/src/als_tiled/bl733/adapters/edf.py b/src/als_tiled/bl733/adapters/edf.py new file mode 100644 index 0000000..4cef526 --- /dev/null +++ b/src/als_tiled/bl733/adapters/edf.py @@ -0,0 +1,96 @@ +import logging +import os +import pathlib +from logging import StreamHandler + +import fabio +from tiled.adapters.array import ArrayAdapter +from tiled.structures.core import Spec +from tiled.utils import path_from_uri + +logger = logging.getLogger("tiled.adapters.edf") +logger.addHandler(StreamHandler()) +logger.setLevel("INFO") + + +def parse_txt_accompanying_edf(filepath): + """Pase a .txt file produced at ALS beamline 7.3.3 into a dictionary. + + Parameters + ---------- + filepath: str or pathlib.Path + Filepath of the .edf file. + """ + txt_filepath = None + if isinstance(filepath, str): + txt_filepath = filepath.replace(".edf", ".txt") + if isinstance(filepath, pathlib.Path): + txt_filepath = filepath.with_suffix(".txt") + + # File does not exist, return empty dictionary + if not os.path.isfile(txt_filepath): + logger.warn(f"{filepath} has no corresponding .txt.") + return dict() + + with open(txt_filepath, "r") as file: + lines = file.readlines() + + # Some lines have the format + # key: value + # others are just values with no key + keyless_lines = 0 + txt_params = dict() + for line in lines: + line_components = list(map(str.strip, line.split(":", maxsplit=1))) + if len(line_components) >= 2: + txt_params[line_components[0]] = line_components[1] + else: + if line_components[0] != "!0": + txt_params[f"Keyless Parameter #{keyless_lines}"] = line_components[0] + keyless_lines += 1 + return txt_params + + +def read(data_uri, structure=None, metadata=None, specs=None, access_policy=None): + """Read a detector image saved as .edf produced at ALS beamline 7.3.3 + + Parameters + ---------- + data_uri: str + Uri of the .edf file, typically a file:// uri. + """ + # TODO Should we catch any read errors here? + filepath = path_from_uri(data_uri) + file = fabio.open(filepath) + array = file.data + + # Merge parameters from the header into other meta data + if metadata is None: + metadata = file.header + else: + metadata = {**metadata, **file.header} + + # If a .txt file with the same name exists + # extract additional meta data from it + txt_params = parse_txt_accompanying_edf(filepath) + metadata = {**metadata, **txt_params} + return ArrayAdapter.from_array(array, metadata=metadata, specs=[Spec("edf")]) + + +async def walk_edf_with_txt( + catalog, + path, + files, + directories, + settings, +): + """ + Possible patters: + 1-1 txt-edf + 1 log, many edfs + 1 txt - 2 edf with _hi _lo + """ + # TODO + unhandled_files = files + unhandled_directories = directories + return unhandled_files, unhandled_directories diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py new file mode 100644 index 0000000..8ab5980 --- /dev/null +++ b/src/als_tiled/bl733/adapters/gb.py @@ -0,0 +1,160 @@ +import logging +import os +import pathlib +from datetime import datetime +from logging import StreamHandler + +import fabio +import numpy as np +from als_tiled.bl733.adapters.edf import parse_txt_accompanying_edf +from tiled.adapters.array import ArrayAdapter +from tiled.structures.core import Spec +from tiled.utils import path_from_uri + +logger = logging.getLogger("tiled.adapters.gb") +logger.addHandler(StreamHandler()) +logger.setLevel("INFO") + + +def read(data_uri, structure=None, metadata=None, specs=None, access_policy=None): + """Read a detector image saved as .gb produced at ALS beamline 7.3.3 + + Parameters + ---------- + data_uri: str + Uri of the .edf file, typically a file:// uri. + """ + filepath = path_from_uri(data_uri) + pixels_x = 1475 + pixels_y = 1679 + data = np.fromfile(filepath, dtype=" lo_date else lo_date + else: + gb_dictionary["Date"] = hi_date if hi_date is not None else lo_date + + return gb_dictionary From d772743481e4b25fc23c4d583be9c2057a46c2a3 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 16 Oct 2025 15:47:00 -0700 Subject: [PATCH 02/24] Add placeholders for adapter tests --- tests/bl733/test_edf.py | 0 tests/bl733/test_gb.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/bl733/test_edf.py create mode 100644 tests/bl733/test_gb.py diff --git a/tests/bl733/test_edf.py b/tests/bl733/test_edf.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/bl733/test_gb.py b/tests/bl733/test_gb.py new file mode 100644 index 0000000..e69de29 From 45a12dc89480813753e71185cd8c09821193b5a9 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 16 Oct 2025 16:56:09 -0700 Subject: [PATCH 03/24] Elevate edf function adapter to class --- src/als_tiled/bl733/adapters/edf.py | 114 ++++++++++++++++------------ src/als_tiled/bl733/adapters/gb.py | 3 +- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/src/als_tiled/bl733/adapters/edf.py b/src/als_tiled/bl733/adapters/edf.py index 4cef526..a6ace6c 100644 --- a/src/als_tiled/bl733/adapters/edf.py +++ b/src/als_tiled/bl733/adapters/edf.py @@ -1,20 +1,26 @@ import logging import os import pathlib -from logging import StreamHandler +from typing import Any, Optional import fabio from tiled.adapters.array import ArrayAdapter -from tiled.structures.core import Spec +from tiled.adapters.utils import init_adapter_from_catalog +from tiled.catalog.orm import Node +from tiled.structures.array import ArrayStructure +from tiled.structures.core import Spec, StructureFamily +from tiled.structures.data_source import DataSource +from tiled.type_aliases import JSON from tiled.utils import path_from_uri -logger = logging.getLogger("tiled.adapters.edf") -logger.addHandler(StreamHandler()) -logger.setLevel("INFO") +logger = logging.getLogger("als_tiled.adapters.bl733.edf") +if not logger.handlers: + logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.INFO) def parse_txt_accompanying_edf(filepath): - """Pase a .txt file produced at ALS beamline 7.3.3 into a dictionary. + """Parse a .txt file produced at ALS beamline 7.3.3 into a dictionary. Parameters ---------- @@ -29,7 +35,7 @@ def parse_txt_accompanying_edf(filepath): # File does not exist, return empty dictionary if not os.path.isfile(txt_filepath): - logger.warn(f"{filepath} has no corresponding .txt.") + logger.warning(f"{filepath} has no corresponding .txt.") return dict() with open(txt_filepath, "r") as file: @@ -51,46 +57,56 @@ def parse_txt_accompanying_edf(filepath): return txt_params -def read(data_uri, structure=None, metadata=None, specs=None, access_policy=None): - """Read a detector image saved as .edf produced at ALS beamline 7.3.3 +class EDFAdapter(ArrayAdapter): + structure_family = StructureFamily.array - Parameters - ---------- - data_uri: str - Uri of the .edf file, typically a file:// uri. - """ - # TODO Should we catch any read errors here? - filepath = path_from_uri(data_uri) - file = fabio.open(filepath) - array = file.data - - # Merge parameters from the header into other meta data - if metadata is None: - metadata = file.header - else: - metadata = {**metadata, **file.header} - - # If a .txt file with the same name exists - # extract additional meta data from it - txt_params = parse_txt_accompanying_edf(filepath) - metadata = {**metadata, **txt_params} - return ArrayAdapter.from_array(array, metadata=metadata, specs=[Spec("edf")]) - - -async def walk_edf_with_txt( - catalog, - path, - files, - directories, - settings, -): - """ - Possible patters: - 1-1 txt-edf - 1 log, many edfs - 1 txt - 2 edf with _hi _lo - """ - # TODO - unhandled_files = files - unhandled_directories = directories - return unhandled_files, unhandled_directories + def __init__( + self, + data_uri: str, + structure: Optional[ArrayStructure] = None, + metadata: Optional[JSON] = None, + specs: Optional[list[Spec]] = None, + **kwargs: Optional[Any], + ) -> None: + """Adapter for `.edf` files (e.g. PILATUS3 2M) at ALS beamline 7.3.3.""" + + filepath = path_from_uri(data_uri) + + metadata = metadata or dict() + with fabio.open(filepath) as edf_file: + array = edf_file.data + edf_metadata = edf_file.header + + # Merge parameters from the header into potentially existing meta data + metadata = {**metadata, **edf_metadata} + + # If a .txt file with the same name exists + # extract additional meta data from it + txt_metadata = parse_txt_accompanying_edf(filepath) + metadata = {**metadata, **txt_metadata} + + super().__init__( + array=array, + structure=structure or ArrayStructure.from_array(array), + metadata=metadata, + specs=(specs or []) + [Spec("als-bl733-edf", version="1.0")], + **kwargs, + ) + + @classmethod + def from_catalog( + cls, + data_source: DataSource, + node: Node, + /, + **kwargs: Optional[Any], + ) -> "EDFAdapter": + return init_adapter_from_catalog(cls, data_source, node, **kwargs) + + @classmethod + def from_uris( + cls, + data_uri: str, + **kwargs: Optional[Any], + ) -> "EDFAdapter": + return cls(data_uri, **kwargs) diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py index 8ab5980..4daaddc 100644 --- a/src/als_tiled/bl733/adapters/gb.py +++ b/src/als_tiled/bl733/adapters/gb.py @@ -6,11 +6,12 @@ import fabio import numpy as np -from als_tiled.bl733.adapters.edf import parse_txt_accompanying_edf from tiled.adapters.array import ArrayAdapter from tiled.structures.core import Spec from tiled.utils import path_from_uri +from als_tiled.bl733.adapters.edf import parse_txt_accompanying_edf + logger = logging.getLogger("tiled.adapters.gb") logger.addHandler(StreamHandler()) logger.setLevel("INFO") From 7dfbfc33b800366e1c0b21442014240a021c920e Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Tue, 17 Mar 2026 17:22:30 -0700 Subject: [PATCH 04/24] Elevate gb function adapter to class --- pyproject.toml | 4 ++ src/als_tiled/bl733/adapters/edf.py | 7 +- src/als_tiled/bl733/adapters/gb.py | 106 +++++++++++++++++++--------- 3 files changed, 77 insertions(+), 40 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 04688b3..c92143a 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,10 @@ tiled_client = [ "tiled[client]" ] +bl733 = [ + "fabio", +] + [project.urls] Homepage = "https://github.com/als-lbl/als_tiled" Repository = "https://github.com/als-lbl/als_tiled" diff --git a/src/als_tiled/bl733/adapters/edf.py b/src/als_tiled/bl733/adapters/edf.py index a6ace6c..4aaa5d1 100644 --- a/src/als_tiled/bl733/adapters/edf.py +++ b/src/als_tiled/bl733/adapters/edf.py @@ -13,13 +13,10 @@ from tiled.type_aliases import JSON from tiled.utils import path_from_uri -logger = logging.getLogger("als_tiled.adapters.bl733.edf") -if not logger.handlers: - logger.addHandler(logging.StreamHandler()) -logger.setLevel(logging.INFO) +logger = logging.getLogger(__name__) -def parse_txt_accompanying_edf(filepath): +def parse_txt_accompanying_edf(filepath: str | pathlib.Path) -> dict[str, Any]: """Parse a .txt file produced at ALS beamline 7.3.3 into a dictionary. Parameters diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py index 4daaddc..0780aa8 100644 --- a/src/als_tiled/bl733/adapters/gb.py +++ b/src/als_tiled/bl733/adapters/gb.py @@ -2,52 +2,88 @@ import os import pathlib from datetime import datetime -from logging import StreamHandler +from typing import Any, Optional import fabio import numpy as np from tiled.adapters.array import ArrayAdapter -from tiled.structures.core import Spec +from tiled.adapters.utils import init_adapter_from_catalog +from tiled.catalog.orm import Node +from tiled.structures.array import ArrayStructure +from tiled.structures.core import Spec, StructureFamily +from tiled.structures.data_source import DataSource +from tiled.type_aliases import JSON from tiled.utils import path_from_uri from als_tiled.bl733.adapters.edf import parse_txt_accompanying_edf -logger = logging.getLogger("tiled.adapters.gb") -logger.addHandler(StreamHandler()) -logger.setLevel("INFO") - - -def read(data_uri, structure=None, metadata=None, specs=None, access_policy=None): - """Read a detector image saved as .gb produced at ALS beamline 7.3.3 - - Parameters - ---------- - data_uri: str - Uri of the .edf file, typically a file:// uri. - """ - filepath = path_from_uri(data_uri) - pixels_x = 1475 - pixels_y = 1679 - data = np.fromfile(filepath, dtype=" None: + """Adapter for a detector image saved as .gb produced at ALS beamline 7.3.3""" + filepath = path_from_uri(data_uri) + data = np.fromfile(filepath, dtype=" "GeneralBinaryPilatus2MAdapter": + return init_adapter_from_catalog(cls, data_source, node, **kwargs) + + @classmethod + def from_uris( + cls, + data_uri: str, + **kwargs: Optional[Any], + ) -> "GeneralBinaryPilatus2MAdapter": + return cls(data_uri, **kwargs) def combine_edf_metadata_for_gb(hi_dict, lo_dict): From 20f1e92b9ec99a571fb2ae62fbb76b246f37db8d Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 18 Mar 2026 12:56:55 -0700 Subject: [PATCH 05/24] Move txt parsing to separate `metadata` module --- src/als_tiled/bl733/adapters/edf.py | 58 +------- src/als_tiled/bl733/adapters/gb.py | 181 ++++++++--------------- src/als_tiled/bl733/adapters/metadata.py | 36 +++++ 3 files changed, 102 insertions(+), 173 deletions(-) create mode 100644 src/als_tiled/bl733/adapters/metadata.py diff --git a/src/als_tiled/bl733/adapters/edf.py b/src/als_tiled/bl733/adapters/edf.py index 4aaa5d1..9c3c3f6 100644 --- a/src/als_tiled/bl733/adapters/edf.py +++ b/src/als_tiled/bl733/adapters/edf.py @@ -1,6 +1,4 @@ import logging -import os -import pathlib from typing import Any, Optional import fabio @@ -13,45 +11,9 @@ from tiled.type_aliases import JSON from tiled.utils import path_from_uri -logger = logging.getLogger(__name__) - - -def parse_txt_accompanying_edf(filepath: str | pathlib.Path) -> dict[str, Any]: - """Parse a .txt file produced at ALS beamline 7.3.3 into a dictionary. - - Parameters - ---------- - filepath: str or pathlib.Path - Filepath of the .edf file. - """ - txt_filepath = None - if isinstance(filepath, str): - txt_filepath = filepath.replace(".edf", ".txt") - if isinstance(filepath, pathlib.Path): - txt_filepath = filepath.with_suffix(".txt") - - # File does not exist, return empty dictionary - if not os.path.isfile(txt_filepath): - logger.warning(f"{filepath} has no corresponding .txt.") - return dict() +from als_tiled.bl733.adapters.metadata import parse_txt_accompanying_edf - with open(txt_filepath, "r") as file: - lines = file.readlines() - - # Some lines have the format - # key: value - # others are just values with no key - keyless_lines = 0 - txt_params = dict() - for line in lines: - line_components = list(map(str.strip, line.split(":", maxsplit=1))) - if len(line_components) >= 2: - txt_params[line_components[0]] = line_components[1] - else: - if line_components[0] != "!0": - txt_params[f"Keyless Parameter #{keyless_lines}"] = line_components[0] - keyless_lines += 1 - return txt_params +logger = logging.getLogger(__name__) class EDFAdapter(ArrayAdapter): @@ -66,21 +28,17 @@ def __init__( **kwargs: Optional[Any], ) -> None: """Adapter for `.edf` files (e.g. PILATUS3 2M) at ALS beamline 7.3.3.""" - filepath = path_from_uri(data_uri) - metadata = metadata or dict() with fabio.open(filepath) as edf_file: array = edf_file.data - edf_metadata = edf_file.header - - # Merge parameters from the header into potentially existing meta data - metadata = {**metadata, **edf_metadata} + metadata_edf = edf_file.header - # If a .txt file with the same name exists - # extract additional meta data from it - txt_metadata = parse_txt_accompanying_edf(filepath) - metadata = {**metadata, **txt_metadata} + metadata = { + **(metadata or {}), + **metadata_edf, + **parse_txt_accompanying_edf(filepath), + } super().__init__( array=array, diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py index 0780aa8..6f763e4 100644 --- a/src/als_tiled/bl733/adapters/gb.py +++ b/src/als_tiled/bl733/adapters/gb.py @@ -1,5 +1,4 @@ import logging -import os import pathlib from datetime import datetime from typing import Any, Optional @@ -15,7 +14,7 @@ from tiled.type_aliases import JSON from tiled.utils import path_from_uri -from als_tiled.bl733.adapters.edf import parse_txt_accompanying_edf +from als_tiled.bl733.adapters.metadata import parse_txt_accompanying_edf logger = logging.getLogger(__name__) @@ -35,9 +34,9 @@ def __init__( specs: Optional[list[Spec]] = None, **kwargs: Optional[Any], ) -> None: - """Adapter for a detector image saved as .gb produced at ALS beamline 7.3.3""" - filepath = path_from_uri(data_uri) - data = np.fromfile(filepath, dtype=" "GeneralBinaryPilatus2MAdapter": return cls(data_uri, **kwargs) + @staticmethod + def _read_edf(filepath_edf: pathlib.Path) -> tuple[dict, datetime | None]: + """Read one EDF file and its companion .txt, returning (metadata, date).""" + metadata_txt = parse_txt_accompanying_edf(filepath_edf) + if not filepath_edf.is_file(): + return metadata_txt, None + with fabio.open(filepath_edf) as edf_file: + header = edf_file.header + date = datetime.strptime(header["Date"], "%a %b %d %H:%M:%S %Y") + return {**metadata_txt, **header}, date + + @staticmethod + def _parse_accompanying_metadata(filepath_gb: pathlib.Path) -> dict: + """Read the hi and lo EDF companions for a .gb file and merge their metadata.""" + filepath_edf_hi = pathlib.Path( + str(filepath_gb.with_suffix(".edf")).replace("sfloat", "hi") + ) + filepath_edf_lo = pathlib.Path( + str(filepath_gb.with_suffix(".edf")).replace("sfloat", "lo") + ) -def combine_edf_metadata_for_gb(hi_dict, lo_dict): - """Combine two dictionaries into one. - - Take the metadata from both and if the same key is present in both, - keep the values of both but index it with hi and lo - - Parameters - ---------- - hi_dict: dict - Dictionary containing metadata from the hi .edf file. - lo_dict: dict - Dictionary containing metadata from the lo .edf file - """ - combined_dict = dict() - - # get all the unique keys from both dictionaries - combined_keys = set(hi_dict.keys()).union(set(lo_dict.keys())) + metadata_hi, date_hi = GeneralBinaryPilatus2MAdapter._read_edf(filepath_edf_hi) + metadata_lo, date_lo = GeneralBinaryPilatus2MAdapter._read_edf(filepath_edf_lo) - # check if the values match for the same key in both dictionaries - for key in combined_keys: - hi_val = hi_dict.get(key) - lo_val = lo_dict.get(key) + combined_metadata = GeneralBinaryPilatus2MAdapter._combine_metadata( + metadata_hi, metadata_lo + ) - if hi_val == lo_val: - # If values are the same, keep one entry - combined_dict[key] = hi_val + if date_hi is not None and date_lo is not None: + combined_metadata["Date"] = date_hi if date_hi > date_lo else date_lo else: - # If values are different, add both with distinct keys - if hi_val is not None: - combined_dict[f"{key}_hi"] = hi_val - if lo_val is not None: - combined_dict[f"{key}_lo"] = lo_val - - return combined_dict - - -def parse_edf_accompanying_gb(file_path): - """Parse a .edf file produced at ALS beamline 7.3.3 into a dictionary. - - Parameters - ---------- - file_path: str or pathlib.Path - Filepath of the .edf file. - """ - - # Generate the hi edf file path - edf_hi_filepath = None - if isinstance(file_path, str): - edf_hi_filepath = file_path.replace("sfloat_2m.gb", "hi_2m.edf") - if isinstance(file_path, pathlib.Path): - edf_hi_filepath = file_path.with_suffix(".edf") - edf_hi_filepath = pathlib.Path(str(edf_hi_filepath).replace("sfloat", "hi")) - - edf_hi_metadata_dict = parse_txt_accompanying_edf(edf_hi_filepath) - - # If the .txt file exists, the metadata is extracted from it - # In case the .txt file does not exist: - # - set the date as None, - # - An empty metadata dictionary is initilized (returned from - # parse_txt_accompanying_edf()) - if not os.path.isfile(edf_hi_filepath): - hi_date = None - else: - hi_file = fabio.open(edf_hi_filepath) - hi_header = hi_file.header - hi_date_str = hi_header.get("Date") - # Parse the string to convert to datetime object - hi_date = datetime.strptime(hi_date_str, "%a %b %d %H:%M:%S %Y") - # Combine the metadata dictionaries - from header and .txt file - edf_hi_metadata_dict = {**edf_hi_metadata_dict, **hi_header} - - edf_lo_filepath = None - if isinstance(file_path, str): - edf_lo_filepath = file_path.replace("sfloat_2m.gb", "lo_2m.edf") - if isinstance(file_path, pathlib.Path): - edf_lo_filepath = file_path.with_suffix(".edf") - edf_lo_filepath = pathlib.Path(str(edf_lo_filepath).replace("sfloat", "lo")) - - edf_lo_metadata_dict = parse_txt_accompanying_edf(edf_lo_filepath) - - # If the .txt file exists, the metadata is extracted from it - # In case the .txt file does not exist: - # - set the date as None, - # - An empty metadata dictionary is initilized (returned from - # parse_txt_accompanying_edf()) - if not os.path.isfile(edf_lo_filepath): - lo_date = None - else: - lo_file = fabio.open(edf_lo_filepath) - lo_header = lo_file.header - lo_date_str = lo_header.get("Date") - # Parse the string to convert to datetime object - lo_date = datetime.strptime(lo_date_str, "%a %b %d %H:%M:%S %Y") - # Combine the metadata dictionaries - from header and .txt file - edf_lo_metadata_dict = {**edf_lo_metadata_dict, **lo_header} - - # Combine the metadata dictionaries - gb_dictionary = combine_edf_metadata_for_gb( - edf_hi_metadata_dict, edf_lo_metadata_dict - ) - - # Compare two dates and select the later one, - # but the string version for better readability - if hi_date is not None and lo_date is not None: - gb_dictionary["Date"] = hi_date if hi_date > lo_date else lo_date - else: - gb_dictionary["Date"] = hi_date if hi_date is not None else lo_date - - return gb_dictionary + combined_metadata["Date"] = date_hi if date_hi is not None else date_lo + + return combined_metadata + + @staticmethod + def _combine_metadata(metadata_hi: dict, metadata_lo: dict) -> dict: + """Combine metadata from hi and lo EDF files. + + Keys with identical values are kept once. Keys with different values are + suffixed with _hi and _lo. + """ + combined_metadata = {} + for key in set(metadata_hi) | set(metadata_lo): + value_hi = metadata_hi.get(key) + value_lo = metadata_lo.get(key) + if value_hi == value_lo: + combined_metadata[key] = value_hi + else: + if value_hi is not None: + combined_metadata[f"{key}_hi"] = value_hi + if value_lo is not None: + combined_metadata[f"{key}_lo"] = value_lo + return combined_metadata diff --git a/src/als_tiled/bl733/adapters/metadata.py b/src/als_tiled/bl733/adapters/metadata.py new file mode 100644 index 0000000..f9d12a0 --- /dev/null +++ b/src/als_tiled/bl733/adapters/metadata.py @@ -0,0 +1,36 @@ +import logging +import pathlib +from typing import Any + +logger = logging.getLogger(__name__) + + +def parse_txt_accompanying_edf(filepath_edf: pathlib.Path) -> dict[str, Any]: + """Parse the .txt metadata file accompanying an EDF file at ALS beamline 7.3.3. + + Parameters + ---------- + filepath_edf: + Path to the .edf file. The companion .txt is expected at the same path + with the extension replaced by .txt. + """ + filepath_txt = filepath_edf.with_suffix(".txt") + + if not filepath_txt.is_file(): + logger.warning(f"{filepath_edf} has no corresponding .txt.") + return {} + + with open(filepath_txt) as f: + lines = f.readlines() + + # Lines have the format "key: value" or are bare values with no key. + metadata: dict[str, Any] = {} + unnamed_count = 0 + for line in lines: + before, sep, after = line.partition(":") + if sep: + metadata[before.strip()] = after.strip() + elif before.strip() != "!0": + metadata[f"unnamed_{unnamed_count}"] = before.strip() + unnamed_count += 1 + return metadata From 6c41eef135b5333b49d66784c3f8560302317428 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 19 Mar 2026 12:18:27 -0700 Subject: [PATCH 06/24] Add unit tests and example config for edf and gp adapters --- example_configs/bl733/config.yaml | 21 ++ pyproject.toml | 2 + tests/bl733/conftest.py | 392 ++++++++++++++++++++++++++++++ tests/bl733/test_edf.py | 77 ++++++ tests/bl733/test_gb.py | 120 +++++++++ tests/conftest.py | 5 + 6 files changed, 617 insertions(+) create mode 100644 example_configs/bl733/config.yaml create mode 100644 tests/bl733/conftest.py diff --git a/example_configs/bl733/config.yaml b/example_configs/bl733/config.yaml new file mode 100644 index 0000000..16d2b89 --- /dev/null +++ b/example_configs/bl733/config.yaml @@ -0,0 +1,21 @@ +file_extensions: + edf: application/x-edf + gb: application/x-gb + +# Placeholder for future exporters that would allow download of arrays as edf or gb +# To be determined if this should include download of metadata as txt files +#media_types: +# array: +# application/x-edf: als_tiled.bl733.adapters.edf:export_edf +# application/x-gb: als_tiled.bl733.adapters.gb:export_gb + +trees: + - path: / + tree: tiled.catalog:from_uri + args: + uri: ./data/catalog.db + readable_storage: [./data] + init_if_not_exists: true + adapters_by_mimetype: + application/x-edf: als_tiled.bl733.adapters.edf:EDFAdapter + application/x-gb: als_tiled.bl733.adapters.gb:GeneralBinaryPilatus2MAdapter diff --git a/pyproject.toml b/pyproject.toml index c92143a..45ee8dc 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ dev = [ "pytest>=7.0", "pytest-cov", + "pytest-asyncio", "black", "isort", "flake8", @@ -38,6 +39,7 @@ dev = [ test = [ "pytest>=7.0", "pytest-cov", + "pytest-asyncio", ] tiled_all = [ diff --git a/tests/bl733/conftest.py b/tests/bl733/conftest.py new file mode 100644 index 0000000..616a74b --- /dev/null +++ b/tests/bl733/conftest.py @@ -0,0 +1,392 @@ +import fabio +import numpy as np +import pytest + +SAMPLE_TXT_CONTENT = """\ +401.000 +10440.000 +235561.000 +Izero: 401.000 +I1 normalization: 235561.000 +Diode normalization: 10440.000 +Normalize by: Diode +Exposure time s: 10.000 +ALS Proposal #: 00622 +ALS ESAF #: 00622-001 +PI: PILastname +Calibration image path: /spot733-data/raw/userdata +Motors: 64 +Sample X Stage: -88.245200 +Sample Y Stage: 3.128650 +Sample X Stage Large: 5.000500 +Sample Y Stage Large: 34.997600 +Sample Alpha Stage: 0.130909 +Sample Phi Stage: 0.051610 +M201 Feedback: 0.010372 +M1 Pitch: 0.202033 +M1 Bend: 48.399999 +BS X: -1.357906 +BS Y: 9.981358 +Sample Y Stage Fine XPS: 0.000000 +Sample Y Stage Labjack: 0.000000 +Sample Rotation Stage: 0.000000 +Slit1 top: 9.263500 +Slit1 bottom: 11.732500 +Slit1 right: 11.109500 +Slit1 left: 11.356000 +Exit Slit top: 7.781500 +Exit Slit bottom: 10.976500 +Exit Slit left: 6.961500 +Exit Slit right: 8.903000 +Sample Y Stage Robot: 0.000000 +Detector Horizontal: 0.000000 +Detector Vertical: 0.000000 +GIWAXS beamstop X: 0.000000 +GIWAXS beamstop Y: 0.000000 +Beamstop X: 0.000000 +Beamstop Y: 0.000000 +Detector Left Motor: 0.000000 +Detector Right Motor: 0.000000 +Motorized Lab Jack: 0.000000 +M1 Alignment Tune: 0.202033 +print head height: 0.000000 +Rotation New: 0.000000 +printer roll: 0.000000 +EZ fast tension stage: 0.000000 +Motorized Lab Jack1: 0.000000 +Sample Rotation Stage ESP: 0.000000 +Printing motor: 0.000000 +GIWAXS beamstop Y thorlabs: 0.000000 +Sample Y Stage Arthur: 0.000000 +Flight Tube Horizontal: 0.000000 +Flight Tube Vertical: 0.000000 +Hacked Ager Stage: 0.000000 +Sample Rotation Stage Miller: 0.000000 +Mono Angle: 0.000000 +Xtal2 Pico 1 Feedback: 0.000000 +Xtal2 Pico 2 Feedback: 0.000000 +Xtal2 Pico 3 Feedback: 0.000000 +Xtal2 Pico 1: 0.000000 +Xtal2 Pico 2: 0.000000 +Xtal2 Pico 3: 0.000000 +Sample Y Stage_old: 0.000000 +AO Waveform: 0.000000 +AO0Traingle: 0.000000 +AO1SquareWave: 0.000000 +Lamda Zup heater: 0.000000 +thermofeedback: 0.000000 +spinCoaterMotor: 0.000000 +BK Power Supply: 0.000000 +BK Control: 0.000000 +EPOS: 0.000000 +EPOS2: 0.000000 +DIOs: 14 +SAXS Protector: 0.000000 +Beamline Shutter Closed: 0.000000 +Beam Current Over Threshold: 1.000000 +Slit 1 in Position: 1.000000 +Slit 2 in Position: 1.000000 +Temp Beamline Shutter Open: 0.000000 +Beamline Shutter Open: 1.000000 +Feedback Interlock: 1.000000 +Beamline Pass Beam: 1.000000 +VacuumBadAtIG304: 0.000000 +Gate Shutter: 0.000000 +Bruker pulses: 0.000000 +Slit Top Good: 1.000000 +Slit Bottom Good: 1.000000 +AIs: 38 +Izero: 401.000000 +GiSAXS Beamstop: 0.521687 +slit1 top current: NaN +slit1 bottom current: NaN +Beam Current: 499.509338 +Beamline Shutter AI: 1.000000 +Beamline Pass Beam AI: 1.000000 +Vertical Beam Position: NaN +IG304Epics: 11.052951 +Izero AI: 0.000000 +I1 AI: -0.235249 +PHI Alignment Beamstop: 0.235471 +AI Channel 6: -0.238456 +AI Channel 7: 0.003273 +thermocoupleAI3: -2.696180 +Pyro: NaN +Raytec - Room Temp: 0.000000 +BK Amps: NaN +BK Volts: NaN +BK Power: NaN +BK Resistance: NaN +Pilatus 300KW trigger pulse: 0.000000 +Pilatus 1M Trigger Pulse: 0.000000 +PCO Invert: 0.000000 +Gate: 0.000000 +I1: 235561.000000 +GiSAXS Beamstop Counter: 522894.000000 +Sum of Slit Current: 10.633427 +Pilatus 100K exp out: 0.000000 +Kramer strain data: 0.000000 +Xtal2 Pico 1: NaN +Xtal2 Pico 2: NaN +Xtal2 Pico 3: NaN +M1 Pitch: 0.202057 +ABS(Vertical Beam Position): NaN +AIat6221 Channel 6: NaN +DCVoltageMonitor: 0.000000 +TC: 0.000000 +!0 +""" + +SAMPLE_TXT_CONTENT_HI = """\ +1070.000 +80.231 +1320.000 +Izero: 1070.000 +I1 normalization: 1320.000 +Diode normalization: 80.231 +Normalize by: Diode +Exposure time s: 0.100 +ALS Proposal #: ALS-00000 +ALS ESAF #: ALS-00000-000 +PI: PILastname +Calibration image path: /spot733-data/raw/userdata +Motors: 71 +Vertical Lift: 100.000000 +Sample X Stage: -6.894300 +Sample Y Stage: 12.023278 +Sample X Stage Large: -25.500000 +Sample Y Stage large: 26.053500 +Sample Alpha Stage: 0.007603 +Sample Phi Stage: 0.000000 +Kapton blocker: 0.000000 +Pinhole Vertical: 0.000000 +Pinhole Horizontal: 0.000000 +M201 Feedback: -0.898026 +M1 Pitch: 0.183040 +M1 Bend: 48.399999 +BS X: -2.540780 +BS Y: 10.369954 +Sample Y Stage Labjack: 0.000000 +Sample Rotation Stage: 0.000000 +Slit1 top: 8.854500 +Slit1 bottom: 11.534500 +Slit1 right: 10.679500 +Slit1 left: 11.434500 +Exit Slit top: 7.082500 +Exit Slit bottom: 7.384500 +Exit Slit left: 7.683000 +Exit Slit right: 8.030000 +Kapton Blocker Horizontal: 0.000000 +Kapton Blocker Vertical: 0.000000 +Sample Y Stage Robot: 0.000000 +Detector Horizontal: 0.000000 +Detector Vertical: 0.000000 +GIWAXS beamstop X: 0.000000 +GIWAXS beamstop Y: 0.000000 +Beamstop X: 0.000000 +Beamstop Y: 0.000000 +Detector Left Motor: 0.000000 +Detector Right Motor: 0.000000 +Motorized Lab Jack: 0.000000 +M1 Alignment Tune: 0.183040 +print head height: 0.000000 +Rotation New: 0.000000 +printer roll: 0.000000 +EZ fast tension stage: 0.000000 +Motorized Lab Jack1: 0.000000 +Sample Rotation Stage ESP: 0.000000 +Printing motor: 0.000000 +GIWAXS beamstop Y thorlabs: 0.000000 +Sample Y Stage Arthur: 0.000000 +Flight Tube Horizontal: 0.000000 +Flight Tube Vertical: 0.000000 +Hacked Ager Stage: 0.000000 +Sample Rotation Stage Miller: 0.000000 +Mono Angle: 0.000000 +Xtal2 Pico 1 Feedback: 0.000000 +Xtal2 Pico 2 Feedback: 0.000000 +Xtal2 Pico 3 Feedback: 0.000000 +Xtal2 Pico 1: 0.000000 +Xtal2 Pico 2: 0.000000 +Xtal2 Pico 3: 0.000000 +Sample Y Stage_old: 0.000000 +AO Waveform: 0.000000 +AO0Traingle: 0.000000 +AO1SquareWave: 0.000000 +Lamda Zup heater: 0.000000 +thermofeedback: 0.000000 +spinCoaterMotor: 0.000000 +BK Power Supply: 0.000000 +BK Control: 0.000000 +EPOS: 0.000000 +EPOS2: 0.000000 +Fake Motor 1: 0.000000 +Fake Motor 2: 0.000000 +DIOs: 14 +SAXS Protector: 0.000000 +Beamline Shutter Closed: 0.000000 +Beam Current Over Threshold: 1.000000 +Slit 1 in Position: 1.000000 +Slit 2 in Position: 1.000000 +Temp Beamline Shutter Open: 0.000000 +Beamline Shutter Open: 1.000000 +Feedback Interlock: 0.000000 +Beamline Pass Beam: 1.000000 +VacuumBadAtIG304: 0.000000 +Gate Shutter: 0.000000 +Bruker pulses: 0.000000 +Slit Top Good: 1.000000 +Slit Bottom Good: 1.000000 +AIs: 39 +Izero: 1070.000000 +GiSAXS Beamstop: 0.378450 +slit1 top current: NaN +slit1 bottom current: NaN +Beam Current: 500.487475 +Beamline Shutter AI: 1.000000 +Beamline Pass Beam AI: 1.000000 +Vertical Beam Position: NaN +IG304Epics: 59.738999 +Izero AI: 0.000000 +I1 AI: -0.124054 +PHI Alignment Beamstop: 0.117137 +AI Channel 6: -0.087247 +AI Channel 7: 0.003600 +thermocoupleAI3: -2.696388 +Pyro: NaN +Raytec - Room Temp: 0.000000 +BK Amps: NaN +BK Volts: NaN +BK Power: NaN +BK Resistance: NaN +Pilatus 300KW trigger pulse: 0.000000 +Pilatus 1M Trigger Pulse: 0.000000 +PCO Invert: 0.000000 +Gate: 0.000000 +I1: 1320.000000 +GiSAXS Beamstop Counter: 4007.000000 +Sum of Slit Current: 4.565782 +Pilatus 100K exp out: 0.000000 +Kramer strain data: 0.000000 +Xtal2 Pico 1: NaN +Xtal2 Pico 2: NaN +Xtal2 Pico 3: NaN +M1 Pitch: 0.183040 +ABS(Vertical Beam Position): NaN +AIat6221 Channel 6: NaN +DCVoltageMonitor: 0.000000 +TC: 0.000000 +Fake Motor 1: 0.000000 +!0 +""" + +# Keys that differ between hi and lo are replaced; keys only in hi or lo are appended. +# All keys in these files are shared between hi and lo — only values differ. +SAMPLE_TXT_CONTENT_LO = ( + SAMPLE_TXT_CONTENT_HI + # Unnamed header lines (Izero, Diode, I1 raw counts) + .replace("1070.000\n80.231\n1320.000\n", "1071.000\n80.199\n1325.000\n") + # Named scan metadata + .replace("Izero: 1070.000\n", "Izero: 1071.000\n") + .replace("I1 normalization: 1320.000\n", "I1 normalization: 1325.000\n") + .replace("Diode normalization: 80.231\n", "Diode normalization: 80.199\n") + # Motor readbacks that drifted between hi and lo exposures + .replace("M201 Feedback: -0.898026\n", "M201 Feedback: -0.898142\n") + .replace( + "M1 Pitch: 0.183040\n", "M1 Pitch: 0.183044\n" + ) # two occurrences, both change + .replace("M1 Alignment Tune: 0.183040\n", "M1 Alignment Tune: 0.183044\n") + # AI channel readbacks + .replace("Izero: 1070.000000\n", "Izero: 1071.000000\n") + .replace("GiSAXS Beamstop: 0.378450\n", "GiSAXS Beamstop: 0.378298\n") + .replace("Beam Current: 500.487475\n", "Beam Current: 500.640318\n") + .replace("I1 AI: -0.124054\n", "I1 AI: -0.124429\n") + .replace("PHI Alignment Beamstop: 0.117137\n", "PHI Alignment Beamstop: 0.117747\n") + .replace("AI Channel 6: -0.087247\n", "AI Channel 6: -0.087482\n") + .replace("AI Channel 7: 0.003600\n", "AI Channel 7: 0.003118\n") + .replace("thermocoupleAI3: -2.696388\n", "thermocoupleAI3: -2.696346\n") + .replace("I1: 1320.000000\n", "I1: 1325.000000\n") + .replace( + "GiSAXS Beamstop Counter: 4007.000000\n", + "GiSAXS Beamstop Counter: 3988.000000\n", + ) + .replace("Sum of Slit Current: 4.565782\n", "Sum of Slit Current: 4.569032\n") +) + +SAMPLE_EDF_HEADER = { + "HeaderID": "EH:000001:000000:000000", + "Image": "1", + "VersionNumber": "0.10", + "ByteOrder": "LowByteFirst", + "DataType": "SignedInteger", + "Dim_1": "1475", + "Dim_2": "1679", + "Date": "Mon Mar 25 17:06:51 2024", + "count_time": "10.000000000", + "title": "# Pixel_size 172e-6 m x 172e-6 m", + "run": "0", +} + +# Hi is acquired after lo; the GB adapter selects the later date. +SAMPLE_EDF_HEADER_HI = { + **SAMPLE_EDF_HEADER, + "count_time": "0.100000001", + "Date": "Wed Oct 29 20:15:23 2025", +} +SAMPLE_EDF_HEADER_LO = { + **SAMPLE_EDF_HEADER, + "count_time": "0.100000001", + "Date": "Wed Oct 29 20:15:16 2025", +} + +SAMPLE_EDF_DATA = np.arange(1475 * 1679, dtype=np.int32).reshape(1679, 1475) + +SAMPLE_GB_DATA = np.arange(1475 * 1679, dtype=" Date: Thu, 19 Mar 2026 12:20:28 -0700 Subject: [PATCH 07/24] Guard against spec duplication on adapter init --- src/als_tiled/bl733/adapters/edf.py | 6 +++++- src/als_tiled/bl733/adapters/gb.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/als_tiled/bl733/adapters/edf.py b/src/als_tiled/bl733/adapters/edf.py index 9c3c3f6..2da8fd1 100644 --- a/src/als_tiled/bl733/adapters/edf.py +++ b/src/als_tiled/bl733/adapters/edf.py @@ -40,11 +40,15 @@ def __init__( **parse_txt_accompanying_edf(filepath), } + edf_spec = Spec("als-bl733-edf", version="1.0") + specs = list(specs or []) + if edf_spec not in specs: + specs.append(edf_spec) super().__init__( array=array, structure=structure or ArrayStructure.from_array(array), metadata=metadata, - specs=(specs or []) + [Spec("als-bl733-edf", version="1.0")], + specs=specs, **kwargs, ) diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py index 6f763e4..21c8a80 100644 --- a/src/als_tiled/bl733/adapters/gb.py +++ b/src/als_tiled/bl733/adapters/gb.py @@ -50,11 +50,15 @@ def __init__( **GeneralBinaryPilatus2MAdapter._parse_accompanying_metadata(filepath_gb), } + gb_spec = Spec("als-bl733-gb", version="1.0") + specs = list(specs or []) + if gb_spec not in specs: + specs.append(gb_spec) super().__init__( array=array, structure=structure or ArrayStructure.from_array(array), metadata=metadata, - specs=(specs or []) + [Spec("als-bl733-gb", version="1.0")], + specs=specs, **kwargs, ) From 78d965a47909b9cc6c90008eb63b48c1b55f413e Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Fri, 20 Mar 2026 09:49:33 -0700 Subject: [PATCH 08/24] Add `__init__.py` to `bl733` and bl733/adapters` --- src/als_tiled/bl733/__init__.py | 0 src/als_tiled/bl733/adapters/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100755 src/als_tiled/bl733/__init__.py create mode 100644 src/als_tiled/bl733/adapters/__init__.py diff --git a/src/als_tiled/bl733/__init__.py b/src/als_tiled/bl733/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/src/als_tiled/bl733/adapters/__init__.py b/src/als_tiled/bl733/adapters/__init__.py new file mode 100644 index 0000000..e69de29 From b333712bcdfb837d03bde2aff31d1e81956c6c8c Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 25 Mar 2026 11:03:17 -0700 Subject: [PATCH 09/24] Update base image to `ghcr.io/bluesky/tiled:0.2.8` and make `tiled[client]>=0.2.8` a core dependency in `pyproject.toml` --- Containerfile | 2 +- pyproject.toml | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Containerfile b/Containerfile index 8d5f187..8a7a68c 100755 --- a/Containerfile +++ b/Containerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/bluesky/tiled:0.2.3 AS base +FROM ghcr.io/bluesky/tiled:0.2.8 AS base USER root diff --git a/pyproject.toml b/pyproject.toml index 45ee8dc..f11d721 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ - + "tiled[client]>=0.2.8", ] [project.optional-dependencies] @@ -41,15 +41,9 @@ test = [ "pytest-cov", "pytest-asyncio", ] - -tiled_all = [ - "tiled[all]" +tiled-all = [ + "tiled[all]>=0.2.8", ] - -tiled_client = [ - "tiled[client]" -] - bl733 = [ "fabio", ] From 94bbd014849939fe9170e75e472be3aeb7b0725a Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 25 Mar 2026 11:05:59 -0700 Subject: [PATCH 10/24] Add `mypy` pre-commit and update others --- .pre-commit-config.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 613f192..f56cd3a 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -10,19 +10,26 @@ repos: - id: debug-statements - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 26.3.1 hooks: - id: black language_version: python3 - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 8.0.1 hooks: - id: isort args: ["--profile", "black"] - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 + rev: 7.3.0 hooks: - id: flake8 args: [--max-line-length=88, --extend-ignore=E203] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.19.1 + hooks: + - id: mypy + args: [--strict] + additional_dependencies: [types-cachetools] From 24e032bc3b08bf4d12adb43fc83615e948c3a5fd Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 25 Mar 2026 18:56:14 -0700 Subject: [PATCH 11/24] Refactor tests to use fixtures and remove backend fixture --- tests/bl733/test_edf.py | 9 +++++---- tests/bl733/test_gb.py | 28 ++++++++++++++++++++++++---- tests/conftest.py | 5 ----- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/tests/bl733/test_edf.py b/tests/bl733/test_edf.py index cded539..2aaaa30 100644 --- a/tests/bl733/test_edf.py +++ b/tests/bl733/test_edf.py @@ -1,5 +1,6 @@ import numpy as np import pytest +import pytest_asyncio from tiled.client import Context, from_context from tiled.client.register import register from tiled.server.app import build_app_from_config @@ -28,7 +29,7 @@ def _tiled_config(tmp_path): } -@pytest.fixture +@pytest_asyncio.fixture async def edf_client(tmp_path, bl733_edf_path): with Context.from_app(build_app_from_config(_tiled_config(tmp_path))) as context: client = from_context(context) @@ -41,7 +42,7 @@ async def edf_client(tmp_path, bl733_edf_path): yield client -@pytest.mark.anyio(backend="asyncio") +@pytest.mark.asyncio async def test_edf_reads_array(edf_client): """Array returned by the tiled client matches the data written to disk.""" expected = np.arange(1475 * 1679, dtype=np.int32).reshape(1679, 1475) @@ -49,7 +50,7 @@ async def test_edf_reads_array(edf_client): np.testing.assert_array_equal(result, expected) -@pytest.mark.anyio(backend="asyncio") +@pytest.mark.asyncio async def test_edf_metadata_includes_header_fields(edf_client): """EDF header fields appear in the tiled entry metadata.""" metadata = edf_client["scan_name_2m"].metadata @@ -59,7 +60,7 @@ async def test_edf_metadata_includes_header_fields(edf_client): assert metadata["Dim_2"] == "1679" -@pytest.mark.anyio(backend="asyncio") +@pytest.mark.asyncio async def test_edf_metadata_includes_txt_fields(edf_client): """Companion .txt fields appear in the tiled entry metadata.""" metadata = edf_client["scan_name_2m"].metadata diff --git a/tests/bl733/test_gb.py b/tests/bl733/test_gb.py index 3cdbabb..ffa407d 100644 --- a/tests/bl733/test_gb.py +++ b/tests/bl733/test_gb.py @@ -1,5 +1,6 @@ import numpy as np import pytest +import pytest_asyncio from tiled.client import Context, from_context from tiled.client.register import register from tiled.server.app import build_app_from_config @@ -31,7 +32,7 @@ def _tiled_config(tmp_path): } -@pytest.fixture +@pytest_asyncio.fixture async def gb_client(tmp_path, bl733_gb_path): with Context.from_app(build_app_from_config(_tiled_config(tmp_path))) as context: client = from_context(context) @@ -44,7 +45,7 @@ async def gb_client(tmp_path, bl733_gb_path): yield client -@pytest.mark.anyio +@pytest.mark.asyncio async def test_gb_reads_array(gb_client): """Array returned by the tiled client matches the data written to disk.""" expected = np.arange(PIXELS_X * PIXELS_Y, dtype=" Date: Wed, 25 Mar 2026 19:01:44 -0700 Subject: [PATCH 12/24] Add `__init__.py` files to tests and tests/bl733 directories --- tests/__init__.py | 0 tests/bl733/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/bl733/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/bl733/__init__.py b/tests/bl733/__init__.py new file mode 100644 index 0000000..e69de29 From a790c9c7e40526c77f58347fdb8be8e9acabaa1f Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 25 Mar 2026 19:20:31 -0700 Subject: [PATCH 13/24] Add mypy configuration `.mypy.ini` and update type hints in code --- .mypy.ini | 19 +++++++++++++++++++ .pre-commit-config.yaml | 1 - pyproject.toml | 6 ------ src/als_tiled/bl733/adapters/gb.py | 8 +++++--- tests/conftest.py | 2 +- tests/test_main.py | 2 +- 6 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 .mypy.ini diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 0000000..1e4c385 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,19 @@ +[mypy] +python_version = 3.11 +ignore_errors = True +ignore_missing_imports = True +disallow_untyped_defs = False + +[mypy-als_tiled.bl733.adapters.*] +ignore_errors = False +ignore_missing_imports = False +check_untyped_defs = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +disallow_untyped_calls = True + +[mypy-fabio] +ignore_missing_imports = True + +[mypy-tiled.*] +ignore_missing_imports = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f56cd3a..39d2c11 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,5 +31,4 @@ repos: rev: v1.19.1 hooks: - id: mypy - args: [--strict] additional_dependencies: [types-cachetools] diff --git a/pyproject.toml b/pyproject.toml index f11d721..5b9a7a3 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,12 +85,6 @@ profile = "black" multi_line_output = 3 line_length = 88 -[tool.mypy] -python_version = "3.11" -warn_return_any = true -warn_unused_configs = true -disallow_untyped_defs = true - [tool.pytest.ini_options] minversion = "7.0" addopts = "-ra -q --cov=als_tiled --cov-report=term-missing" diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py index 21c8a80..fd673a4 100644 --- a/src/als_tiled/bl733/adapters/gb.py +++ b/src/als_tiled/bl733/adapters/gb.py @@ -81,7 +81,7 @@ def from_uris( return cls(data_uri, **kwargs) @staticmethod - def _read_edf(filepath_edf: pathlib.Path) -> tuple[dict, datetime | None]: + def _read_edf(filepath_edf: pathlib.Path) -> tuple[dict[str, Any], datetime | None]: """Read one EDF file and its companion .txt, returning (metadata, date).""" metadata_txt = parse_txt_accompanying_edf(filepath_edf) if not filepath_edf.is_file(): @@ -92,7 +92,7 @@ def _read_edf(filepath_edf: pathlib.Path) -> tuple[dict, datetime | None]: return {**metadata_txt, **header}, date @staticmethod - def _parse_accompanying_metadata(filepath_gb: pathlib.Path) -> dict: + def _parse_accompanying_metadata(filepath_gb: pathlib.Path) -> dict[str, Any]: """Read the hi and lo EDF companions for a .gb file and merge their metadata.""" filepath_edf_hi = pathlib.Path( str(filepath_gb.with_suffix(".edf")).replace("sfloat", "hi") @@ -116,7 +116,9 @@ def _parse_accompanying_metadata(filepath_gb: pathlib.Path) -> dict: return combined_metadata @staticmethod - def _combine_metadata(metadata_hi: dict, metadata_lo: dict) -> dict: + def _combine_metadata( + metadata_hi: dict[str, Any], metadata_lo: dict[str, Any] + ) -> dict[str, Any]: """Combine metadata from hi and lo EDF files. Keys with identical values are kept once. Keys with different values are diff --git a/tests/conftest.py b/tests/conftest.py index 3531d76..3998466 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,6 @@ @pytest.fixture -def sample_data(): +def sample_data() -> dict[str, str]: """Sample data for testing.""" return {"test": "data"} diff --git a/tests/test_main.py b/tests/test_main.py index 4b766ab..3b5343d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,2 +1,2 @@ -def test_main_function(): +def test_main_function() -> None: pass From a8b44240718284eaa57ba91f9ef5be580607b0e9 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 25 Mar 2026 19:28:01 -0700 Subject: [PATCH 14/24] Add `als_tiled[bl733]` to test dependencies, Otherwise test CI fails with missing module error --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5b9a7a3..0f7cb6d 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ test = [ "pytest>=7.0", "pytest-cov", "pytest-asyncio", + "als_tiled[bl733]", ] tiled-all = [ "tiled[all]>=0.2.8", From b44c4f90d4fdc58d02b83345945b7138916a6f70 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 25 Mar 2026 19:34:12 -0700 Subject: [PATCH 15/24] Update mypy configuration for tests to not ignore errors --- .mypy.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.mypy.ini b/.mypy.ini index 1e4c385..a5a3c5a 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -12,6 +12,10 @@ disallow_untyped_defs = True disallow_incomplete_defs = True disallow_untyped_calls = True +[mypy-tests.*] +ignore_errors = False +disallow_untyped_defs = False + [mypy-fabio] ignore_missing_imports = True From 9a6dd60d68e70bb7f3feb1e49f2d75902710fd29 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 25 Mar 2026 19:37:14 -0700 Subject: [PATCH 16/24] Tests need Tiled server dependencies --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0f7cb6d..beae133 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ test = [ "pytest>=7.0", "pytest-cov", "pytest-asyncio", - "als_tiled[bl733]", + "als_tiled[bl733,tiled-all]", ] tiled-all = [ "tiled[all]>=0.2.8", From 7531ae7c349c890ed6f469deade6db1de2387e7a Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Wed, 25 Mar 2026 19:53:46 -0700 Subject: [PATCH 17/24] Expands mypy type checks to include tests --- .github/workflows/ci-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 923a8d7..2b02a61 100755 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -47,7 +47,7 @@ jobs: - name: Type checking with mypy run: | - mypy src + mypy src tests test: runs-on: ubuntu-latest From 32598f82352d2adacd7d0c424997abc91a060b42 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 26 Mar 2026 12:00:05 -0700 Subject: [PATCH 18/24] Use already defined detector size constants for tests --- .mypy.ini | 1 + tests/bl733/conftest.py | 10 ++++++++-- tests/bl733/test_edf.py | 5 ++++- tests/bl733/test_gb.py | 17 ++++++++++------- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.mypy.ini b/.mypy.ini index a5a3c5a..6e4c0ac 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,5 +1,6 @@ [mypy] python_version = 3.11 +mypy_path = src ignore_errors = True ignore_missing_imports = True disallow_untyped_defs = False diff --git a/tests/bl733/conftest.py b/tests/bl733/conftest.py index 616a74b..e248736 100644 --- a/tests/bl733/conftest.py +++ b/tests/bl733/conftest.py @@ -2,6 +2,8 @@ import numpy as np import pytest +from als_tiled.bl733.adapters.gb import PILATUS_2M_PIXELS_X, PILATUS_2M_PIXELS_Y + SAMPLE_TXT_CONTENT = """\ 401.000 10440.000 @@ -339,9 +341,13 @@ "Date": "Wed Oct 29 20:15:16 2025", } -SAMPLE_EDF_DATA = np.arange(1475 * 1679, dtype=np.int32).reshape(1679, 1475) +SAMPLE_EDF_DATA = np.arange( + PILATUS_2M_PIXELS_X * PILATUS_2M_PIXELS_Y, dtype=np.int32 +).reshape(PILATUS_2M_PIXELS_Y, PILATUS_2M_PIXELS_X) -SAMPLE_GB_DATA = np.arange(1475 * 1679, dtype=" Date: Thu, 26 Mar 2026 17:10:02 -0700 Subject: [PATCH 19/24] Only check header instead of full file for gb adapter --- src/als_tiled/bl733/adapters/gb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py index fd673a4..0ac42f7 100644 --- a/src/als_tiled/bl733/adapters/gb.py +++ b/src/als_tiled/bl733/adapters/gb.py @@ -86,8 +86,7 @@ def _read_edf(filepath_edf: pathlib.Path) -> tuple[dict[str, Any], datetime | No metadata_txt = parse_txt_accompanying_edf(filepath_edf) if not filepath_edf.is_file(): return metadata_txt, None - with fabio.open(filepath_edf) as edf_file: - header = edf_file.header + header = fabio.openheader(filepath_edf).header date = datetime.strptime(header["Date"], "%a %b %d %H:%M:%S %Y") return {**metadata_txt, **header}, date From deb0f68c45899ada30b32c609527bbb2b254c783 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 26 Mar 2026 17:13:28 -0700 Subject: [PATCH 20/24] Normalize date format across edf and gb adapters --- src/als_tiled/bl733/adapters/edf.py | 5 +++++ src/als_tiled/bl733/adapters/gb.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/als_tiled/bl733/adapters/edf.py b/src/als_tiled/bl733/adapters/edf.py index 2da8fd1..ee60e95 100644 --- a/src/als_tiled/bl733/adapters/edf.py +++ b/src/als_tiled/bl733/adapters/edf.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime from typing import Any, Optional import fabio @@ -34,6 +35,10 @@ def __init__( array = edf_file.data metadata_edf = edf_file.header + if "Date" in metadata_edf: + date = datetime.strptime(metadata_edf["Date"], "%a %b %d %H:%M:%S %Y") + metadata_edf["Date"] = date.isoformat() + metadata = { **(metadata or {}), **metadata_edf, diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py index 0ac42f7..46098db 100644 --- a/src/als_tiled/bl733/adapters/gb.py +++ b/src/als_tiled/bl733/adapters/gb.py @@ -107,10 +107,15 @@ def _parse_accompanying_metadata(filepath_gb: pathlib.Path) -> dict[str, Any]: metadata_hi, metadata_lo ) + date = None if date_hi is not None and date_lo is not None: - combined_metadata["Date"] = date_hi if date_hi > date_lo else date_lo - else: - combined_metadata["Date"] = date_hi if date_hi is not None else date_lo + date = date_hi if date_hi > date_lo else date_lo + elif date_hi is not None: + date = date_hi + elif date_lo is not None: + date = date_lo + if date is not None: + combined_metadata["Date"] = date.isoformat() return combined_metadata From 96894d65e0dc0836cd530918ad6f01f169afd2cf Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 26 Mar 2026 17:16:03 -0700 Subject: [PATCH 21/24] Install optional bl733 dependencies in docker image --- Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Containerfile b/Containerfile index 8a7a68c..7ac25f2 100755 --- a/Containerfile +++ b/Containerfile @@ -9,4 +9,4 @@ USER app COPY --chown=app:app pyproject.toml README.md ./ COPY --chown=app:app src/ ./src/ RUN python -m ensurepip -RUN python -m pip install --upgrade --no-cache-dir . +RUN python -m pip install --upgrade --no-cache-dir ".[bl733]" From de6538209b5edd0311e547190b48275e06beb1c6 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 26 Mar 2026 17:21:30 -0700 Subject: [PATCH 22/24] Debug logs for loading files --- src/als_tiled/bl733/adapters/edf.py | 1 + src/als_tiled/bl733/adapters/gb.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/als_tiled/bl733/adapters/edf.py b/src/als_tiled/bl733/adapters/edf.py index ee60e95..fa20316 100644 --- a/src/als_tiled/bl733/adapters/edf.py +++ b/src/als_tiled/bl733/adapters/edf.py @@ -30,6 +30,7 @@ def __init__( ) -> None: """Adapter for `.edf` files (e.g. PILATUS3 2M) at ALS beamline 7.3.3.""" filepath = path_from_uri(data_uri) + logger.debug("Loading EDF file produced by ALS beamline 7.3.3: %s", filepath) with fabio.open(filepath) as edf_file: array = edf_file.data diff --git a/src/als_tiled/bl733/adapters/gb.py b/src/als_tiled/bl733/adapters/gb.py index 46098db..c134398 100644 --- a/src/als_tiled/bl733/adapters/gb.py +++ b/src/als_tiled/bl733/adapters/gb.py @@ -36,6 +36,7 @@ def __init__( ) -> None: """Adapter for a stitched detector image .gb produced at ALS beamline 7.3.3.""" filepath_gb = path_from_uri(data_uri) + logger.debug("Loading GB file produced by ALS beamline 7.3.3: %s", filepath_gb) data = np.fromfile(filepath_gb, dtype=" tuple[dict[str, Any], datetime | No """Read one EDF file and its companion .txt, returning (metadata, date).""" metadata_txt = parse_txt_accompanying_edf(filepath_edf) if not filepath_edf.is_file(): + logger.warning( + f"GeneralBinary file is missing accompanying EDF file {filepath_edf}." + ) return metadata_txt, None header = fabio.openheader(filepath_edf).header date = datetime.strptime(header["Date"], "%a %b %d %H:%M:%S %Y") From 1036baedc1ef71db930b3696274d2e11717d6ae4 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 26 Mar 2026 17:23:28 -0700 Subject: [PATCH 23/24] No need to guard against date not being present in header --- src/als_tiled/bl733/adapters/edf.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/als_tiled/bl733/adapters/edf.py b/src/als_tiled/bl733/adapters/edf.py index fa20316..e48d02d 100644 --- a/src/als_tiled/bl733/adapters/edf.py +++ b/src/als_tiled/bl733/adapters/edf.py @@ -36,9 +36,8 @@ def __init__( array = edf_file.data metadata_edf = edf_file.header - if "Date" in metadata_edf: - date = datetime.strptime(metadata_edf["Date"], "%a %b %d %H:%M:%S %Y") - metadata_edf["Date"] = date.isoformat() + date = datetime.strptime(metadata_edf["Date"], "%a %b %d %H:%M:%S %Y") + metadata_edf["Date"] = date.isoformat() metadata = { **(metadata or {}), From 23723e3d1efa18981cc84f337badd2f4e246ba45 Mon Sep 17 00:00:00 2001 From: Wiebke Koepp Date: Thu, 26 Mar 2026 17:25:46 -0700 Subject: [PATCH 24/24] Delete `test_main.py` placeholder --- tests/test_main.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 tests/test_main.py diff --git a/tests/test_main.py b/tests/test_main.py deleted file mode 100644 index 3b5343d..0000000 --- a/tests/test_main.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_main_function() -> None: - pass