From 07c8447302c114903b94c16c45d2dc60095c02ff Mon Sep 17 00:00:00 2001 From: Jane Van Lam <75lam@cua.edu> Date: Thu, 19 Mar 2026 07:12:44 -0400 Subject: [PATCH] update packages for cp313, replace preadator --- .../{bumpversion.cfg => .bumpversion.cfg} | 2 +- .../polus-stack-z-slice-plugin/Dockerfile | 34 +-- .../polus-stack-z-slice-plugin/README.md | 2 +- .../images/polus-stack-z-slice-plugin/VERSION | 2 +- .../build-docker.sh | 2 +- .../polus-stack-z-slice-plugin/plugin.json | 5 +- .../polus-stack-z-slice-plugin/pyproject.toml | 18 ++ .../polus-stack-z-slice-plugin/run-plugin.sh | 2 +- .../src/__init__.py | 1 + .../polus-stack-z-slice-plugin/src/main.py | 275 ++++++++++-------- .../src/requirements.txt | 3 +- 11 files changed, 196 insertions(+), 150 deletions(-) rename transforms/images/polus-stack-z-slice-plugin/{bumpversion.cfg => .bumpversion.cfg} (80%) create mode 100644 transforms/images/polus-stack-z-slice-plugin/pyproject.toml create mode 100644 transforms/images/polus-stack-z-slice-plugin/src/__init__.py diff --git a/transforms/images/polus-stack-z-slice-plugin/bumpversion.cfg b/transforms/images/polus-stack-z-slice-plugin/.bumpversion.cfg similarity index 80% rename from transforms/images/polus-stack-z-slice-plugin/bumpversion.cfg rename to transforms/images/polus-stack-z-slice-plugin/.bumpversion.cfg index 42067ae27..82e520523 100644 --- a/transforms/images/polus-stack-z-slice-plugin/bumpversion.cfg +++ b/transforms/images/polus-stack-z-slice-plugin/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.4 +current_version = 1.2.6 commit = True tag = False diff --git a/transforms/images/polus-stack-z-slice-plugin/Dockerfile b/transforms/images/polus-stack-z-slice-plugin/Dockerfile index b5b077a2e..5fcdfb975 100644 --- a/transforms/images/polus-stack-z-slice-plugin/Dockerfile +++ b/transforms/images/polus-stack-z-slice-plugin/Dockerfile @@ -1,22 +1,14 @@ -# Get image containing bfio -FROM polusai/bfio:2.1.9 - -COPY VERSION / - -ARG EXEC_DIR="/opt/executables" -ARG DATA_DIR="/data" - -#Create folders -RUN mkdir -p ${EXEC_DIR} \ - && mkdir -p ${DATA_DIR}/inputs \ - && mkdir ${DATA_DIR}/outputs - -#Copy executable -COPY src ${EXEC_DIR}/ - -RUN pip3 install -r ${EXEC_DIR}/requirements.txt --no-cache-dir - +# Build from repo root (monorepo) or from this tool directory — both work. +FROM polusai/bfio:2.5.0 +ENV EXEC_DIR="/opt/executables" POLUS_IMG_EXT=".ome.tif" POLUS_TAB_EXT=".csv" POLUS_LOG="INFO" WORKDIR ${EXEC_DIR} - -# Default command. Additional arguments are provided through the command line -ENTRYPOINT ["python3", "/opt/executables/main.py"] \ No newline at end of file +ENV TOOL_DIR="transforms/images/polus-stack-z-slice-plugin" +RUN mkdir -p image-tools +COPY . ${EXEC_DIR}/image-tools +RUN pip3 install -U pip setuptools wheel \ + && python3 -c 'import sys; assert sys.version_info>=(3,11)' \ + && R="${EXEC_DIR}/image-tools" && M="$R/$TOOL_DIR" \ + && if [ -f "$M/pyproject.toml" ]; then pip3 install --no-cache-dir "$M"; \ + else pip3 install --no-cache-dir "$R"; fi +ENTRYPOINT ["python3", "-m", "main"] +CMD ["--help"] diff --git a/transforms/images/polus-stack-z-slice-plugin/README.md b/transforms/images/polus-stack-z-slice-plugin/README.md index 8cde5e459..2a6f36407 100644 --- a/transforms/images/polus-stack-z-slice-plugin/README.md +++ b/transforms/images/polus-stack-z-slice-plugin/README.md @@ -18,7 +18,7 @@ To build the Docker image for the conversion plugin, run ## Input Filename Pattern -This plugin uses the +This plugin uses the [filepattern](https://github.com/LabShare/polus-plugins/tree/master/utils/polus-filepattern-util) utility to indicate which files to stack. In particular, defining a filename variable is surrounded by `{}`, and the variable name and number of spaces diff --git a/transforms/images/polus-stack-z-slice-plugin/VERSION b/transforms/images/polus-stack-z-slice-plugin/VERSION index b966e81a4..3c43790f5 100644 --- a/transforms/images/polus-stack-z-slice-plugin/VERSION +++ b/transforms/images/polus-stack-z-slice-plugin/VERSION @@ -1 +1 @@ -1.2.4 \ No newline at end of file +1.2.6 diff --git a/transforms/images/polus-stack-z-slice-plugin/build-docker.sh b/transforms/images/polus-stack-z-slice-plugin/build-docker.sh index 4638bde0a..58f6abaa7 100755 --- a/transforms/images/polus-stack-z-slice-plugin/build-docker.sh +++ b/transforms/images/polus-stack-z-slice-plugin/build-docker.sh @@ -1,4 +1,4 @@ #!/bin/bash version=$(=2.5.0", + "filepattern>=2.2.1", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.package-dir] +"" = "src" + +[tool.setuptools] +py-modules = ["main"] diff --git a/transforms/images/polus-stack-z-slice-plugin/run-plugin.sh b/transforms/images/polus-stack-z-slice-plugin/run-plugin.sh index 2bb554f42..8cdd3cc47 100755 --- a/transforms/images/polus-stack-z-slice-plugin/run-plugin.sh +++ b/transforms/images/polus-stack-z-slice-plugin/run-plugin.sh @@ -15,4 +15,4 @@ docker run --mount type=bind,source=${datapath},target=/data/ \ polusai/stack-z-slice-plugin:${version} \ --filePattern ${filePattern} \ --inpDir ${inpDir} \ - --outDir ${outDir} \ No newline at end of file + --outDir ${outDir} diff --git a/transforms/images/polus-stack-z-slice-plugin/src/__init__.py b/transforms/images/polus-stack-z-slice-plugin/src/__init__.py new file mode 100644 index 000000000..aa1cf3054 --- /dev/null +++ b/transforms/images/polus-stack-z-slice-plugin/src/__init__.py @@ -0,0 +1 @@ +"""Polus stack Z-slice plugin package.""" diff --git a/transforms/images/polus-stack-z-slice-plugin/src/main.py b/transforms/images/polus-stack-z-slice-plugin/src/main.py index 6b18b575c..1e1f33c18 100644 --- a/transforms/images/polus-stack-z-slice-plugin/src/main.py +++ b/transforms/images/polus-stack-z-slice-plugin/src/main.py @@ -1,138 +1,175 @@ -import argparse, logging, math, filepattern, time, queue -from bfio import BioReader, BioWriter -import pathlib -from preadator import ProcessManager - -logging.basicConfig(format='%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s', - datefmt='%d-%b-%y %H:%M:%S') +"""CLI entrypoint: stack tiled TIFF slices into a single volumetric OME-TIFF.""" +from __future__ import annotations + +import argparse +import logging +import os +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path +from typing import Any + +import filepattern +from bfio import BioReader +from bfio import BioWriter + +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +logger = logging.getLogger("main") # length/width of the chunk each _merge_layers thread processes at once chunk_size = 8192 # Units for conversion -UNITS = {'m': 10**9, - 'cm': 10**7, - 'mm': 10**6, - 'µm': 10**3, - 'nm': 1, - 'Å': 10**-1} - -def _merge_layers(input_files,output_path): - - with ProcessManager.process(output_path.name): - - # Get the number of layers to stack - z_size = 0 - for f in input_files: - with BioReader(f['file']) as br: - z_size += br.z - - # Get some basic info about the files to stack - with BioReader(input_files[0]['file']) as br: - - # Get the physical z-distance if available, set to physical x if not - ps_z = br.ps_z - - # If the z-distances are undefined, average the x and y together - if None in ps_z: - # Get the size and units for x and y - x_val,x_units = br.ps_x - y_val,y_units = br.ps_y - - # Convert x and y values to the same units and average - z_val = (x_val*UNITS[x_units] + y_val*UNITS[y_units])/2 - - # Set z units to the smaller of the units between x and y - z_units = x_units if UNITS[x_units] < UNITS[y_units] else y_units - - # Convert z to the proper unit scale - z_val /= UNITS[z_units] - ps_z = (z_val,z_units) - ProcessManager.log('Could not find physical z-size. Using the average of x & y {}.'.format(ps_z)) - - # Hold a reference to the metadata once the file gets closed - metadata = br.metadata - - # Create the output file within a context manager - with BioWriter(output_path,metadata=metadata,max_workers=ProcessManager._active_threads) as bw: - - # Adjust the dimensions before writing - bw.z = z_size - bw.ps_z = ps_z - - # ZIndex tracking for the output file - zi = 0 - - # Start stacking - for file in input_files: - - # Open an image - with BioReader(file['file'],max_workers=ProcessManager._active_threads) as br: - - # Open z-layers one at a time - for z in range(br.z): - - # Use tiled reading in x&y to conserve memory - # At most, [chunk_size, chunk_size] pixels are loaded - for xs in range(0,br.x,chunk_size): - xe = min([br.x,xs + chunk_size]) - - for ys in range(0,br.y,chunk_size): - ye = min([br.y,ys + chunk_size]) - - bw[ys:ye,xs:xe,zi:zi+1,...] = br[ys:ye,xs:xe,z:z+1,...] - - zi += 1 - - # update the BioWriter in case the ProcessManager found more threads - bw.max_workers = ProcessManager._active_threads - -def main(input_dir: pathlib.Path, - file_pattern: str, - output_dir: pathlib.Path - ) -> None: - +UNITS = { + "m": 10**9, + "cm": 10**7, + "mm": 10**6, + "µm": 10**3, + "nm": 1, + "Å": 10**-1, +} + + +def _merge_layers(input_files: list[dict[str, Any]], output_path: Path) -> None: + """Stack input BioFormats files along Z and write a single OME-TIFF.""" + max_workers = max(1, (os.cpu_count() or 4) // 2) + + # Get the number of layers to stack + z_size = 0 + for f in input_files: + with BioReader(f["file"]) as br: + z_size += br.z + + # Get some basic info about the files to stack + with BioReader(input_files[0]["file"]) as br: + # Get the physical z-distance if available, set to physical x if not + ps_z = br.ps_z + + # If the z-distances are undefined, average the x and y together + if None in ps_z: + # Get the size and units for x and y + x_val, x_units = br.ps_x + y_val, y_units = br.ps_y + + # Convert x and y values to the same units and average + z_val = (x_val * UNITS[x_units] + y_val * UNITS[y_units]) / 2 + + # Set z units to the smaller of the units between x and y + z_units = x_units if UNITS[x_units] < UNITS[y_units] else y_units + + # Convert z to the proper unit scale + z_val /= UNITS[z_units] + ps_z = (z_val, z_units) + logger.info( + "Could not find physical z-size. Using the average of x & y {}.".format( + ps_z, + ), + ) + + # Hold a reference to the metadata once the file gets closed + metadata = br.metadata + + # Create the output file within a context manager + with BioWriter(output_path, metadata=metadata, max_workers=max_workers) as bw: + # Adjust the dimensions before writing + bw.z = z_size + bw.ps_z = ps_z + + # ZIndex tracking for the output file + zi = 0 + + # Start stacking + for file in input_files: + # Open an image + with BioReader(file["file"], max_workers=max_workers) as br: + # Open z-layers one at a time + for z in range(br.z): + # Use tiled reading in x&y to conserve memory + # At most, [chunk_size, chunk_size] pixels are loaded + for xs in range(0, br.x, chunk_size): + xe = min([br.x, xs + chunk_size]) + + for ys in range(0, br.y, chunk_size): + ye = min([br.y, ys + chunk_size]) + + bw[ys:ye, xs:xe, zi : zi + 1, ...] = br[ + ys:ye, + xs:xe, + z : z + 1, + ..., + ] + + zi += 1 + + # update the BioWriter max_workers + bw.max_workers = max_workers + + +def main(input_dir: Path, file_pattern: str, output_dir: Path) -> None: + """Group input files by Z via filepattern. + + Merge each group in a worker process. + """ # create the filepattern object - fp = filepattern.FilePattern(input_dir,file_pattern) - - for files in fp(group_by='z'): + fp = filepattern.FilePattern(input_dir, file_pattern) - output_name = fp.output_name(files) - output_file = output_dir.joinpath(output_name) + with ProcessPoolExecutor() as executor: + futures = [] + for files in fp(group_by="z"): + output_name = fp.output_name(files) + output_file = output_dir.joinpath(output_name) + futures.append(executor.submit(_merge_layers, files, output_file)) + for f in futures: + f.result() - ProcessManager.submit_process(_merge_layers,files,output_file) - - ProcessManager.join_processes() if __name__ == "__main__": # Initialize the main thread logger - logger = logging.getLogger('main') + logger = logging.getLogger("main") logger.setLevel(logging.INFO) # Setup the Argument parsing - logger.info('Parsing arguments...') - parser = argparse.ArgumentParser(prog='main', description='Compile individual tiled tiff images into a single volumetric tiled tiff.') - - parser.add_argument('--inpDir', dest='input_dir', type=str, - help='Path to folder with tiled tiff files', required=True) - parser.add_argument('--outDir', dest='output_dir', type=str, - help='The output directory for ome.tif files', required=True) - parser.add_argument('--filePattern', dest='file_pattern', type=str, - help='A filename pattern specifying variables in filenames.', required=True) + logger.info("Parsing arguments...") + parser = argparse.ArgumentParser( + prog="main", + description=( + "Compile individual tiled TIFF images into a single volumetric " + "tiled TIFF." + ), + ) + + parser.add_argument( + "--inpDir", + dest="input_dir", + type=str, + help="Path to folder with tiled tiff files", + required=True, + ) + parser.add_argument( + "--outDir", + dest="output_dir", + type=str, + help="The output directory for ome.tif files", + required=True, + ) + parser.add_argument( + "--filePattern", + dest="file_pattern", + type=str, + help="A filename pattern specifying variables in filenames.", + required=True, + ) args = parser.parse_args() - input_dir = pathlib.Path(args.input_dir) + input_dir = Path(args.input_dir) if input_dir.joinpath("images").is_dir(): input_dir = input_dir.joinpath("images") - output_dir = pathlib.Path(args.output_dir) + output_dir = Path(args.output_dir) file_pattern = args.file_pattern - logger.info(f'input_dir = {input_dir}') - logger.info(f'output_dir = {output_dir}') - logger.info(f'file_pattern = {file_pattern}') - logger.info(f'max_threads: {ProcessManager.num_processes()}') - - ProcessManager.init_processes('main','stack') - - main(input_dir, - file_pattern, - output_dir) \ No newline at end of file + logger.info(f"input_dir = {input_dir}") + logger.info(f"output_dir = {output_dir}") + logger.info(f"file_pattern = {file_pattern}") + + main(input_dir, file_pattern, output_dir) diff --git a/transforms/images/polus-stack-z-slice-plugin/src/requirements.txt b/transforms/images/polus-stack-z-slice-plugin/src/requirements.txt index 2a7cb144c..634c4b837 100644 --- a/transforms/images/polus-stack-z-slice-plugin/src/requirements.txt +++ b/transforms/images/polus-stack-z-slice-plugin/src/requirements.txt @@ -1,2 +1 @@ -filepattern==1.4.7 -preadator==0.2.0 +filepattern>=2.2.1