Skip to content

Segment_WSI migration from TF1 to TF2 #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ishaq-ansari wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Segment_WSI migration from TF1 to TF2 #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a0c12ea
temp Working implementation for issue #12
Dec 20, 2025
3eb0a14
updated sbatch and added checkpoint vars
Dec 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 14 additions & 41 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
# All plugins of HistomicsTK should derive from this docker image


# start from nvidia/cuda 10.0
# FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
FROM tensorflow/tensorflow:1.15.4-gpu-py3
# start from TensorFlow 2.x with GPU support
FROM tensorflow/tensorflow:2.15.0-gpu
LABEL com.nvidia.volumes.needed="nvidia_driver"

LABEL maintainer="Brendon Lutnick - Sarder Lab. <brendonl@buffalo.edu>"
LABEL description="HistomicsTK with TensorFlow 2.x and DeepLabV3+ support"

CMD echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! STARTING THE BUILD !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# RUN mkdir /usr/local/nvidia && ln -s /usr/local/cuda-10.0/compat /usr/local/nvidia/lib
Expand All @@ -24,9 +24,7 @@ RUN rm \

RUN apt-get update && \
apt-get install --yes --no-install-recommends software-properties-common && \
# As of 2018-04-16 this repo has the latest release of Python 2.7 (2.7.14) \
# add-apt-repository ppa:jonathonf/python-2.7 && \
add-apt-repository ppa:deadsnakes/ppa && \
RUN add-apt-repository ppa:deadsnakes/ppa && \
apt-get autoremove && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN apt-get update && \
Expand All @@ -35,22 +33,12 @@ RUN apt-get update && \
#keyboard-configuration \
git \
wget \
python-qt4 \
python3-pyqt4 \
curl \
ca-certificates \
libcurl4-openssl-dev \
libexpat1-dev \
unzip \
libhdf5-dev \
libpython-dev \
libpython3-dev \
python2.7-dev \
python-tk \
# We can't go higher than 3.7 and use tensorflow 1.x \
python3.6-dev \
python3.6-distutils \
python3-tk \
software-properties-common \
libssl-dev \
# Standard build tools \
Expand All @@ -60,8 +48,6 @@ RUN apt-get update && \
automake \
libtool \
pkg-config \
# needed for supporting CUDA \
# libcupti-dev \
# useful later \
libmemcached-dev && \
#apt-get autoremove && \
Expand All @@ -81,19 +67,9 @@ RUN apt-get install 'ffmpeg'\
# RUN apt-get install nvidia-driver-455 -y

WORKDIR /
# Make Python3 the default and install pip. Whichever is done last determines
# the default python version for pip.
RUN rm /usr/bin/python && \
ln /usr/bin/python3.6 /usr/bin/python && \
rm /usr/local/bin/python && \
ln /usr/bin/python3.6 /usr/local/bin/python && \
ln /usr/bin/python3.6 /usr/local/bin/python3
RUN which python && \
# Make Python3 the default and install pip. TF 2.15 comes with Python 3.11
RUN which python && \
python --version
# RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
RUN curl -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \
python get-pip.py && \
rm get-pip.py

ENV build_path=$PWD/build

Expand All @@ -113,18 +89,16 @@ COPY . $htk_path/
WORKDIR $htk_path

# Install HistomicsTK and its dependencies
# Upgrade setuptools, as the version in Conda won't upgrade cleanly unless it
# is ignored.
RUN pip install --no-cache-dir --upgrade --ignore-installed pip setuptools && \
# pip install --no-cache-dir 'tensorflow<2' && \
# Install large_image memcached extras \
# Upgrade setuptools and pip
RUN pip install --no-cache-dir --upgrade pip setuptools && \
# Install large_image with memcached extras \
pip install --no-cache-dir 'large-image[memcached]' && \
# Install HistomicsTK \
pip install --no-cache-dir . --find-links https://girder.github.io/large_image_wheels && \
# Install tf-slim \
# Install TF-Slim for TF2 compatibility \
pip install --no-cache-dir 'tf-slim>=1.1.0' && \
# Install pillow_lut \
pip install --no-cache-dir 'pillow-lut' && \
# Install HistomicsTK \
pip install --no-cache-dir . --find-links https://girder.github.io/large_image_wheels && \
# clean up \
rm -rf /root/.cache/pip/*

Expand All @@ -137,9 +111,8 @@ RUN python --version && pip --version && pip freeze
# pregenerate font cache
RUN python -c "from matplotlib import pylab"

# Suppress warnings
RUN sed -i 's/^_PRINT_DEPRECATION_WARNINGS = True/_PRINT_DEPRECATION_WARNINGS = False/g' /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/util/deprecation.py && \
sed -i 's/rename = get_rename_v2(full_name)/rename = False/g' /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/util/module_wrapper.py
# Note: TF2 deprecation warnings are handled via tf.compat.v1 API usage in code
# No need to suppress warnings in the TF installation

# define entrypoint through which all CLIs can be run
WORKDIR $htk_path/histomicstk/cli
Expand Down
61 changes: 61 additions & 0 deletions build_docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/bin/bash
# Build Docker image for HistomicsTK with TensorFlow 2.x

set -e

# Configuration
IMAGE_NAME="histomicstk-tf2"
IMAGE_TAG="latest"
DOCKERFILE="Dockerfile"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

echo -e "${GREEN}======================================${NC}"
echo -e "${GREEN}Building HistomicsTK Docker Image${NC}"
echo -e "${GREEN}======================================${NC}"
echo ""
echo -e "Image: ${YELLOW}${IMAGE_NAME}:${IMAGE_TAG}${NC}"
echo -e "Dockerfile: ${YELLOW}${DOCKERFILE}${NC}"
echo ""

# Check if Dockerfile exists
if [ ! -f "$DOCKERFILE" ]; then
echo -e "${RED}Error: Dockerfile not found!${NC}"
exit 1
fi

# Build the Docker image
echo -e "${GREEN}Starting Docker build...${NC}"
echo ""

docker build \
-t "${IMAGE_NAME}:${IMAGE_TAG}" \
-f "${DOCKERFILE}" \
. 2>&1 | tee docker_build.log

# Check if build was successful
if [ ${PIPESTATUS[0]} -eq 0 ]; then
echo ""
echo -e "${GREEN}======================================${NC}"
echo -e "${GREEN}Docker build completed successfully!${NC}"
echo -e "${GREEN}======================================${NC}"
echo ""
echo -e "Image: ${YELLOW}${IMAGE_NAME}:${IMAGE_TAG}${NC}"
echo ""
echo "You can run the container with:"
echo -e "${YELLOW}docker run --gpus all -it ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash${NC}"
echo ""
echo "Build log saved to: docker_build.log"
else
echo ""
echo -e "${RED}======================================${NC}"
echo -e "${RED}Docker build failed!${NC}"
echo -e "${RED}======================================${NC}"
echo ""
echo "Check docker_build.log for details"
exit 1
fi
1 change: 1 addition & 0 deletions docker_build.log
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
./build_docker.sh: line 35: docker: command not found
68 changes: 68 additions & 0 deletions environment.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: test-tf2-env
channels:
- conda-forge
- nvidia
- pytorch
- anaconda
- bioconda
- defaults
dependencies:
- python=3.10
- pip
- ca-certificates
- openssl
# Core scientific computing
- numpy>=1.12.1
- scipy>=0.19.0
- matplotlib
- scikit-image>=0.14.2
- scikit-learn>=0.18.1
- pandas>=0.19.2
# Image processing libraries
- pillow>=3.2.0
- imageio>=2.3.0
- imagecodecs
- tifffile
- openslide
- openslide-python
# GDAL and geospatial libraries (install via conda to avoid build issues)
- gdal
- libgdal
- geos
- proj
- shapely
# GUI libraries (if needed for visualization)
- pyside6
- qt6-main
# System libraries
- libpng
- libjpeg-turbo
- libtiff
- libxml2
- libxslt
- pip:
# TensorFlow 2.x requirements
- tensorflow>=2.15.0,<3.0.0
- tensorflow-addons>=0.22.0
- tf-slim>=1.1.0
# Core scientific packages
- nimfa>=1.3.2
- "opencv-python-headless<4.7"
- sqlalchemy
- pyvips
- termcolor
- openpyxl
- "xlrd<2"
# dask packages
- "dask[dataframe]>=1.1.0"
- distributed>=1.21.6
# large image sources (without GDAL source to avoid conflicts)
- large-image
- large-image-source-tiff
- large-image-source-pil
- girder-slicer-cli-web
- girder-client
# cli
- ctk-cli
# Six for compatibility
- six
74 changes: 72 additions & 2 deletions histomicstk/cli/SegmentWSI/SegmentWSI.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import os, zipfile, json
import sys
from histomicstk.cli.utils import CLIArgumentParser
from glob import glob
import sys
import girder_client

def main(args):

Expand Down Expand Up @@ -61,7 +62,76 @@ def get_base_model_name(model_file):
cmd = "python3 ../deeplab/vis.py --model_variant xception_65 --atrous_rates 6 --atrous_rates 12 --atrous_rates 18 --output_stride 16 --decoder_output_stride 4 --save_json_annotation True --checkpoint_dir {} --dataset_dir '{}' --json_filename '{}' --vis_crop_size {} --wsi_downsample {} --tile_step {} --min_size {} --vis_batch_size {} --vis_remove_border {} --simplify_contours {} --num_classes {} --class_names '{}' --save_heatmap={} --heatmap_stride {} --gpu {}".format(model, args.inputImageFile, args.outputAnnotationFile, args.patch_size, args.wsi_downsample, args.tile_stride, args.min_size, args.batch_size, args.remove_border, args.simplify_contours, num_classes, compartments, args.save_heatmap, args.heatmap_stride, args.gpu)
print(cmd)
sys.stdout.flush()
os.system(cmd)
exit_code = os.system(cmd)

if exit_code != 0:
print("ERROR: vis.py execution failed with exit code {}".format(exit_code))
sys.exit(1)

# Upload to Girder if credentials are provided
if hasattr(args, 'girderApiUrl') and args.girderApiUrl and \
hasattr(args, 'girderToken') and args.girderToken:

print("\n=== Uploading annotations to Girder ===")

try:
# Connect to Girder
gc = girder_client.GirderClient(apiUrl=args.girderApiUrl)
gc.setToken(args.girderToken)
print("Connected to Girder at: {}".format(args.girderApiUrl))

# Get the item ID (either from args or resolve from input image)
if hasattr(args, 'girderItemId') and args.girderItemId:
item_id = args.girderItemId
else:
# If no item ID provided, try to get it from the input image reference
# This assumes the input image is a Girder resource ID
try:
resource = gc.get('resource/{}'.format(args.inputImageFile))
item_id = resource.get('_id')
except:
print("WARNING: Could not resolve item ID from input image. Skipping Girder upload.")
return

print("Uploading to item: {}".format(item_id))

# Load the annotation JSON
output_json_path = args.outputAnnotationFile
if not output_json_path.endswith('.json'):
output_json_path = output_json_path + '.json'

if not os.path.exists(output_json_path):
print("ERROR: Output annotation file not found: {}".format(output_json_path))
return

# Post annotation to Girder
with open(output_json_path, 'r') as f:
payload = json.load(f)
result = gc.post(
path="annotation",
parameters={"itemId": item_id},
data=json.dumps(payload),
headers={"Content-Type": "application/json"}
)
print("Annotation posted successfully. ID: {}".format(result.get('_id', 'unknown')))

# Also upload the JSON file to the item
output_basename = os.path.basename(output_json_path)
uploaded = gc.uploadFileToItem(
item_id,
output_json_path,
filename=output_basename,
mimeType="application/json"
)
print("JSON file uploaded successfully: {}".format(uploaded.get('name', output_basename)))
print("=== Girder upload complete ===\n")

except Exception as e:
print("ERROR uploading to Girder: {}".format(str(e)))
import traceback
traceback.print_exc()
else:
print("\nNo Girder credentials provided. Annotations saved locally only.")


if __name__ == "__main__":
Expand Down
25 changes: 25 additions & 0 deletions histomicstk/cli/SegmentWSI/SegmentWSI.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,4 +111,29 @@
<default>0</default>
</integer>
</parameters>
<parameters advanced="true">
<label>Girder Upload</label>
<description>Optional parameters for uploading annotations to Girder/DSA</description>
<string>
<name>girderApiUrl</name>
<label>Girder API URL</label>
<description>The base URL of the Girder API (e.g., http://localhost:8080/api/v1). If provided with girderToken, annotations will be uploaded to Girder.</description>
<longflag>girderApiUrl</longflag>
<default></default>
</string>
<string>
<name>girderToken</name>
<label>Girder API Token</label>
<description>Authentication token for Girder API access. Required if girderApiUrl is provided.</description>
<longflag>girderToken</longflag>
<default></default>
</string>
<string>
<name>girderItemId</name>
<label>Girder Item ID</label>
<description>The Girder item ID to upload annotations to. If not provided, will attempt to resolve from input image.</description>
<longflag>girderItemId</longflag>
<default></default>
</string>
</parameters>
</executable>
Loading