From 442da76b57c0c3227ced42cfd38ca2dd0e047fcf Mon Sep 17 00:00:00 2001 From: Sayat Mimar Date: Tue, 14 Nov 2023 13:16:24 -0500 Subject: [PATCH 1/9] adding slurm script --- histomicstk/deeplab/slurm_cript.sh | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 histomicstk/deeplab/slurm_cript.sh diff --git a/histomicstk/deeplab/slurm_cript.sh b/histomicstk/deeplab/slurm_cript.sh new file mode 100644 index 00000000..3698ff48 --- /dev/null +++ b/histomicstk/deeplab/slurm_cript.sh @@ -0,0 +1,36 @@ +#!/bin/sh +#SBATCH --account=pinaki.sarder +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem-per-cpu=7000mb +#SBATCH --partition=gpu +#SBATCH --gpus=geforce +#SBATCH --time=72:00:00 +#SBATCH --output=./slurm_log.out +#SBATCH --job-name="segmentation_frozen" +echo "SLURM_JOBID="$SLURM_JOBID +echo "SLURM_JOB_NODELIST="$SLURM_JOB_NODELIST +echo "SLURM_NNODES="$SLURM_NNODES +echo "SLURMTMPDIR="$SLURMTMPDIR + +echo "working directory = "$SLURM_SUBMIT_DIR +ulimit -s unlimited +module load singularity +ls +ml + +# Add your userid here: +USER=sayat.mimar +# Add the name of the folder containing WSIs here +PROJECT=segmentation + +CODESDIR=/blue/pinaki.sarder/sayat.mimar/segmentation_test/Histo-cloud/histomicstk/deeplab + +DATADIR=/$CODESDIR/training_data +MODELDIR=$CODESDIR/trained_model + +CONTAINER=$CODESDIR/myhistorepo_histo_img.sif + +singularity exec --nv -B $(pwd):/exec/,$DATADIR/:/data,$MODELDIR/:/model/ $CONTAINER python3 /exec/train.py --model_variant xception_65 --atrous_rates 6 --atrous_rates 12 --atrous_rates 18 --output_stride 16 --decoder_output_stride 4 --train_crop_size 400 --train_batch_size 2 --training_number_of_steps 5000 --slow_start_step 1000 --augment_prob 0.01 --slow_start_learning_rate 1e-05 --base_learning_rate 0.0005 --tf_initial_checkpoint /model/model.ckpt-400000 --dataset_dir /data/ --train_logdir $CODESDIR/log_dir --save_interval_secs 600 --num_clones 1 --global_step 0 --end_learning_rate 0.0 --learning_power 0.9 --ignore_label 2 --decay_steps 0 --last_layer_gradient_multiplier 10.0 --wsi_downsample 1 --wsi_downsample 2 --wsi_downsample 3 --wsi_downsample 4 --initialize_last_layer=false --fine_tune_batch_norm=false --last_layers_contain_logits_only=false --upsample_logits=true + From a6756963f64969cd70d9afa9efe5167eda12f1d8 Mon Sep 17 00:00:00 2001 From: Sayat Date: Mon, 17 Apr 2023 16:33:41 -0400 Subject: [PATCH 2/9] add czi extenstion in the list --- histomicstk/deeplab/datasets/wsi_data_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/histomicstk/deeplab/datasets/wsi_data_generator.py b/histomicstk/deeplab/datasets/wsi_data_generator.py index e640a2ec..af37644a 100644 --- a/histomicstk/deeplab/datasets/wsi_data_generator.py +++ b/histomicstk/deeplab/datasets/wsi_data_generator.py @@ -60,7 +60,7 @@ def __init__(self, include_background_prob = 0, augment_prob = 0, num_of_classes = None, - wsi_ext=['.svs', '.ndpi', '.scn'], + wsi_ext=['.svs', '.ndpi', '.scn', '.czi'], min_resize_value=None, max_resize_value=None, resize_factor=None, From 9965b7c476b9968094c0b5d4e1664d996aae7506 Mon Sep 17 00:00:00 2001 From: sayatmimar Date: Tue, 18 Apr 2023 14:48:02 -0400 Subject: [PATCH 3/9] add excel dependencies --- .../ExtractFeaturesFromAnnotations.py | 2 +- setup.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/histomicstk/cli/ExtractFeaturesFromAnnotations/ExtractFeaturesFromAnnotations.py b/histomicstk/cli/ExtractFeaturesFromAnnotations/ExtractFeaturesFromAnnotations.py index f6959a81..51b2c975 100644 --- a/histomicstk/cli/ExtractFeaturesFromAnnotations/ExtractFeaturesFromAnnotations.py +++ b/histomicstk/cli/ExtractFeaturesFromAnnotations/ExtractFeaturesFromAnnotations.py @@ -57,7 +57,7 @@ def main(args): slidename = file['name'] _ = os.system("printf '\n---\n\nFOUND: [{}]\n'".format(slidename)) - df = pd.read_excel(args.slide_labels) + df = pd.read_excel(args.slide_labels, engine = 'openpyxl') slide_label = df.loc[df.iloc[:,0] == slidename] if not slide_label.empty: label = slide_label.iloc[0,1:].to_dict() diff --git a/setup.py b/setup.py index 004e014a..53f8c5de 100644 --- a/setup.py +++ b/setup.py @@ -68,6 +68,8 @@ def prerelease_local_scheme(version): 'matplotlib', 'pyvips', 'termcolor', + 'openpyxl', + 'xlrd<2', # dask packages 'dask[dataframe]>=1.1.0', 'distributed>=1.21.6', From 62cb99ea5dc6d6c3e393b42e33855e1ace7d3810 Mon Sep 17 00:00:00 2001 From: sayatmimar Date: Tue, 2 May 2023 13:59:41 -0400 Subject: [PATCH 4/9] first commit --- README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..e69de29b From 6c1b18ddd1377903636713104ea851c298270dca Mon Sep 17 00:00:00 2001 From: sayatmimar Date: Tue, 2 May 2023 14:12:48 -0400 Subject: [PATCH 5/9] rm readme --- README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 README.md diff --git a/README.md b/README.md deleted file mode 100644 index e69de29b..00000000 From f6bc398fb35578d7a0428f1c60b54e38b9ec795f Mon Sep 17 00:00:00 2001 From: sayatmimar Date: Tue, 17 Oct 2023 16:59:50 -0400 Subject: [PATCH 6/9] add ome.tiff and image read logic --- .../deeplab/datasets/wsi_data_generator.py | 2 +- .../utils/wsi_dataset_util_large_image.py | 26 +++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/histomicstk/deeplab/datasets/wsi_data_generator.py b/histomicstk/deeplab/datasets/wsi_data_generator.py index af37644a..03f9e176 100644 --- a/histomicstk/deeplab/datasets/wsi_data_generator.py +++ b/histomicstk/deeplab/datasets/wsi_data_generator.py @@ -60,7 +60,7 @@ def __init__(self, include_background_prob = 0, augment_prob = 0, num_of_classes = None, - wsi_ext=['.svs', '.ndpi', '.scn', '.czi'], + wsi_ext=['.svs', '.ndpi', '.scn', '.czi','.ome.tif'], min_resize_value=None, max_resize_value=None, resize_factor=None, diff --git a/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py b/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py index 5ea83a9f..dc7f3913 100644 --- a/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py +++ b/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py @@ -20,6 +20,22 @@ ################################ main function ################################# ################################################################################ +def get_image(filename): + """ + Read the inut image and perform channel decomposition if necessary. + """ + + slide = large_image.open(filename) + + if slide.frames == 3 and slide.bandCount == 1: + slide = large_image.open(filename, style={'bands': [{'framedelta': 0, 'palette': '#f00'},{'framedelta': 1, 'palette': '#0f0'},{'framedelta': 2, 'palette': '#00f'}]}) + elif slide.frames == 1 and slide.bandCount == 3: + pass + else: + raise Exception('Cannot do channel decomposition with [{}] frame and [{}] band image.'.format(slide.frames, slide.bandCount)) + + return slide + def get_wsi_patch(filename, patch_size=256, downsample=[1], include_background_prob=0.1, augment=0, ignore_label=255): ''' takes a wsi and returns a random patch of patch_size @@ -47,7 +63,7 @@ def get_wsi_patch(filename, patch_size=256, downsample=[1], include_background_p except: base_name = filename.split('.')[0] - wsi = large_image.getTileSource(filename) + wsi = get_image(filename) l_dims = get_slide_size(wsi=wsi) xml_path = '{}.xml'.format(base_name) @@ -91,7 +107,7 @@ def get_patch_from_points(filename, point, patch_size, downsample=1, wsi=None, c base_name = filename.split('.')[0] if wsi==None: - wsi = large_image.getTileSource(filename) + wsi = get_image(filename) # t = time.time() # print('t0: {}'.format(time.time()-t)) @@ -255,7 +271,7 @@ def get_grid_list(slide_path, patch_size, downsample, tile_step, wsi=None): slide_mask = get_slide_mask(slide_path, save_mask=False) # open slide once globally for efficency if wsi == None: - wsi = large_image.getTileSource(slide_path) + wsi = get_image(slide_path) level_dims = get_slide_size(wsi=wsi) thumbnail_size = float(max(slide_mask.shape)) @@ -341,7 +357,7 @@ def save_wsi_thumbnail_mask(filename, save_mask=True, thumbnail_size=2000): try: filename = filename.numpy() except: filename = filename - wsi = large_image.getTileSource(filename) + wsi = get_image(filename) # def find_tissue_mask(): # thumbnail, _ = wsi.getThumbnail(width=thumbnail_size, height=thumbnail_size, format=large_image.tilesource.TILE_FORMAT_PIL) @@ -446,7 +462,7 @@ def get_slide_label(filename, data_label_xlsx): def get_slide_size(filename=None, wsi=None): if filename: - wsi = large_image.getTileSource(filename) + wsi = get_image(filename) width = wsi.sizeX height = wsi.sizeY From 81599b140a6b01f589c99a16df94b36f4b9fbba9 Mon Sep 17 00:00:00 2001 From: sayatmimar Date: Tue, 17 Oct 2023 17:12:14 -0400 Subject: [PATCH 7/9] allow bandcount of 3 or 4 --- histomicstk/deeplab/utils/wsi_dataset_util_large_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py b/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py index dc7f3913..713c715a 100644 --- a/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py +++ b/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py @@ -26,10 +26,10 @@ def get_image(filename): """ slide = large_image.open(filename) - + if slide.frames == 3 and slide.bandCount == 1: slide = large_image.open(filename, style={'bands': [{'framedelta': 0, 'palette': '#f00'},{'framedelta': 1, 'palette': '#0f0'},{'framedelta': 2, 'palette': '#00f'}]}) - elif slide.frames == 1 and slide.bandCount == 3: + elif slide.frames == 1 and slide.bandCount in [3, 4]: pass else: raise Exception('Cannot do channel decomposition with [{}] frame and [{}] band image.'.format(slide.frames, slide.bandCount)) From 1de0015545a1e5043bd4452e4c11407375a0495a Mon Sep 17 00:00:00 2001 From: sayatmimar Date: Mon, 6 Nov 2023 12:41:46 -0500 Subject: [PATCH 8/9] add f-string --- histomicstk/deeplab/utils/wsi_dataset_util_large_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py b/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py index 713c715a..b96b35a6 100644 --- a/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py +++ b/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py @@ -32,7 +32,7 @@ def get_image(filename): elif slide.frames == 1 and slide.bandCount in [3, 4]: pass else: - raise Exception('Cannot do channel decomposition with [{}] frame and [{}] band image.'.format(slide.frames, slide.bandCount)) + raise Exception(f"Cannot do channel decomposition with [{}] frame and [{}] band image.".format(slide.frames, slide.bandCount)) return slide From 6673c77dafcdb1290ecbafb2196ba1c83a4cc9ba Mon Sep 17 00:00:00 2001 From: sayatmimar Date: Wed, 8 Nov 2023 15:22:20 -0500 Subject: [PATCH 9/9] fix f_string --- histomicstk/deeplab/utils/wsi_dataset_util_large_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py b/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py index b96b35a6..650653b0 100644 --- a/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py +++ b/histomicstk/deeplab/utils/wsi_dataset_util_large_image.py @@ -32,7 +32,7 @@ def get_image(filename): elif slide.frames == 1 and slide.bandCount in [3, 4]: pass else: - raise Exception(f"Cannot do channel decomposition with [{}] frame and [{}] band image.".format(slide.frames, slide.bandCount)) + raise Exception(f"Cannot do channel decomposition with {[{}]} frame and {[{}]} band image.".format(slide.frames, slide.bandCount)) return slide