Hep2-Segmentation/pair_and_split_data.py at main · prasathlab/Hep2-Segmentation · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import yaml
import glob
from file_utils import *
import psutil
import pdb

pdb.set_trace()
yaml_file = r"/data/aronow/Balaji_Iyer/Projects/Hep-2_Segmentation/src/configs/exp1_cluster_res_unet.yaml"
with open(yaml_file, 'r') as stream:
    try:
        cfg = yaml.load(stream)
        print(cfg)
    except yaml.YAMLError as exc:
        print(exc)

# ---------------------------------------------------------------------------------------------
pid = os.getpid()
py = psutil.Process(pid)
memoryUse = py.memory_info()[0]/2.**30  # memory use in GB...I think
print('memory use:', memoryUse)
pdb.set_trace()

#Load the train images.
all_files = glob.glob(cfg["pair_and_split_data"]["data_dir"] + "/*.tif")
pdb.set_trace()
paired_df = pair_image_mask_names(all_files)
paired_df.to_csv("all_paired_names.tsv", sep="\t")

pdb.set_trace()
test_ratio = cfg["pair_and_split_data"]["test_ratio"]
n_test_samples = round(paired_df.shape[0]*test_ratio)
test_df = paired_df.sample(n=n_test_samples, replace=False)
train_val_df = paired_df[~paired_df.isin(test_df)].dropna()
pdb.set_trace()

val_ratio = cfg["pair_and_split_data"]["val_ratio"]
n_val_samples = round(paired_df.shape[0]*val_ratio)
val_df = train_val_df.sample(n=n_val_samples, replace=False)
train_df = train_val_df[~train_val_df.isin(val_df)].dropna()

test_df.to_csv("test_data.tsv", sep="\t")
val_df.to_csv("val_data.tsv", sep="\t")
train_df.to_csv("train_data.tsv", sep="\t")
pdb.set_trace()


memoryUse = py.memory_info()[0]/2.**30  # memory use in GB...I think
print('memory use:', memoryUse)
pdb.set_trace()

debug = 1