From 3de94b866aa818d663fcfdb37dea278e28dbb9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 16 Aug 2021 18:35:21 -0500 Subject: [PATCH 01/11] Create mlcube template files --- chest_xray/mlcube.yaml | 13 +++++++++++++ chest_xray/platforms/docker.yaml | 8 ++++++++ chest_xray/run/download_ckpt.yaml | 9 +++++++++ chest_xray/run/download_data.yaml | 10 ++++++++++ chest_xray/run/infer.yml | 9 +++++++++ chest_xray/tasks/download_ckpt.yaml | 11 +++++++++++ chest_xray/tasks/download_data.yaml | 14 ++++++++++++++ chest_xray/tasks/infer.yaml | 12 ++++++++++++ 8 files changed, 86 insertions(+) create mode 100644 chest_xray/mlcube.yaml create mode 100644 chest_xray/platforms/docker.yaml create mode 100644 chest_xray/run/download_ckpt.yaml create mode 100644 chest_xray/run/download_data.yaml create mode 100644 chest_xray/run/infer.yml create mode 100644 chest_xray/tasks/download_ckpt.yaml create mode 100644 chest_xray/tasks/download_data.yaml create mode 100644 chest_xray/tasks/infer.yaml diff --git a/chest_xray/mlcube.yaml b/chest_xray/mlcube.yaml new file mode 100644 index 0000000..6250940 --- /dev/null +++ b/chest_xray/mlcube.yaml @@ -0,0 +1,13 @@ +schema_version: 1.0.0 +schema_type: mlcube_root + +name: chest_xray +author: MLPerf Best Practices Working Group +version: 0.1.0 +mlcube_spec_version: 0.1.0 + +tasks: + - 'tasks/download_data.yaml' + - 'tasks/download_ckpt.yaml' + - 'tasks/infer.yaml' + diff --git a/chest_xray/platforms/docker.yaml b/chest_xray/platforms/docker.yaml new file mode 100644 index 0000000..cef5ea6 --- /dev/null +++ b/chest_xray/platforms/docker.yaml @@ -0,0 +1,8 @@ +schema_type: mlcube_platform +schema_version: 0.1.0 + +platform: + name: docker + version: ">=18.01" +container: + image: "mlperf/mlbox_chest_xray:0.0.1" \ No newline at end of file diff --git a/chest_xray/run/download_ckpt.yaml b/chest_xray/run/download_ckpt.yaml new file mode 100644 index 0000000..e393561 --- /dev/null +++ b/chest_xray/run/download_ckpt.yaml @@ -0,0 +1,9 @@ +schema_type: mlcube_invoke +schema_version: 1.0.0 + +task_name: download_ckpt + +input_binding: {} + +output_binding: + ckpt: $WORKSPACE/data \ No newline at end of file diff --git a/chest_xray/run/download_data.yaml b/chest_xray/run/download_data.yaml new file mode 100644 index 0000000..cfd1a4c --- /dev/null +++ b/chest_xray/run/download_data.yaml @@ -0,0 +1,10 @@ +schema_type: mlcube_invoke +schema_version: 1.0.0 + +task_name: download_data + +input_binding: {} + +output_binding: + data_dir: $WORKSPACE/data + log_dir: $WORKSPACE/download_data_logs \ No newline at end of file diff --git a/chest_xray/run/infer.yml b/chest_xray/run/infer.yml new file mode 100644 index 0000000..54c806c --- /dev/null +++ b/chest_xray/run/infer.yml @@ -0,0 +1,9 @@ +schema_type: mlcube_invoke +schema_version: 1.0.0 + +task_name: infer + +input_binding: {} + +output_binding: + output_file: $WORKSPACE/inferences.txt diff --git a/chest_xray/tasks/download_ckpt.yaml b/chest_xray/tasks/download_ckpt.yaml new file mode 100644 index 0000000..8df2a7e --- /dev/null +++ b/chest_xray/tasks/download_ckpt.yaml @@ -0,0 +1,11 @@ +# Schema +schema_version: 1.0.0 +schema_type: mlcube_task + +# Task Inputs +inputs: [] + +# Task Outputs +outputs: + - name: ckpt + type: file \ No newline at end of file diff --git a/chest_xray/tasks/download_data.yaml b/chest_xray/tasks/download_data.yaml new file mode 100644 index 0000000..6ed5ad1 --- /dev/null +++ b/chest_xray/tasks/download_data.yaml @@ -0,0 +1,14 @@ +# Schema +schema_version: 1.0.0 +schema_type: mlcube_task + +# Task Inputs +inputs: [] + +# Task Outputs +outputs: + - name: data_dir + type: directory + + - name: log_dir + type: directory \ No newline at end of file diff --git a/chest_xray/tasks/infer.yaml b/chest_xray/tasks/infer.yaml new file mode 100644 index 0000000..bd02ce4 --- /dev/null +++ b/chest_xray/tasks/infer.yaml @@ -0,0 +1,12 @@ +# Schema +schema_version: 1.0.0 +schema_type: mlcube_task + +# Task Inputs +# TODO: check if this is correct or we need an input +inputs: [] + +# Task Outputs +outputs: + - name: output_file + type: file \ No newline at end of file From 0e6e3d67147977258098ea7d2d6ee93bd497e967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 18 Aug 2021 12:51:17 -0500 Subject: [PATCH 02/11] Reformat to config v2.0 --- chest_xray/mlcube.yaml | 13 - chest_xray/platforms/docker.yaml | 8 - chest_xray/run/download_ckpt.yaml | 9 - chest_xray/run/download_data.yaml | 10 - chest_xray/run/infer.yml | 9 - chest_xray/tasks/download_ckpt.yaml | 11 - chest_xray/tasks/download_data.yaml | 14 - chest_xray/tasks/infer.yaml | 12 - chexpert/README.md | 43 ++ chexpert/mlcube/mlcube.yaml | 30 ++ chexpert/project/Dockerfile | 31 ++ chexpert/project/chexpert.py | 183 +++++++++ chexpert/project/data/dataset.py | 78 ++++ chexpert/project/data/imgaug.py | 39 ++ chexpert/project/data/utils.py | 68 ++++ chexpert/project/download_model.sh | 17 + chexpert/project/mlcube.py | 71 ++++ chexpert/project/model/attention_map.py | 186 +++++++++ chexpert/project/model/backbone/__init__.py | 0 chexpert/project/model/backbone/densenet.py | 235 +++++++++++ chexpert/project/model/backbone/inception.py | 394 +++++++++++++++++++ chexpert/project/model/backbone/vgg.py | 215 ++++++++++ chexpert/project/model/classifier.py | 163 ++++++++ chexpert/project/model/global_pool.py | 154 ++++++++ chexpert/project/model/utils.py | 36 ++ chexpert/project/requirements.txt | 11 + 26 files changed, 1954 insertions(+), 86 deletions(-) delete mode 100644 chest_xray/mlcube.yaml delete mode 100644 chest_xray/platforms/docker.yaml delete mode 100644 chest_xray/run/download_ckpt.yaml delete mode 100644 chest_xray/run/download_data.yaml delete mode 100644 chest_xray/run/infer.yml delete mode 100644 chest_xray/tasks/download_ckpt.yaml delete mode 100644 chest_xray/tasks/download_data.yaml delete mode 100644 chest_xray/tasks/infer.yaml create mode 100644 chexpert/README.md create mode 100644 chexpert/mlcube/mlcube.yaml create mode 100644 chexpert/project/Dockerfile create mode 100644 chexpert/project/chexpert.py create mode 100644 chexpert/project/data/dataset.py create mode 100644 chexpert/project/data/imgaug.py create mode 100644 chexpert/project/data/utils.py create mode 100755 chexpert/project/download_model.sh create mode 100644 chexpert/project/mlcube.py create mode 100644 chexpert/project/model/attention_map.py create mode 100644 chexpert/project/model/backbone/__init__.py create mode 100644 chexpert/project/model/backbone/densenet.py create mode 100644 chexpert/project/model/backbone/inception.py create mode 100644 chexpert/project/model/backbone/vgg.py create mode 100644 chexpert/project/model/classifier.py create mode 100644 chexpert/project/model/global_pool.py create mode 100644 chexpert/project/model/utils.py create mode 100644 chexpert/project/requirements.txt diff --git a/chest_xray/mlcube.yaml b/chest_xray/mlcube.yaml deleted file mode 100644 index 6250940..0000000 --- a/chest_xray/mlcube.yaml +++ /dev/null @@ -1,13 +0,0 @@ -schema_version: 1.0.0 -schema_type: mlcube_root - -name: chest_xray -author: MLPerf Best Practices Working Group -version: 0.1.0 -mlcube_spec_version: 0.1.0 - -tasks: - - 'tasks/download_data.yaml' - - 'tasks/download_ckpt.yaml' - - 'tasks/infer.yaml' - diff --git a/chest_xray/platforms/docker.yaml b/chest_xray/platforms/docker.yaml deleted file mode 100644 index cef5ea6..0000000 --- a/chest_xray/platforms/docker.yaml +++ /dev/null @@ -1,8 +0,0 @@ -schema_type: mlcube_platform -schema_version: 0.1.0 - -platform: - name: docker - version: ">=18.01" -container: - image: "mlperf/mlbox_chest_xray:0.0.1" \ No newline at end of file diff --git a/chest_xray/run/download_ckpt.yaml b/chest_xray/run/download_ckpt.yaml deleted file mode 100644 index e393561..0000000 --- a/chest_xray/run/download_ckpt.yaml +++ /dev/null @@ -1,9 +0,0 @@ -schema_type: mlcube_invoke -schema_version: 1.0.0 - -task_name: download_ckpt - -input_binding: {} - -output_binding: - ckpt: $WORKSPACE/data \ No newline at end of file diff --git a/chest_xray/run/download_data.yaml b/chest_xray/run/download_data.yaml deleted file mode 100644 index cfd1a4c..0000000 --- a/chest_xray/run/download_data.yaml +++ /dev/null @@ -1,10 +0,0 @@ -schema_type: mlcube_invoke -schema_version: 1.0.0 - -task_name: download_data - -input_binding: {} - -output_binding: - data_dir: $WORKSPACE/data - log_dir: $WORKSPACE/download_data_logs \ No newline at end of file diff --git a/chest_xray/run/infer.yml b/chest_xray/run/infer.yml deleted file mode 100644 index 54c806c..0000000 --- a/chest_xray/run/infer.yml +++ /dev/null @@ -1,9 +0,0 @@ -schema_type: mlcube_invoke -schema_version: 1.0.0 - -task_name: infer - -input_binding: {} - -output_binding: - output_file: $WORKSPACE/inferences.txt diff --git a/chest_xray/tasks/download_ckpt.yaml b/chest_xray/tasks/download_ckpt.yaml deleted file mode 100644 index 8df2a7e..0000000 --- a/chest_xray/tasks/download_ckpt.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Schema -schema_version: 1.0.0 -schema_type: mlcube_task - -# Task Inputs -inputs: [] - -# Task Outputs -outputs: - - name: ckpt - type: file \ No newline at end of file diff --git a/chest_xray/tasks/download_data.yaml b/chest_xray/tasks/download_data.yaml deleted file mode 100644 index 6ed5ad1..0000000 --- a/chest_xray/tasks/download_data.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Schema -schema_version: 1.0.0 -schema_type: mlcube_task - -# Task Inputs -inputs: [] - -# Task Outputs -outputs: - - name: data_dir - type: directory - - - name: log_dir - type: directory \ No newline at end of file diff --git a/chest_xray/tasks/infer.yaml b/chest_xray/tasks/infer.yaml deleted file mode 100644 index bd02ce4..0000000 --- a/chest_xray/tasks/infer.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Schema -schema_version: 1.0.0 -schema_type: mlcube_task - -# Task Inputs -# TODO: check if this is correct or we need an input -inputs: [] - -# Task Outputs -outputs: - - name: output_file - type: file \ No newline at end of file diff --git a/chexpert/README.md b/chexpert/README.md new file mode 100644 index 0000000..07a5fdc --- /dev/null +++ b/chexpert/README.md @@ -0,0 +1,43 @@ +# MLCube: Chexpert Example + +### Project setup +```Python +# Create Python environment +virtualenv -p python3 ./env && source ./env/bin/activate + +# Install MLCube and MLCube docker runner from GitHub repository (normally, users will just run `pip install mlcube mlcube_docker`) +git clone https://github.com/sergey-serebryakov/mlbox.git && cd mlbox && git checkout feature/configV2 +cd ./mlcube && python setup.py bdist_wheel && pip install --force-reinstall ./dist/mlcube-* && cd .. +cd ./runners/mlcube_docker && python setup.py bdist_wheel && pip install --force-reinstall --no-deps ./dist/mlcube_docker-* && cd ../../.. +``` + +## Clone MLCube examples and go to chexpert +``` +git clone https://github.com/mlperf/mlcube_examples.git && cd ./mlcube_examples +git fetch origin pull/XX/head:chest-xray-example && git checkout chest-xray-example +cd ./chexpert +``` + +## Run Chexpert MLCube on a local machine with Docker runner +``` +# Run Chexpert training tasks: download data, train model and evaluate model +mlcube run --task download_data +mlcube run --task download_model +mlcube run --task infer +``` + +Parameters defined in **mlcube.yaml** can be overridden using: `param=input`, example: + +``` +mlcube run --task download_model data_dir=path_to_custom_dir +``` + +We are targeting pull-type installation, so MLCubes should be available on docker hub. If not, try this: + +``` +mlcube run ... -Pdocker.build_strategy=auto +``` + +By default, at the end of the download_model task, Chexpert model will be saved in `workspace/model`. + +By default, at the end of the infer task, results will be saved in `workspace/inferences.txt`. \ No newline at end of file diff --git a/chexpert/mlcube/mlcube.yaml b/chexpert/mlcube/mlcube.yaml new file mode 100644 index 0000000..4307679 --- /dev/null +++ b/chexpert/mlcube/mlcube.yaml @@ -0,0 +1,30 @@ +name: MLCommons Chexpert +description: MLCommons Chexpert examplefor inference with the Chexpert model. +authors: + - {name: "MLCommons Best Practices Working Group"} + +platform: + accelerator_count: 0 + +docker: + # Image name. + image: mlcommons/chexpert:0.0.1 + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + download_data: + # Download mnist dataset + parameters: + outputs: {data_dir: data/, log_dir: download_logs/} + download_model: + # Train model + parameters: + outputs: {model_dir: model/} + infer: + # evaluate model + parameters: + inputs: {data_dir: data/, model_dir: model/} + outputs: {log_dir: inference_logs/, out_dir: ./} \ No newline at end of file diff --git a/chexpert/project/Dockerfile b/chexpert/project/Dockerfile new file mode 100644 index 0000000..db58fc1 --- /dev/null +++ b/chexpert/project/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 +MAINTAINER MLPerf MLBox Working Group + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + python3-dev \ + curl \ + wget \ + libsm6 libxext6 libxrender-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update + +RUN apt-get install python3.7 -y + +RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ + python3.7 get-pip.py && \ + rm get-pip.py + +COPY ./requirements.txt project/requirements.txt + +RUN python3.7 -m pip install --upgrade pip + +RUN python3.7 -m pip install --no-cache-dir -r project/requirements.txt + +COPY . /project + +WORKDIR /project + +ENTRYPOINT ["python3.7", "mlcube.py"] \ No newline at end of file diff --git a/chexpert/project/chexpert.py b/chexpert/project/chexpert.py new file mode 100644 index 0000000..ae3bb62 --- /dev/null +++ b/chexpert/project/chexpert.py @@ -0,0 +1,183 @@ +import os +import yaml +import sys +import argparse +import logging +import logging.config +import json +import time +from enum import Enum +from typing import List +from easydict import EasyDict as edict +import torch +import numpy as np +from torch.utils.data import DataLoader +from torch.nn import DataParallel +import torch.nn.functional as F + +# sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../') + +from data.dataset import ImageDataset # noqa +from model.classifier import Classifier # noqa + +logger = logging.getLogger(__name__) + +class Task(str, Enum): + DownloadData = 'download_data' + DownloadCkpt = 'download_ckpt' + Infer = 'infer' + +def create_directory(path: str) -> None: + if not os.path.exists(path): + os.makedirs(path, exist_ok=True) + + +def download_data(task_args: List[str]) -> None: + """ Task: download_data. + + Input parameters: + --data_dir + """ + logger.info(f"Starting '{Task.DownloadData}' task") + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', '--data-dir', type=str, default=None, help="Path to a dataset file.") + args = parser.parse_args(args=task_args) + + if args.data_dir is None: + raise ValueError("Data directory is not specified (did you use --data-dir=PATH?)") + os.makedirs(args.data_dir, exist_ok=True) + + if not args.data_dir.startswith("/"): + logger.warning("Data directory seems to be a relative path.") + + # TODO: check if data has already been downloaded. If so, return + + # TODO: actually download the data + return + + +def infer(task_args: List[str]) -> None: + """ Task: infer + + Input parameters: + --data_dir, --ckpt_dir, --out_dir + """ + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', '--data-dir', type=str, default=None, help="Dataset path.") + parser.add_argument('--model_dir', '--model-dir', type=str, default=None, help="Model location.") + parser.add_argument('--out_dir', '--out-dir', type=str, default=None, help="Model output directory.") + + args = parser.parse_args(args=task_args) + run(args) + + +def get_pred(output, cfg): + if cfg.criterion == 'BCE' or cfg.criterion == "FL": + for num_class in cfg.num_classes: + assert num_class == 1 + pred = torch.sigmoid(output.view(-1)).cpu().detach().numpy() + elif cfg.criterion == 'CE': + for num_class in cfg.num_classes: + assert num_class >= 2 + prob = F.softmax(output) + pred = prob[:, 1].cpu().detach().numpy() + else: + raise Exception('Unknown criterion : {}'.format(cfg.criterion)) + + return pred + + +def test_epoch(cfg, model, device, dataloader, out_csv_path): + torch.set_grad_enabled(False) + steps = len(dataloader) + dataiter = iter(dataloader) + num_tasks = len(cfg.num_classes) + + test_header = [ + 'Path', + 'Cardiomegaly', + 'Edema', + 'Consolidation', + 'Atelectasis', + 'Pleural Effusion'] + + with open(out_csv_path, 'w') as f: + f.write(','.join(test_header) + '\n') + for step in range(steps): + image, path = next(dataiter) + image = image.to(device) + output, __ = model(image) + batch_size = len(path) + pred = np.zeros((num_tasks, batch_size)) + + for i in range(num_tasks): + pred[i] = get_pred(output[i], cfg) + + for i in range(batch_size): + batch = ','.join(map(lambda x: '{}'.format(x), pred[:, i])) + result = path[i] + ',' + batch + f.write(result + '\n') + logging.info('{}, Image : {}, Prob : {}'.format( + time.strftime("%Y-%m-%d %H:%M:%S"), path[i], batch)) + + +def run(args): + ckpt_path = os.path.join(args.model_dir, 'model.pth') + config_path = os.path.join(args.model_dir, 'config.json') + print(config_path) + with open(config_path) as f: + cfg = edict(json.load(f)) + + device = torch.device('cuda' if torch.cuda.is_available() else "cpu") + + ckpt = torch.load(ckpt_path, map_location=device) + model = Classifier(cfg).to(device).eval() + model.load_state_dict(ckpt) + + out_csv_path = os.path.join(args.out_dir, "inferences.csv") + in_csv_path = os.path.join(args.data_dir, "valid.csv") + + dataloader_test = DataLoader( + ImageDataset(in_csv_path, cfg, args.data_dir, mode='test'), + batch_size=cfg.dev_batch_size, + drop_last=False, shuffle=False) + + + test_epoch(cfg, model, device, dataloader_test, out_csv_path) + + +def main(): + """ + chexpert.py task task_specific_parameters... + """ + parser = argparse.ArgumentParser() + parser.add_argument('--log_dir', '--log-dir', type=str, required=True, help="Logging directory.") + mlcube_args, task_args = parser.parse_known_args() + + os.makedirs(mlcube_args.log_dir, exist_ok=True) + logger_config = { + "version": 1, + "disable_existing_loggers": True, + "formatters": { + "standard": {"format": "%(asctime)s - %(name)s - %(threadName)s - %(levelname)s - %(message)s"}, + }, + "handlers": { + "file_handler": { + "class": "logging.FileHandler", + "level": "INFO", + "formatter": "standard", + "filename": os.path.join(mlcube_args.log_dir, f"mlcube_chexpert_infer.log") + } + }, + "loggers": { + "": {"level": "INFO", "handlers": ["file_handler"]}, + "__main__": {"level": "NOTSET", "propagate": "yes"}, + "tensorflow": {"level": "NOTSET", "propagate": "yes"} + } + } + logging.config.dictConfig(logger_config) + infer(task_args) + + +if __name__ == '__main__': + main() diff --git a/chexpert/project/data/dataset.py b/chexpert/project/data/dataset.py new file mode 100644 index 0000000..70cba93 --- /dev/null +++ b/chexpert/project/data/dataset.py @@ -0,0 +1,78 @@ +import numpy as np +from torch.utils.data import Dataset +import cv2 +import os +from PIL import Image +from data.imgaug import GetTransforms +from data.utils import transform +np.random.seed(0) + + +class ImageDataset(Dataset): + def __init__(self, label_path, cfg, data_path, mode='train'): + self.cfg = cfg + self._label_header = None + self._image_paths = [] + self._labels = [] + self._mode = mode + self.dict = [{'1.0': '1', '': '0', '0.0': '0', '-1.0': '0'}, + {'1.0': '1', '': '0', '0.0': '0', '-1.0': '1'}, ] + with open(label_path) as f: + header = f.readline().strip('\n').split(',') + self._label_header = [ + header[7], + header[10], + header[11], + header[13], + header[15]] + for line in f: + labels = [] + fields = line.strip('\n').split(',') + image_path = os.path.join(data_path, fields[0]) + flg_enhance = False + for index, value in enumerate(fields[5:]): + if index == 5 or index == 8: + labels.append(self.dict[1].get(value)) + if self.dict[1].get( + value) == '1' and \ + self.cfg.enhance_index.count(index) > 0: + flg_enhance = True + elif index == 2 or index == 6 or index == 10: + labels.append(self.dict[0].get(value)) + if self.dict[0].get( + value) == '1' and \ + self.cfg.enhance_index.count(index) > 0: + flg_enhance = True + # labels = ([self.dict.get(n, n) for n in fields[5:]]) + labels = list(map(int, labels)) + self._image_paths.append(image_path) + assert os.path.exists(image_path), image_path + self._labels.append(labels) + if flg_enhance and self._mode == 'train': + for i in range(self.cfg.enhance_times): + self._image_paths.append(image_path) + self._labels.append(labels) + self._num_image = len(self._image_paths) + + def __len__(self): + return self._num_image + + def __getitem__(self, idx): + image = cv2.imread(self._image_paths[idx], 0) + image = Image.fromarray(image) + if self._mode == 'train': + image = GetTransforms(image, type=self.cfg.use_transforms_type) + image = np.array(image) + image = transform(image, self.cfg) + labels = np.array(self._labels[idx]).astype(np.float32) + + path = self._image_paths[idx] + + if self._mode == 'train' or self._mode == 'dev': + return (image, labels) + elif self._mode == 'test': + return (image, path) + elif self._mode == 'heatmap': + return (image, path, labels) + else: + raise Exception('Unknown mode : {}'.format(self._mode)) diff --git a/chexpert/project/data/imgaug.py b/chexpert/project/data/imgaug.py new file mode 100644 index 0000000..67d2777 --- /dev/null +++ b/chexpert/project/data/imgaug.py @@ -0,0 +1,39 @@ +import cv2 +import torchvision.transforms as tfs + + +def Common(image): + + image = cv2.equalizeHist(image) + image = cv2.GaussianBlur(image, (3, 3), 0) + + return image + + +def Aug(image): + img_aug = tfs.Compose([ + tfs.RandomAffine(degrees=(-15, 15), translate=(0.05, 0.05), + scale=(0.95, 1.05), fillcolor=128) + ]) + image = img_aug(image) + + return image + + +def GetTransforms(image, target=None, type='common'): + # taget is not support now + if target is not None: + raise Exception( + 'Target is not support now ! ') + # get type + if type.strip() == 'Common': + image = Common(image) + return image + elif type.strip() == 'None': + return image + elif type.strip() == 'Aug': + image = Aug(image) + return image + else: + raise Exception( + 'Unknown transforms_type : '.format(type)) diff --git a/chexpert/project/data/utils.py b/chexpert/project/data/utils.py new file mode 100644 index 0000000..30b0e03 --- /dev/null +++ b/chexpert/project/data/utils.py @@ -0,0 +1,68 @@ +import numpy as np +import cv2 + + +def border_pad(image, cfg): + h, w, c = image.shape + + if cfg.border_pad == 'zero': + image = np.pad(image, ((0, cfg.long_side - h), + (0, cfg.long_side - w), (0, 0)), + mode='constant', + constant_values=0.0) + elif cfg.border_pad == 'pixel_mean': + image = np.pad(image, ((0, cfg.long_side - h), + (0, cfg.long_side - w), (0, 0)), + mode='constant', + constant_values=cfg.pixel_mean) + else: + image = np.pad(image, ((0, cfg.long_side - h), + (0, cfg.long_side - w), (0, 0)), + mode=cfg.border_pad) + + return image + + +def fix_ratio(image, cfg): + h, w, c = image.shape + + if h >= w: + ratio = h * 1.0 / w + h_ = cfg.long_side + w_ = round(h_ / ratio) + else: + ratio = w * 1.0 / h + w_ = cfg.long_side + h_ = round(w_ / ratio) + + image = cv2.resize(image, dsize=(w_, h_), interpolation=cv2.INTER_LINEAR) + image = border_pad(image, cfg) + + return image + + +def transform(image, cfg): + assert image.ndim == 2, "image must be gray image" + if cfg.use_equalizeHist: + image = cv2.equalizeHist(image) + + if cfg.gaussian_blur > 0: + image = cv2.GaussianBlur( + image, + (cfg.gaussian_blur, cfg.gaussian_blur), 0) + + image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) + + image = fix_ratio(image, cfg) + # augmentation for train or co_train + + # normalization + image = image.astype(np.float32) - cfg.pixel_mean + # vgg and resnet do not use pixel_std, densenet and inception use. + if cfg.pixel_std: + image /= cfg.pixel_std + # normal image tensor : H x W x C + # torch image tensor : C X H X W + image = image.transpose((2, 0, 1)) + + return image diff --git a/chexpert/project/download_model.sh b/chexpert/project/download_model.sh new file mode 100755 index 0000000..8ec360d --- /dev/null +++ b/chexpert/project/download_model.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +MODEL_FILE="model.pth" +CONFIG_FILE="config.json" +MODEL_DIR="${MODEL_DIR:-./checkpoints}" +MODEL_PATH="${MODEL_DIR}/${MODEL_FILE}" +CONFIG_PATH="${MODEL_DIR}/${CONFIG_FILE}" + +if [ ! -d "$MODEL_DIR" ] +then + mkdir $MODEL_DIR + chmod go+rx $MODEL_DIR +# python utils/download_librispeech.py utils/librispeech.csv $DATA_DIR -e ${DATA_ROOT_DIR}/ +fi +curl https://raw.githubusercontent.com/jfhealthcare/Chexpert/master/config/pre_train.pth --output ${MODEL_PATH} +curl https://raw.githubusercontent.com/jfhealthcare/Chexpert/master/config/example.json --output ${CONFIG_PATH} + diff --git a/chexpert/project/mlcube.py b/chexpert/project/mlcube.py new file mode 100644 index 0000000..e6a4557 --- /dev/null +++ b/chexpert/project/mlcube.py @@ -0,0 +1,71 @@ +"""MLCube handler file""" +import os +import yaml +import typer +import shutil +import subprocess +from pathlib import Path + + +app = typer.Typer() + +class DownloadDataTask(object): + """Download task Class + It defines the environment variables: + DATA_ROOT_DIR: Directory path to download the dataset + Then executes the download script""" + @staticmethod + def run(data_dir: str) -> None: + + command = f"python 01_download_dataset.py --data_dir {data_dir}" + splitted_command = command.split() + process = subprocess.Popen(splitted_command, cwd=".") + process.wait() + +class DownloadModelTask(object): + """Preprocess dataset task Class + It defines the environment variables: + DATA_ROOT_DIR: Dataset directory path + Then executes the preprocess script""" + @staticmethod + def run(model_dir: str) -> None: + + env = os.environ.copy() + env.update({ + 'MODEL_DIR': model_dir, + }) + + process = subprocess.Popen("./download_model.sh", cwd=".", env=env) + process.wait() + +class InferTask(object): + """Preprocess dataset task Class + It defines the environment variables: + DATA_DIR: Dataset directory path + All other parameters are defined in the parameters_file + Then executes the benchmark script""" + @staticmethod + def run(log_dir: str, data_dir: str, model_dir: str, out_dir) -> None: + cmd = f"python3.7 chexpert.py --log_dir={log_dir} --data_dir={data_dir} --model_dir={model_dir} --out_dir={out_dir}" + splitted_cmd = cmd.split() + + process = subprocess.Popen(splitted_cmd, cwd=".") + process.wait() + +@app.command("download_data") +def download_data(data_dir: str = typer.Option(..., '--data_dir')): + DownloadDataTask.run(data_dir) + +@app.command("download_model") +def download_model(model_dir: str = typer.Option(..., '--model_dir')): + DownloadModelTask.run(model_dir) + +@app.command("infer") +def infer(log_dir: str = typer.Option(..., '--log_dir'), + data_dir: str = typer.Option(..., '--data_dir'), + model_dir: str = typer.Option(..., '--model_dir'), + out_dir: str = typer.Option(..., '--out_dir')): + InferTask.run(log_dir, data_dir, model_dir, out_dir) + +if __name__ == '__main__': + app() \ No newline at end of file diff --git a/chexpert/project/model/attention_map.py b/chexpert/project/model/attention_map.py new file mode 100644 index 0000000..8957934 --- /dev/null +++ b/chexpert/project/model/attention_map.py @@ -0,0 +1,186 @@ +import torch +from torch import nn +from torch.nn import functional as F + +from model.utils import get_norm + + +class Conv2dNormRelu(nn.Module): + + def __init__(self, in_ch, out_ch, kernel_size=3, stride=1, padding=0, + bias=True, norm_type='Unknown'): + super(Conv2dNormRelu, self).__init__() + + self.conv = nn.Sequential( + nn.Conv2d(in_ch, out_ch, kernel_size, stride, padding, bias=bias), + get_norm(norm_type, out_ch), + nn.ReLU(inplace=True)) + + def forward(self, x): + return self.conv(x) + + +class CAModule(nn.Module): + """ + Re-implementation of Squeeze-and-Excitation (SE) block described in: + *Hu et al., Squeeze-and-Excitation Networks, arXiv:1709.01507* + code reference: + https://github.com/kobiso/CBAM-keras/blob/master/models/attention_module.py + """ + + def __init__(self, num_channels, reduc_ratio=2): + super(CAModule, self).__init__() + self.num_channels = num_channels + self.reduc_ratio = reduc_ratio + + self.fc1 = nn.Linear(num_channels, num_channels // reduc_ratio, + bias=True) + self.fc2 = nn.Linear(num_channels // reduc_ratio, num_channels, + bias=True) + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + + def forward(self, feat_map): + # attention branch--squeeze operation + gap_out = feat_map.view(feat_map.size()[0], self.num_channels, + -1).mean(dim=2) + + # attention branch--excitation operation + fc1_out = self.relu(self.fc1(gap_out)) + fc2_out = self.sigmoid(self.fc2(fc1_out)) + + # attention operation + fc2_out = fc2_out.view(fc2_out.size()[0], fc2_out.size()[1], 1, 1) + feat_map = torch.mul(feat_map, fc2_out) + + return feat_map + + +class SAModule(nn.Module): + """ + Re-implementation of spatial attention module (SAM) described in: + *Liu et al., Dual Attention Network for Scene Segmentation, cvpr2019 + code reference: + https://github.com/junfu1115/DANet/blob/master/encoding/nn/attention.py + """ + + def __init__(self, num_channels): + super(SAModule, self).__init__() + self.num_channels = num_channels + + self.conv1 = nn.Conv2d(in_channels=num_channels, + out_channels=num_channels // 8, kernel_size=1) + self.conv2 = nn.Conv2d(in_channels=num_channels, + out_channels=num_channels // 8, kernel_size=1) + self.conv3 = nn.Conv2d(in_channels=num_channels, + out_channels=num_channels, kernel_size=1) + self.gamma = nn.Parameter(torch.zeros(1)) + + def forward(self, feat_map): + batch_size, num_channels, height, width = feat_map.size() + + conv1_proj = self.conv1(feat_map).view(batch_size, -1, + width * height).permute(0, 2, 1) + + conv2_proj = self.conv2(feat_map).view(batch_size, -1, width * height) + + relation_map = torch.bmm(conv1_proj, conv2_proj) + attention = F.softmax(relation_map, dims=-1) + + conv3_proj = self.conv3(feat_map).view(batch_size, -1, width * height) + + feat_refine = torch.bmm(conv3_proj, attention.permute(0, 2, 1)) + feat_refine = feat_refine.view(batch_size, num_channels, height, width) + + feat_map = self.gamma * feat_refine + feat_map + + return feat_map + + +class FPAModule(nn.Module): + """ + Re-implementation of feature pyramid attention (FPA) described in: + *Li et al., Pyramid Attention Network for Semantic segmentation, Face++2018 + """ + + def __init__(self, num_channels, norm_type): + super(FPAModule, self).__init__() + + # global pooling branch + self.gap_branch = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + Conv2dNormRelu(num_channels, num_channels, kernel_size=1, + norm_type=norm_type)) + + # middle branch + self.mid_branch = Conv2dNormRelu(num_channels, num_channels, + kernel_size=1, norm_type=norm_type) + + self.downsample1 = Conv2dNormRelu(num_channels, 1, kernel_size=7, + stride=2, padding=3, + norm_type=norm_type) + + self.downsample2 = Conv2dNormRelu(1, 1, kernel_size=5, stride=2, + padding=2, norm_type=norm_type) + + self.downsample3 = Conv2dNormRelu(1, 1, kernel_size=3, stride=2, + padding=1, norm_type=norm_type) + + self.scale1 = Conv2dNormRelu(1, 1, kernel_size=7, padding=3, + norm_type=norm_type) + self.scale2 = Conv2dNormRelu(1, 1, kernel_size=5, padding=2, + norm_type=norm_type) + self.scale3 = Conv2dNormRelu(1, 1, kernel_size=3, padding=1, + norm_type=norm_type) + + def forward(self, feat_map): + height, width = feat_map.size(2), feat_map.size(3) + gap_branch = self.gap_branch(feat_map) + gap_branch = nn.Upsample(size=(height, width), mode='bilinear', + align_corners=False)(gap_branch) + + mid_branch = self.mid_branch(feat_map) + + scale1 = self.downsample1(feat_map) + scale2 = self.downsample2(scale1) + scale3 = self.downsample3(scale2) + + scale3 = self.scale3(scale3) + scale3 = nn.Upsample(size=(height // 4, width // 4), mode='bilinear', + align_corners=False)(scale3) + scale2 = self.scale2(scale2) + scale3 + scale2 = nn.Upsample(size=(height // 2, width // 2), mode='bilinear', + align_corners=False)(scale2) + scale1 = self.scale1(scale1) + scale2 + scale1 = nn.Upsample(size=(height, width), mode='bilinear', + align_corners=False)(scale1) + + feat_map = torch.mul(scale1, mid_branch) + gap_branch + + return feat_map + + +class AttentionMap(nn.Module): + + def __init__(self, cfg, num_channels): + super(AttentionMap, self).__init__() + self.cfg = cfg + self.channel_attention = CAModule(num_channels) + self.spatial_attention = SAModule(num_channels) + self.pyramid_attention = FPAModule(num_channels, cfg.norm_type) + + def cuda(self, device=None): + return self._apply(lambda t: t.cuda(device)) + + def forward(self, feat_map): + if self.cfg.attention_map == "CAM": + return self.channel_attention(feat_map) + elif self.cfg.attention_map == "SAM": + return self.spatial_attention(feat_map) + elif self.cfg.attention_map == "FPA": + return self.pyramid_attention(feat_map) + elif self.cfg.attention_map == "None": + return feat_map + else: + Exception('Unknown attention type : {}' + .format(self.cfg.attention_map)) diff --git a/chexpert/project/model/backbone/__init__.py b/chexpert/project/model/backbone/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chexpert/project/model/backbone/densenet.py b/chexpert/project/model/backbone/densenet.py new file mode 100644 index 0000000..fcd40d6 --- /dev/null +++ b/chexpert/project/model/backbone/densenet.py @@ -0,0 +1,235 @@ +import re +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.model_zoo as model_zoo +from collections import OrderedDict +from model.utils import get_norm + +__all__ = ['DenseNet', 'densenet121', 'densenet169', 'densenet201', 'densenet161'] # noqa + + +model_urls = { + 'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth', # noqa + 'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth', # noqa + 'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth', # noqa + 'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth', # noqa +} + + +class _DenseLayer(nn.Sequential): + def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, + norm_type='Unknown'): + super(_DenseLayer, self).__init__() + self.add_module('norm1', get_norm(norm_type, num_input_features)), + self.add_module('relu1', nn.ReLU(inplace=True)), + self.add_module('conv1', nn.Conv2d(num_input_features, bn_size * + growth_rate, kernel_size=1, stride=1, bias=False)), + self.add_module('norm2', get_norm(norm_type, bn_size * growth_rate)), + self.add_module('relu2', nn.ReLU(inplace=True)), + self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate, + kernel_size=3, stride=1, padding=1, bias=False)), + self.drop_rate = drop_rate + + def forward(self, x): + new_features = super(_DenseLayer, self).forward(x) + if self.drop_rate > 0: + new_features = F.dropout(new_features, p=self.drop_rate, training=self.training) # noqa + return torch.cat([x, new_features], 1) + + +class _DenseBlock(nn.Sequential): + def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, norm_type='Unknown'): # noqa + super(_DenseBlock, self).__init__() + for i in range(num_layers): + layer = _DenseLayer(num_input_features + i * growth_rate, growth_rate, bn_size, drop_rate, norm_type=norm_type) # noqa + self.add_module('denselayer%d' % (i + 1), layer) + + +class _Transition(nn.Sequential): + def __init__(self, num_input_features, num_output_features, + norm_type='Unknown'): + super(_Transition, self).__init__() + self.add_module('norm', get_norm(norm_type, num_input_features)) + self.add_module('relu', nn.ReLU(inplace=True)) + self.add_module('conv', nn.Conv2d(num_input_features, num_output_features, # noqa + kernel_size=1, stride=1, bias=False)) + self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2)) + + +class DenseNet(nn.Module): + r"""Densenet-BC model class, based on + `"Densely Connected Convolutional Networks" `_ # noqa + + Args: + growth_rate (int) - how many filters to add each layer (`k` in paper) + block_config (list of 4 ints) - how many layers in each pooling block + num_init_features (int) - the number of filters to learn in the first convolution layer # noqa + bn_size (int) - multiplicative factor for number of bottle neck layers + (i.e. bn_size * k features in the bottleneck layer) + drop_rate (float) - dropout rate after each dense layer + num_classes (int) - number of classification classes + """ + + def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16), + norm_type='Unknown', num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000): # noqa + + super(DenseNet, self).__init__() + + # First convolution + self.features = nn.Sequential(OrderedDict([ + ('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, padding=3, bias=False)), # noqa + ('norm0', get_norm(norm_type, num_init_features)), + ('relu0', nn.ReLU(inplace=True)), + ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)), + ])) + + # Each denseblock + num_features = num_init_features + for i, num_layers in enumerate(block_config): + block = _DenseBlock(num_layers=num_layers, num_input_features=num_features, norm_type=norm_type, # noqa + bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate) # noqa + self.features.add_module('denseblock%d' % (i + 1), block) + num_features = num_features + num_layers * growth_rate + if i != len(block_config) - 1: + trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2, norm_type=norm_type) # noqa + self.features.add_module('transition%d' % (i + 1), trans) + num_features = num_features // 2 + + # Final batch norm + self.features.add_module('norm5', get_norm(norm_type, num_features)) + + # Linear layer + self.classifier = nn.Linear(num_features, num_classes) + self.num_features = num_features + + # Official init from torch repo. + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.GroupNorm): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.InstanceNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def forward(self, x): + features = self.features(x) + out = F.relu(features, inplace=True) + # out = F.adaptive_avg_pool2d(out, (1, 1)).view(features.size(0), -1) + # out = self.classifier(out) + return out + + +def densenet121(cfg, **kwargs): + r"""Densenet-121 model from + `"Densely Connected Convolutional Networks" `_ # noqa + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 24, 16), # noqa + norm_type=cfg.norm_type, **kwargs) + if cfg.pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') # noqa + state_dict = model_zoo.load_url(model_urls['densenet121']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict, strict=False) + return model + + +def densenet169(cfg, **kwargs): + r"""Densenet-169 model from + `"Densely Connected Convolutional Networks" `_ # noqa + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 32, 32), # noqa + norm_type=cfg.norm_type, **kwargs) + if cfg.pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') # noqa + state_dict = model_zoo.load_url(model_urls['densenet169']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict, strict=False) + return model + + +def densenet201(cfg, **kwargs): + r"""Densenet-201 model from + `"Densely Connected Convolutional Networks" `_ # noqa + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 48, 32), # noqa + norm_type=cfg.norm_type, **kwargs) + if cfg.pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') # noqa + state_dict = model_zoo.load_url(model_urls['densenet201']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict, strict=False) + return model + + +def densenet161(cfg, **kwargs): + r"""Densenet-161 model from + `"Densely Connected Convolutional Networks" `_ # noqa + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = DenseNet(num_init_features=96, growth_rate=48, block_config=(6, 12, 36, 24), # noqa + norm_type=cfg.norm_type, **kwargs) + if cfg.pretrained: + # '.'s are no longer allowed in module names, but pervious _DenseLayer + # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. + # They are also in the checkpoints in model_urls. This pattern is used + # to find such keys. + pattern = re.compile( + r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') # noqa + state_dict = model_zoo.load_url(model_urls['densenet161']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict, strict=False) + return model diff --git a/chexpert/project/model/backbone/inception.py b/chexpert/project/model/backbone/inception.py new file mode 100644 index 0000000..9463401 --- /dev/null +++ b/chexpert/project/model/backbone/inception.py @@ -0,0 +1,394 @@ +import re +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.model_zoo as model_zoo +from model.utils import get_norm + +__all__ = ['Inception3', 'inception_v3'] + + +model_urls = { + # Inception v3 ported from TensorFlow + 'inception_v3_google': 'https://download.pytorch.org/models/inception_v3_google-1a9a5a14.pth', # noqa +} + + +def inception_v3(cfg, **kwargs): + r"""Inception v3 model architecture from + `"Rethinking the Inception Architecture for Computer Vision" `_. # noqa + + .. note:: + **Important**: In contrast to the other models the inception_v3 expects tensors with a size of # noqa + N x 3 x 299 x 299, so ensure your images are sized accordingly. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if cfg.pretrained: + if 'transform_input' not in kwargs: + kwargs['transform_input'] = True + model = Inception3(norm_type=cfg.norm_type, **kwargs) + + pattern = re.compile(r'^(.*bn\d\.(?:weight|bias|running_mean|running_var))$') # noqa + state_dict = model_zoo.load_url(model_urls['inception_v3_google']) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1).replace('bn', 'norm') + state_dict[new_key] = state_dict[key] + del state_dict[key] + model.load_state_dict(state_dict, strict=False) + return model + + return Inception3(norm_type=cfg.norm_type, **kwargs) + + +class Inception3(nn.Module): + + def __init__(self, num_classes=1000, norm_type='Unknown', aux_logits=True, transform_input=False): # noqa + super(Inception3, self).__init__() + self.aux_logits = aux_logits + self.transform_input = transform_input + self.Conv2d_1a_3x3 = BasicConv2d(3, 32, norm_type=norm_type, + kernel_size=3, stride=2) + self.Conv2d_2a_3x3 = BasicConv2d(32, 32, norm_type=norm_type, + kernel_size=3) + self.Conv2d_2b_3x3 = BasicConv2d(32, 64, norm_type=norm_type, + kernel_size=3, padding=1) + self.Conv2d_3b_1x1 = BasicConv2d(64, 80, norm_type=norm_type, + kernel_size=1) + self.Conv2d_4a_3x3 = BasicConv2d(80, 192, norm_type=norm_type, + kernel_size=3) + self.Mixed_5b = InceptionA(192, pool_features=32, norm_type=norm_type) + self.Mixed_5c = InceptionA(256, pool_features=64, norm_type=norm_type) + self.Mixed_5d = InceptionA(288, pool_features=64, norm_type=norm_type) + self.Mixed_6a = InceptionB(288, norm_type=norm_type) + self.Mixed_6b = InceptionC(768, channels_7x7=128, norm_type=norm_type) + self.Mixed_6c = InceptionC(768, channels_7x7=160, norm_type=norm_type) + self.Mixed_6d = InceptionC(768, channels_7x7=160, norm_type=norm_type) + self.Mixed_6e = InceptionC(768, channels_7x7=192, norm_type=norm_type) + if aux_logits: + self.AuxLogits = InceptionAux(768, num_classes, + norm_type=norm_type) + self.Mixed_7a = InceptionD(768, norm_type=norm_type) + self.Mixed_7b = InceptionE(1280, norm_type=norm_type) + self.Mixed_7c = InceptionE(2048, norm_type=norm_type) + self.fc = nn.Linear(2048, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): + import scipy.stats as stats + stddev = m.stddev if hasattr(m, 'stddev') else 0.1 + X = stats.truncnorm(-2, 2, scale=stddev) + values = torch.Tensor(X.rvs(m.weight.numel())) + values = values.view(m.weight.size()) + m.weight.data.copy_(values) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.GroupNorm): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.InstanceNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def forward(self, x): + if self.transform_input: + x_ch0 = torch.unsqueeze(x[:, 0], 1) * (0.229 / 0.5) + (0.485 - 0.5) / 0.5 # noqa + x_ch1 = torch.unsqueeze(x[:, 1], 1) * (0.224 / 0.5) + (0.456 - 0.5) / 0.5 # noqa + x_ch2 = torch.unsqueeze(x[:, 2], 1) * (0.225 / 0.5) + (0.406 - 0.5) / 0.5 # noqa + x = torch.cat((x_ch0, x_ch1, x_ch2), 1) + # N x 3 x 299 x 299 + x = self.Conv2d_1a_3x3(x) + # N x 32 x 149 x 149 + x = self.Conv2d_2a_3x3(x) + # N x 32 x 147 x 147 + x = self.Conv2d_2b_3x3(x) + # N x 64 x 147 x 147 + x = F.max_pool2d(x, kernel_size=3, stride=2) + # N x 64 x 73 x 73 + x = self.Conv2d_3b_1x1(x) + # N x 80 x 73 x 73 + x = self.Conv2d_4a_3x3(x) + # N x 192 x 71 x 71 + x = F.max_pool2d(x, kernel_size=3, stride=2) + # N x 192 x 35 x 35 + x = self.Mixed_5b(x) + # N x 256 x 35 x 35 + x = self.Mixed_5c(x) + # N x 288 x 35 x 35 + x = self.Mixed_5d(x) + # N x 288 x 35 x 35 + x = self.Mixed_6a(x) + # N x 768 x 17 x 17 + x = self.Mixed_6b(x) + # N x 768 x 17 x 17 + x = self.Mixed_6c(x) + # N x 768 x 17 x 17 + x = self.Mixed_6d(x) + # N x 768 x 17 x 17 + x = self.Mixed_6e(x) + # N x 768 x 17 x 17 + # if self.training and self.aux_logits: + # aux = self.AuxLogits(x) + # N x 768 x 17 x 17 + x = self.Mixed_7a(x) + # N x 1280 x 8 x 8 + x = self.Mixed_7b(x) + # N x 2048 x 8 x 8 + x = self.Mixed_7c(x) + # N x 2048 x 8 x 8 + # Adaptive average pooling + # x = F.adaptive_avg_pool2d(x, (1, 1)) + # N x 2048 x 1 x 1 + # x = F.dropout(x, training=self.training) + # N x 2048 x 1 x 1 + # x = x.view(x.size(0), -1) + # N x 2048 + # x = self.fc(x) + # N x 1000 (num_classes) + # if self.training and self.aux_logits: + # return x, aux + return x + + +class InceptionA(nn.Module): + + def __init__(self, in_channels, pool_features, norm_type='Unknown'): + super(InceptionA, self).__init__() + self.branch1x1 = BasicConv2d(in_channels, 64, + norm_type=norm_type, kernel_size=1) + + self.branch5x5_1 = BasicConv2d(in_channels, 48, norm_type=norm_type, + kernel_size=1) + self.branch5x5_2 = BasicConv2d(48, 64, norm_type=norm_type, + kernel_size=5, padding=2) + + self.branch3x3dbl_1 = BasicConv2d(in_channels, 64, norm_type=norm_type, + kernel_size=1) + self.branch3x3dbl_2 = BasicConv2d(64, 96, norm_type=norm_type, + kernel_size=3, padding=1) + self.branch3x3dbl_3 = BasicConv2d(96, 96, norm_type=norm_type, + kernel_size=3, padding=1) + + self.branch_pool = BasicConv2d(in_channels, pool_features, norm_type=norm_type, kernel_size=1) # noqa + + def forward(self, x): + branch1x1 = self.branch1x1(x) + + branch5x5 = self.branch5x5_1(x) + branch5x5 = self.branch5x5_2(branch5x5) + + branch3x3dbl = self.branch3x3dbl_1(x) + branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl) + branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl) + + branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1) + branch_pool = self.branch_pool(branch_pool) + + outputs = [branch1x1, branch5x5, branch3x3dbl, branch_pool] + return torch.cat(outputs, 1) + + +class InceptionB(nn.Module): + + def __init__(self, in_channels, norm_type='Unknown'): + super(InceptionB, self).__init__() + self.branch3x3 = BasicConv2d(in_channels, 384, norm_type=norm_type, + kernel_size=3, stride=2) + + self.branch3x3dbl_1 = BasicConv2d(in_channels, 64, + norm_type=norm_type, kernel_size=1) + self.branch3x3dbl_2 = BasicConv2d(64, 96, norm_type=norm_type, + kernel_size=3, padding=1) + self.branch3x3dbl_3 = BasicConv2d(96, 96, norm_type=norm_type, + kernel_size=3, stride=2) + + def forward(self, x): + branch3x3 = self.branch3x3(x) + + branch3x3dbl = self.branch3x3dbl_1(x) + branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl) + branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl) + + branch_pool = F.max_pool2d(x, kernel_size=3, stride=2) + + outputs = [branch3x3, branch3x3dbl, branch_pool] + return torch.cat(outputs, 1) + + +class InceptionC(nn.Module): + + def __init__(self, in_channels, channels_7x7, norm_type='Unknown'): + super(InceptionC, self).__init__() + self.branch1x1 = BasicConv2d(in_channels, 192, + norm_type=norm_type, kernel_size=1) + + c7 = channels_7x7 + self.branch7x7_1 = BasicConv2d(in_channels, c7, + norm_type=norm_type, kernel_size=1) + self.branch7x7_2 = BasicConv2d(c7, c7, norm_type=norm_type, + kernel_size=(1, 7), padding=(0, 3)) # noqa + self.branch7x7_3 = BasicConv2d(c7, 192, norm_type=norm_type, + kernel_size=(7, 1), padding=(3, 0)) # noqa + + self.branch7x7dbl_1 = BasicConv2d(in_channels, c7, + norm_type=norm_type, kernel_size=1) + self.branch7x7dbl_2 = BasicConv2d(c7, c7, norm_type=norm_type, + kernel_size=(7, 1), padding=(3, 0)) # noqa + self.branch7x7dbl_3 = BasicConv2d(c7, c7, norm_type=norm_type, + kernel_size=(1, 7), padding=(0, 3)) # noqa + self.branch7x7dbl_4 = BasicConv2d(c7, c7, norm_type=norm_type, + kernel_size=(7, 1), padding=(3, 0)) # noqa + self.branch7x7dbl_5 = BasicConv2d(c7, 192, norm_type=norm_type, + kernel_size=(1, 7), padding=(0, 3)) # noqa + + self.branch_pool = BasicConv2d(in_channels, 192, + norm_type=norm_type, kernel_size=1) + + def forward(self, x): + branch1x1 = self.branch1x1(x) + + branch7x7 = self.branch7x7_1(x) + branch7x7 = self.branch7x7_2(branch7x7) + branch7x7 = self.branch7x7_3(branch7x7) + + branch7x7dbl = self.branch7x7dbl_1(x) + branch7x7dbl = self.branch7x7dbl_2(branch7x7dbl) + branch7x7dbl = self.branch7x7dbl_3(branch7x7dbl) + branch7x7dbl = self.branch7x7dbl_4(branch7x7dbl) + branch7x7dbl = self.branch7x7dbl_5(branch7x7dbl) + + branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1) + branch_pool = self.branch_pool(branch_pool) + + outputs = [branch1x1, branch7x7, branch7x7dbl, branch_pool] + return torch.cat(outputs, 1) + + +class InceptionD(nn.Module): + + def __init__(self, in_channels, norm_type='Unknown'): + super(InceptionD, self).__init__() + self.branch3x3_1 = BasicConv2d(in_channels, 192, + norm_type=norm_type, kernel_size=1) + self.branch3x3_2 = BasicConv2d(192, 320, norm_type=norm_type, + kernel_size=3, stride=2) + + self.branch7x7x3_1 = BasicConv2d(in_channels, 192, norm_type=norm_type, + kernel_size=1) + self.branch7x7x3_2 = BasicConv2d(192, 192, norm_type=norm_type, + kernel_size=(1, 7), padding=(0, 3)) # noqa + self.branch7x7x3_3 = BasicConv2d(192, 192, norm_type=norm_type, + kernel_size=(7, 1), padding=(3, 0)) # noqa + self.branch7x7x3_4 = BasicConv2d(192, 192, norm_type=norm_type, + kernel_size=3, stride=2) + + def forward(self, x): + branch3x3 = self.branch3x3_1(x) + branch3x3 = self.branch3x3_2(branch3x3) + + branch7x7x3 = self.branch7x7x3_1(x) + branch7x7x3 = self.branch7x7x3_2(branch7x7x3) + branch7x7x3 = self.branch7x7x3_3(branch7x7x3) + branch7x7x3 = self.branch7x7x3_4(branch7x7x3) + + branch_pool = F.max_pool2d(x, kernel_size=3, stride=2) + outputs = [branch3x3, branch7x7x3, branch_pool] + return torch.cat(outputs, 1) + + +class InceptionE(nn.Module): + + def __init__(self, in_channels, norm_type='Unknown'): + super(InceptionE, self).__init__() + self.branch1x1 = BasicConv2d(in_channels, 320, + norm_type=norm_type, kernel_size=1) + + self.branch3x3_1 = BasicConv2d(in_channels, 384, + norm_type=norm_type, kernel_size=1) + self.branch3x3_2a = BasicConv2d(384, 384, norm_type=norm_type, + kernel_size=(1, 3), padding=(0, 1)) # noqa + self.branch3x3_2b = BasicConv2d(384, 384, norm_type=norm_type, + kernel_size=(3, 1), padding=(1, 0)) # noqa + + self.branch3x3dbl_1 = BasicConv2d(in_channels, 448, + norm_type=norm_type, kernel_size=1) + self.branch3x3dbl_2 = BasicConv2d(448, 384, norm_type=norm_type, + kernel_size=3, padding=1) + self.branch3x3dbl_3a = BasicConv2d(384, 384, norm_type=norm_type, + kernel_size=(1, 3), padding=(0, 1)) # noqa + self.branch3x3dbl_3b = BasicConv2d(384, 384, norm_type=norm_type, + kernel_size=(3, 1), padding=(1, 0)) # noqa + + self.branch_pool = BasicConv2d(in_channels, 192, + norm_type=norm_type, kernel_size=1) + + def forward(self, x): + branch1x1 = self.branch1x1(x) + + branch3x3 = self.branch3x3_1(x) + branch3x3 = [ + self.branch3x3_2a(branch3x3), + self.branch3x3_2b(branch3x3), + ] + branch3x3 = torch.cat(branch3x3, 1) + + branch3x3dbl = self.branch3x3dbl_1(x) + branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl) + branch3x3dbl = [ + self.branch3x3dbl_3a(branch3x3dbl), + self.branch3x3dbl_3b(branch3x3dbl), + ] + branch3x3dbl = torch.cat(branch3x3dbl, 1) + + branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1) + branch_pool = self.branch_pool(branch_pool) + + outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool] + return torch.cat(outputs, 1) + + +class InceptionAux(nn.Module): + + def __init__(self, in_channels, num_classes, norm_type='Unknown'): + super(InceptionAux, self).__init__() + self.conv0 = BasicConv2d(in_channels, 128, norm_type=norm_type, + kernel_size=1) + self.conv1 = BasicConv2d(128, 768, norm_type=norm_type, kernel_size=5) + self.conv1.stddev = 0.01 + self.fc = nn.Linear(768, num_classes) + self.fc.stddev = 0.001 + + def forward(self, x): + # N x 768 x 17 x 17 + x = F.avg_pool2d(x, kernel_size=5, stride=3) + # N x 768 x 5 x 5 + x = self.conv0(x) + # N x 128 x 5 x 5 + x = self.conv1(x) + # N x 768 x 1 x 1 + # Adaptive average pooling + x = F.adaptive_avg_pool2d(x, (1, 1)) + # N x 768 x 1 x 1 + x = x.view(x.size(0), -1) + # N x 768 + x = self.fc(x) + # N x 1000 + return x + + +class BasicConv2d(nn.Module): + + def __init__(self, in_channels, out_channels, norm_type='Unknown', + **kwargs): + super(BasicConv2d, self).__init__() + self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs) + self.norm = get_norm(norm_type, out_channels, eps=0.001) + + def forward(self, x): + x = self.conv(x) + x = self.norm(x) + return F.relu(x, inplace=True) diff --git a/chexpert/project/model/backbone/vgg.py b/chexpert/project/model/backbone/vgg.py new file mode 100644 index 0000000..0ca8507 --- /dev/null +++ b/chexpert/project/model/backbone/vgg.py @@ -0,0 +1,215 @@ +import torch.nn as nn +import torch.utils.model_zoo as model_zoo +from model.utils import get_norm + +__all__ = [ + 'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn', + 'vgg19_bn', 'vgg19', +] + + +model_urls = { + 'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth', + 'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth', + 'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth', + 'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth', + 'vgg11_bn': 'https://download.pytorch.org/models/vgg11_bn-6002323d.pth', + 'vgg13_bn': 'https://download.pytorch.org/models/vgg13_bn-abd245e5.pth', + 'vgg16_bn': 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth', + 'vgg19_bn': 'https://download.pytorch.org/models/vgg19_bn-c79401a0.pth', +} + + +class VGG(nn.Module): + + def __init__(self, features, num_classes=1000, init_weights=True): + super(VGG, self).__init__() + self.features = features + self.avgpool = nn.AdaptiveAvgPool2d((7, 7)) + self.classifier = nn.Sequential( + nn.Linear(512 * 7 * 7, 4096), + nn.ReLU(True), + nn.Dropout(), + nn.Linear(4096, 4096), + nn.ReLU(True), + nn.Dropout(), + nn.Linear(4096, num_classes), + ) + if init_weights: + self._initialize_weights() + + def forward(self, x): + x = self.features(x) + # x = self.avgpool(x) + # x = x.view(x.size(0), -1) + # x = self.classifier(x) + return x + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') # noqa + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.GroupNorm): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.InstanceNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + +def make_layers(cfg, batch_norm=False, + norm_type='Unknown'): + layers = [] + in_channels = 3 + for v in cfg: + if v == 'M': + layers += [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) + if batch_norm: + layers += [conv2d, get_norm(norm_type, v), + nn.ReLU(inplace=True)] + else: + layers += [conv2d, nn.ReLU(inplace=True)] + in_channels = v + return nn.Sequential(*layers) + + +cfg = { + 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], + 'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], # noqa + 'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], # noqa + 'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], # noqa +} + + +def vgg11(config, **kwargs): + """VGG 11-layer model (configuration "A") + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if config.pretrained: + kwargs['init_weights'] = False + model = VGG(make_layers(cfg['A']), **kwargs) + if config.pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['vgg11']), + strict=False) + return model + + +def vgg11_bn(config, **kwargs): + """VGG 11-layer model (configuration "A") with batch normalization + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if config.pretrained: + kwargs['init_weights'] = False + model = VGG(make_layers(cfg['A'], batch_norm=True, + norm_type=config.norm_type), **kwargs) + if config.pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['vgg11_bn']), + strict=False) + return model + + +def vgg13(config, **kwargs): + """VGG 13-layer model (configuration "B") + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if config.pretrained: + kwargs['init_weights'] = False + model = VGG(make_layers(cfg['B']), **kwargs) + if config.pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['vgg13']), + strict=False) + return model + + +def vgg13_bn(config, **kwargs): + """VGG 13-layer model (configuration "B") with batch normalization + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if config.pretrained: + kwargs['init_weights'] = False + model = VGG(make_layers(cfg['B'], batch_norm=True, + norm_type=config.norm_type), **kwargs) + if config.pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['vgg13_bn']), + strict=False) + return model + + +def vgg16(config, **kwargs): + """VGG 16-layer model (configuration "D") + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if config.pretrained: + kwargs['init_weights'] = False + model = VGG(make_layers(cfg['D']), **kwargs) + if config.pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['vgg16']), + strict=False) + return model + + +def vgg16_bn(config, **kwargs): + """VGG 16-layer model (configuration "D") with batch normalization + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if config.pretrained: + kwargs['init_weights'] = False + model = VGG(make_layers(cfg['D'], batch_norm=True, + norm_type=config.norm_type), **kwargs) + if config.pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['vgg16_bn']), + strict=False) + return model + + +def vgg19(config, **kwargs): + """VGG 19-layer model (configuration "E") + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if config.pretrained: + kwargs['init_weights'] = False + model = VGG(make_layers(cfg['E']), **kwargs) + if config.pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['vgg19']), + strict=False) + return model + + +def vgg19_bn(config, **kwargs): + """VGG 19-layer model (configuration 'E') with batch normalization + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + if config.pretrained: + kwargs['init_weights'] = False + model = VGG(make_layers(cfg['E'], batch_norm=True, + norm_type=config.norm_type), **kwargs) + if config.pretrained: + model.load_state_dict(model_zoo.load_url(model_urls['vgg19_bn']), + strict=False) + return model diff --git a/chexpert/project/model/classifier.py b/chexpert/project/model/classifier.py new file mode 100644 index 0000000..8a4bef8 --- /dev/null +++ b/chexpert/project/model/classifier.py @@ -0,0 +1,163 @@ +from torch import nn + +import torch.nn.functional as F +from model.backbone.vgg import (vgg19, vgg19_bn) +from model.backbone.densenet import (densenet121, densenet169, densenet201) +from model.backbone.inception import (inception_v3) +from model.global_pool import GlobalPool +from model.attention_map import AttentionMap + + +BACKBONES = {'vgg19': vgg19, + 'vgg19_bn': vgg19_bn, + 'densenet121': densenet121, + 'densenet169': densenet169, + 'densenet201': densenet201, + 'inception_v3': inception_v3} + + +BACKBONES_TYPES = {'vgg19': 'vgg', + 'vgg19_bn': 'vgg', + 'densenet121': 'densenet', + 'densenet169': 'densenet', + 'densenet201': 'densenet', + 'inception_v3': 'inception'} + + +class Classifier(nn.Module): + + def __init__(self, cfg): + super(Classifier, self).__init__() + self.cfg = cfg + self.backbone = BACKBONES[cfg.backbone](cfg) + self.global_pool = GlobalPool(cfg) + self.expand = 1 + if cfg.global_pool == 'AVG_MAX': + self.expand = 2 + elif cfg.global_pool == 'AVG_MAX_LSE': + self.expand = 3 + self._init_classifier() + self._init_bn() + self._init_attention_map() + + def _init_classifier(self): + for index, num_class in enumerate(self.cfg.num_classes): + if BACKBONES_TYPES[self.cfg.backbone] == 'vgg': + setattr( + self, + "fc_" + str(index), + nn.Conv2d( + 512 * self.expand, + num_class, + kernel_size=1, + stride=1, + padding=0, + bias=True)) + elif BACKBONES_TYPES[self.cfg.backbone] == 'densenet': + setattr( + self, + "fc_" + + str(index), + nn.Conv2d( + self.backbone.num_features * + self.expand, + num_class, + kernel_size=1, + stride=1, + padding=0, + bias=True)) + elif BACKBONES_TYPES[self.cfg.backbone] == 'inception': + setattr( + self, + "fc_" + str(index), + nn.Conv2d( + 2048 * self.expand, + num_class, + kernel_size=1, + stride=1, + padding=0, + bias=True)) + else: + raise Exception( + 'Unknown backbone type : {}'.format(self.cfg.backbone) + ) + + classifier = getattr(self, "fc_" + str(index)) + if isinstance(classifier, nn.Conv2d): + classifier.weight.data.normal_(0, 0.01) + classifier.bias.data.zero_() + + def _init_bn(self): + for index, num_class in enumerate(self.cfg.num_classes): + if BACKBONES_TYPES[self.cfg.backbone] == 'vgg': + setattr(self, "bn_" + str(index), + nn.BatchNorm2d(512 * self.expand)) + elif BACKBONES_TYPES[self.cfg.backbone] == 'densenet': + setattr( + self, + "bn_" + + str(index), + nn.BatchNorm2d( + self.backbone.num_features * + self.expand)) + elif BACKBONES_TYPES[self.cfg.backbone] == 'inception': + setattr(self, "bn_" + str(index), + nn.BatchNorm2d(2048 * self.expand)) + else: + raise Exception( + 'Unknown backbone type : {}'.format(self.cfg.backbone) + ) + + def _init_attention_map(self): + if BACKBONES_TYPES[self.cfg.backbone] == 'vgg': + setattr(self, "attention_map", AttentionMap(self.cfg, 512)) + elif BACKBONES_TYPES[self.cfg.backbone] == 'densenet': + setattr( + self, + "attention_map", + AttentionMap( + self.cfg, + self.backbone.num_features)) + elif BACKBONES_TYPES[self.cfg.backbone] == 'inception': + setattr(self, "attention_map", AttentionMap(self.cfg, 2048)) + else: + raise Exception( + 'Unknown backbone type : {}'.format(self.cfg.backbone) + ) + + def cuda(self, device=None): + return self._apply(lambda t: t.cuda(device)) + + def forward(self, x): + # (N, C, H, W) + feat_map = self.backbone(x) + # [(N, 1), (N,1),...] + logits = list() + # [(N, H, W), (N, H, W),...] + logit_maps = list() + for index, num_class in enumerate(self.cfg.num_classes): + if self.cfg.attention_map != "None": + feat_map = self.attention_map(feat_map) + + classifier = getattr(self, "fc_" + str(index)) + # (N, 1, H, W) + logit_map = None + if not (self.cfg.global_pool == 'AVG_MAX' or + self.cfg.global_pool == 'AVG_MAX_LSE'): + logit_map = classifier(feat_map) + logit_maps.append(logit_map.squeeze()) + # (N, C, 1, 1) + feat = self.global_pool(feat_map, logit_map) + + if self.cfg.fc_bn: + bn = getattr(self, "bn_" + str(index)) + feat = bn(feat) + feat = F.dropout(feat, p=self.cfg.fc_drop, training=self.training) + # (N, num_class, 1, 1) + + logit = classifier(feat) + # (N, num_class) + logit = logit.squeeze(-1).squeeze(-1) + logits.append(logit) + + return (logits, logit_maps) diff --git a/chexpert/project/model/global_pool.py b/chexpert/project/model/global_pool.py new file mode 100644 index 0000000..4ed7f4c --- /dev/null +++ b/chexpert/project/model/global_pool.py @@ -0,0 +1,154 @@ +import torch +from torch import nn + + +class PcamPool(nn.Module): + + def __init__(self): + super(PcamPool, self).__init__() + + def forward(self, feat_map, logit_map): + assert logit_map is not None + + prob_map = torch.sigmoid(logit_map) + weight_map = prob_map / prob_map.sum(dim=2, keepdim=True)\ + .sum(dim=3, keepdim=True) + feat = (feat_map * weight_map).sum(dim=2, keepdim=True)\ + .sum(dim=3, keepdim=True) + + return feat + + +class LogSumExpPool(nn.Module): + + def __init__(self, gamma): + super(LogSumExpPool, self).__init__() + self.gamma = gamma + + def forward(self, feat_map): + """ + Numerically stable implementation of the operation + Arguments: + feat_map(Tensor): tensor with shape (N, C, H, W) + return(Tensor): tensor with shape (N, C, 1, 1) + """ + (N, C, H, W) = feat_map.shape + + # (N, C, 1, 1) m + m, _ = torch.max( + feat_map, dim=-1, keepdim=True)[0].max(dim=-2, keepdim=True) + + # (N, C, H, W) value0 + value0 = feat_map - m + area = 1.0 / (H * W) + g = self.gamma + + # TODO: split dim=(-1, -2) for onnx.export + return m + 1 / g * torch.log(area * torch.sum( + torch.exp(g * value0), dim=(-1, -2), keepdim=True)) + + +class ExpPool(nn.Module): + + def __init__(self): + super(ExpPool, self).__init__() + + def forward(self, feat_map): + """ + Numerically stable implementation of the operation + Arguments: + feat_map(Tensor): tensor with shape (N, C, H, W) + return(Tensor): tensor with shape (N, C, 1, 1) + """ + + EPSILON = 1e-7 + (N, C, H, W) = feat_map.shape + m, _ = torch.max( + feat_map, dim=-1, keepdim=True)[0].max(dim=-2, keepdim=True) + + # caculate the sum of exp(xi) + # TODO: split dim=(-1, -2) for onnx.export + sum_exp = torch.sum(torch.exp(feat_map - m), + dim=(-1, -2), keepdim=True) + + # prevent from dividing by zero + sum_exp += EPSILON + + # caculate softmax in shape of (H,W) + exp_weight = torch.exp(feat_map - m) / sum_exp + weighted_value = feat_map * exp_weight + + # TODO: split dim=(-1, -2) for onnx.export + return torch.sum(weighted_value, dim=(-1, -2), keepdim=True) + + +class LinearPool(nn.Module): + + def __init__(self): + super(LinearPool, self).__init__() + + def forward(self, feat_map): + """ + Arguments: + feat_map(Tensor): tensor with shape (N, C, H, W) + return(Tensor): tensor with shape (N, C, 1, 1) + """ + EPSILON = 1e-7 + (N, C, H, W) = feat_map.shape + + # sum feat_map's last two dimention into a scalar + # so the shape of sum_input is (N,C,1,1) + # TODO: split dim=(-1, -2) for onnx.export + sum_input = torch.sum(feat_map, dim=(-1, -2), keepdim=True) + + # prevent from dividing by zero + sum_input += EPSILON + + # caculate softmax in shape of (H,W) + linear_weight = feat_map / sum_input + weighted_value = feat_map * linear_weight + + # TODO: split dim=(-1, -2) for onnx.export + return torch.sum(weighted_value, dim=(-1, -2), keepdim=True) + + +class GlobalPool(nn.Module): + + def __init__(self, cfg): + super(GlobalPool, self).__init__() + self.cfg = cfg + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.maxpool = nn.AdaptiveMaxPool2d((1, 1)) + self.exp_pool = ExpPool() + self.pcampool = PcamPool() + self.linear_pool = LinearPool() + self.lse_pool = LogSumExpPool(cfg.lse_gamma) + + def cuda(self, device=None): + return self._apply(lambda t: t.cuda(device)) + + def forward(self, feat_map, logit_map): + if self.cfg.global_pool == 'AVG': + return self.avgpool(feat_map) + elif self.cfg.global_pool == 'MAX': + return self.maxpool(feat_map) + elif self.cfg.global_pool == 'PCAM': + return self.pcampool(feat_map, logit_map) + elif self.cfg.global_pool == 'AVG_MAX': + a = self.avgpool(feat_map) + b = self.maxpool(feat_map) + return torch.cat((a, b), 1) + elif self.cfg.global_pool == 'AVG_MAX_LSE': + a = self.avgpool(feat_map) + b = self.maxpool(feat_map) + c = self.lse_pool(feat_map) + return torch.cat((a, b, c), 1) + elif self.cfg.global_pool == 'EXP': + return self.exp_pool(feat_map) + elif self.cfg.global_pool == 'LINEAR': + return self.linear_pool(feat_map) + elif self.cfg.global_pool == 'LSE': + return self.lse_pool(feat_map) + else: + raise Exception('Unknown pooling type : {}' + .format(self.cfg.global_pool)) diff --git a/chexpert/project/model/utils.py b/chexpert/project/model/utils.py new file mode 100644 index 0000000..22012df --- /dev/null +++ b/chexpert/project/model/utils.py @@ -0,0 +1,36 @@ +import torch.nn as nn +from torch.optim import SGD, Adadelta, Adagrad, Adam, RMSprop + + +def get_norm(norm_type, num_features, num_groups=32, eps=1e-5): + if norm_type == 'BatchNorm': + return nn.BatchNorm2d(num_features, eps=eps) + elif norm_type == "GroupNorm": + return nn.GroupNorm(num_groups, num_features, eps=eps) + elif norm_type == "InstanceNorm": + return nn.InstanceNorm2d(num_features, eps=eps, + affine=True, track_running_stats=True) + else: + raise Exception('Unknown Norm Function : {}'.format(norm_type)) + + +def get_optimizer(params, cfg): + if cfg.optimizer == 'SGD': + return SGD(params, lr=cfg.lr, momentum=cfg.momentum, + weight_decay=cfg.weight_decay) + elif cfg.optimizer == 'Adadelta': + return Adadelta(params, lr=cfg.lr, weight_decay=cfg.weight_decay) + elif cfg.optimizer == 'Adagrad': + return Adagrad(params, lr=cfg.lr, weight_decay=cfg.weight_decay) + elif cfg.optimizer == 'Adam': + return Adam(params, lr=cfg.lr, weight_decay=cfg.weight_decay) + elif cfg.optimizer == 'RMSprop': + return RMSprop(params, lr=cfg.lr, momentum=cfg.momentum, + weight_decay=cfg.weight_decay) + else: + raise Exception('Unknown optimizer : {}'.format(cfg.optimizer)) + + +def tensor2numpy(input_tensor): + # device cuda Tensor to host numpy + return input_tensor.cpu().detach().numpy() diff --git a/chexpert/project/requirements.txt b/chexpert/project/requirements.txt new file mode 100644 index 0000000..6b1af5f --- /dev/null +++ b/chexpert/project/requirements.txt @@ -0,0 +1,11 @@ +numpy==1.16.2 +matplotlib==3.0.3 +scikit-learn==0.20.3 +# tensorflow==1.15.4 +tensorboardX==1.6 +easydict==1.9 +opencv-python==4.0.0.21 +PyYAML +typer +torch +torchvision From d4449658e903bc915b9f3a4c91bdfc8e0b9f758e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Thu, 19 Aug 2021 21:06:07 -0500 Subject: [PATCH 03/11] Add preprocess task --- chexpert/mlcube/mlcube.yaml | 11 +++--- chexpert/project/mlcube.py | 59 +++++++++++++++++-------------- chexpert/project/preprocess.py | 25 +++++++++++++ chexpert/project/requirements.txt | 1 + 4 files changed, 64 insertions(+), 32 deletions(-) create mode 100644 chexpert/project/preprocess.py diff --git a/chexpert/mlcube/mlcube.yaml b/chexpert/mlcube/mlcube.yaml index 4307679..6aceb80 100644 --- a/chexpert/mlcube/mlcube.yaml +++ b/chexpert/mlcube/mlcube.yaml @@ -15,16 +15,15 @@ docker: build_file: "Dockerfile" tasks: - download_data: - # Download mnist dataset - parameters: - outputs: {data_dir: data/, log_dir: download_logs/} download_model: - # Train model + # Download model files parameters: outputs: {model_dir: model/} + preprocess: + parameters: + inputs: {data_dir: data/CheXpert-v1.0-small} infer: # evaluate model parameters: - inputs: {data_dir: data/, model_dir: model/} + inputs: {data_dir: data/CheXpert-v1.0-small, model_dir: model/} outputs: {log_dir: inference_logs/, out_dir: ./} \ No newline at end of file diff --git a/chexpert/project/mlcube.py b/chexpert/project/mlcube.py index e6a4557..8f2c08f 100644 --- a/chexpert/project/mlcube.py +++ b/chexpert/project/mlcube.py @@ -9,24 +9,12 @@ app = typer.Typer() -class DownloadDataTask(object): - """Download task Class - It defines the environment variables: - DATA_ROOT_DIR: Directory path to download the dataset - Then executes the download script""" - @staticmethod - def run(data_dir: str) -> None: - - command = f"python 01_download_dataset.py --data_dir {data_dir}" - splitted_command = command.split() - process = subprocess.Popen(splitted_command, cwd=".") - process.wait() - class DownloadModelTask(object): - """Preprocess dataset task Class - It defines the environment variables: - DATA_ROOT_DIR: Dataset directory path - Then executes the preprocess script""" + """ + Downloads model config and checkpoint files + Arguments: + - model_dir [str]: path for storing the model. + """ @staticmethod def run(model_dir: str) -> None: @@ -38,12 +26,31 @@ def run(model_dir: str) -> None: process = subprocess.Popen("./download_model.sh", cwd=".", env=env) process.wait() +class PreprocessTask(object): + """ + Task for preprocessing the data + + Arguments: + - data_dir: data location. + """ + @staticmethod + def run(data_dir: str) -> None: + cmd = f"python3.7 preprocess.py --data_dir={data_dir}" + splitted_cmd = cmd.split() + + process = subprocess.Popen(splitted_cmd, cwd=".") + process.wait() + class InferTask(object): - """Preprocess dataset task Class - It defines the environment variables: - DATA_DIR: Dataset directory path - All other parameters are defined in the parameters_file - Then executes the benchmark script""" + """ + Inference task for generating predictions on the CheXpert dataset. + + Arguments: + - log_dir [str]: logging location. + - data_dir [str]: data location. + - model_dir [str]: model location. + - out_dir [str]: location for storing the predictions. + """ @staticmethod def run(log_dir: str, data_dir: str, model_dir: str, out_dir) -> None: cmd = f"python3.7 chexpert.py --log_dir={log_dir} --data_dir={data_dir} --model_dir={model_dir} --out_dir={out_dir}" @@ -52,14 +59,14 @@ def run(log_dir: str, data_dir: str, model_dir: str, out_dir) -> None: process = subprocess.Popen(splitted_cmd, cwd=".") process.wait() -@app.command("download_data") -def download_data(data_dir: str = typer.Option(..., '--data_dir')): - DownloadDataTask.run(data_dir) - @app.command("download_model") def download_model(model_dir: str = typer.Option(..., '--model_dir')): DownloadModelTask.run(model_dir) +@app.command("preprocess") +def preprocess(data_dir: str = typer.Option(..., '--data_dir')): + PreprocessTask.run(data_dir) + @app.command("infer") def infer(log_dir: str = typer.Option(..., '--log_dir'), data_dir: str = typer.Option(..., '--data_dir'), diff --git a/chexpert/project/preprocess.py b/chexpert/project/preprocess.py new file mode 100644 index 0000000..4e6629f --- /dev/null +++ b/chexpert/project/preprocess.py @@ -0,0 +1,25 @@ +import pandas as pd +import os +import argparse + +class Preprocessor: + def __init__(self, data_dir): + self.data_csv_path = os.path.join(data_dir, 'valid.csv') + + def run(self): + df = pd.read_csv(self.data_csv_path) + img_path_lists = df['Path'].str.split('/') + + # Ensure the path has not been modified already + assert len(img_path_lists.iloc[0]) == 5, "Data has already been preprocessed" + + # Modify image path so that it is relative to the file location + df['Path'] = img_path_lists.str[1:].str.join('/') + df.to_csv(self.data_csv_path, index=False) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', '--data-dir', type=str, required=True, help='Location of chexpert dataset') + args = parser.parse_args() + preprocessor = Preprocessor(args.data_dir) + preprocessor.run() \ No newline at end of file diff --git a/chexpert/project/requirements.txt b/chexpert/project/requirements.txt index 6b1af5f..c340706 100644 --- a/chexpert/project/requirements.txt +++ b/chexpert/project/requirements.txt @@ -9,3 +9,4 @@ PyYAML typer torch torchvision +pandas From a0202539344ef775adfb573156371d661854fdb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Thu, 19 Aug 2021 21:06:23 -0500 Subject: [PATCH 04/11] Document data retrieval --- chexpert/README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/chexpert/README.md b/chexpert/README.md index 07a5fdc..e668581 100644 --- a/chexpert/README.md +++ b/chexpert/README.md @@ -18,11 +18,28 @@ git fetch origin pull/XX/head:chest-xray-example && git checkout chest-xray-exam cd ./chexpert ``` +## Get the data +Because the Chexpert Dataset contains sensitive information, signing an user agreement is required before obtaining the data. This means that we cannot automate the data download process. To obtain the dataset: + +1. sign up at the [Chexpert Dataset Download Agreement](https://stanfordmlgroup.github.io/competitions/chexpert/#agreement) and download the small dataset from the link sent to your email. +2. Unzip and place the `CheXpert-v1.0-small` folder inside `mlcube/workspace/data` folder. Your folder structure should look like this: + +``` +. +├── mlcube +│ └── workspace +│ └── Data +│ └── CheXpert-v1.0-small +│ ├── valid +│ └── valid.csv +└── project +``` + ## Run Chexpert MLCube on a local machine with Docker runner ``` # Run Chexpert training tasks: download data, train model and evaluate model -mlcube run --task download_data mlcube run --task download_model +mlcube run --task preprocess mlcube run --task infer ``` From a0c2084fb41bf9dee72523c5efcd7912a2fcba87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 20 Aug 2021 10:31:21 -0500 Subject: [PATCH 05/11] Add brief description on dataset and model used --- chexpert/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/chexpert/README.md b/chexpert/README.md index e668581..074a8da 100644 --- a/chexpert/README.md +++ b/chexpert/README.md @@ -1,4 +1,9 @@ # MLCube: Chexpert Example +This example demonstrates how to use MLCube to work with a computer vision model trained on the CheXpert Dataset. + +CheXpert is a large dataset of chest X-rays and competition for automated chest x-ray interpretation, which features uncertainty labels and radiologist-labeled reference standard evaluation sets. + +The model used here is based on the top 1 solution of the CheXpert challenge, which can be found [here](https://github.com/jfhealthcare/Chexpert). ### Project setup ```Python From 2d7fc315882dba5b561e52920fde8212139e744b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 20 Aug 2021 11:34:47 -0500 Subject: [PATCH 06/11] Specify PR number for cloning --- chexpert/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chexpert/README.md b/chexpert/README.md index 074a8da..5625e00 100644 --- a/chexpert/README.md +++ b/chexpert/README.md @@ -19,7 +19,7 @@ cd ./runners/mlcube_docker && python setup.py bdist_wheel && pip install --forc ## Clone MLCube examples and go to chexpert ``` git clone https://github.com/mlperf/mlcube_examples.git && cd ./mlcube_examples -git fetch origin pull/XX/head:chest-xray-example && git checkout chest-xray-example +git fetch origin pull/34/head:chest-xray-example && git checkout chest-xray-example cd ./chexpert ``` @@ -62,4 +62,4 @@ mlcube run ... -Pdocker.build_strategy=auto By default, at the end of the download_model task, Chexpert model will be saved in `workspace/model`. -By default, at the end of the infer task, results will be saved in `workspace/inferences.txt`. \ No newline at end of file +By default, at the end of the infer task, results will be saved in `workspace/inferences.txt`. From 99ad819041b7baa19f25e201e1cfdd7ff675f844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 20 Aug 2021 14:42:38 -0500 Subject: [PATCH 07/11] Update mlcube.yaml --- chexpert/mlcube/mlcube.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chexpert/mlcube/mlcube.yaml b/chexpert/mlcube/mlcube.yaml index 6aceb80..afd403d 100644 --- a/chexpert/mlcube/mlcube.yaml +++ b/chexpert/mlcube/mlcube.yaml @@ -23,7 +23,7 @@ tasks: parameters: inputs: {data_dir: data/CheXpert-v1.0-small} infer: - # evaluate model + # predict on data parameters: inputs: {data_dir: data/CheXpert-v1.0-small, model_dir: model/} - outputs: {log_dir: inference_logs/, out_dir: ./} \ No newline at end of file + outputs: {log_dir: inference_logs/, out_dir: ./} From 5e1ab7b09bfd10d21a45ed471d5ab7b0570cc460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 23 Aug 2021 11:42:14 -0500 Subject: [PATCH 08/11] Update README.md --- chexpert/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chexpert/README.md b/chexpert/README.md index 5625e00..880647b 100644 --- a/chexpert/README.md +++ b/chexpert/README.md @@ -42,7 +42,7 @@ Because the Chexpert Dataset contains sensitive information, signing an user agr ## Run Chexpert MLCube on a local machine with Docker runner ``` -# Run Chexpert training tasks: download data, train model and evaluate model +# Run Chexpert training tasks: download data, download the model and generate predictions mlcube run --task download_model mlcube run --task preprocess mlcube run --task infer From 42860564bd10f336f2e182915d7856135090c568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 23 Aug 2021 18:49:56 -0500 Subject: [PATCH 09/11] Make output path consistent with input data --- chexpert/project/data/dataset.py | 52 +++++++++++++++++++------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/chexpert/project/data/dataset.py b/chexpert/project/data/dataset.py index 70cba93..db40f20 100644 --- a/chexpert/project/data/dataset.py +++ b/chexpert/project/data/dataset.py @@ -5,50 +5,59 @@ from PIL import Image from data.imgaug import GetTransforms from data.utils import transform + np.random.seed(0) class ImageDataset(Dataset): - def __init__(self, label_path, cfg, data_path, mode='train'): + def __init__(self, label_path, cfg, data_path, mode="train"): self.cfg = cfg + self.data_path = data_path self._label_header = None self._image_paths = [] self._labels = [] self._mode = mode - self.dict = [{'1.0': '1', '': '0', '0.0': '0', '-1.0': '0'}, - {'1.0': '1', '': '0', '0.0': '0', '-1.0': '1'}, ] + self.dict = [ + {"1.0": "1", "": "0", "0.0": "0", "-1.0": "0"}, + {"1.0": "1", "": "0", "0.0": "0", "-1.0": "1"}, + ] with open(label_path) as f: - header = f.readline().strip('\n').split(',') + header = f.readline().strip("\n").split(",") self._label_header = [ header[7], header[10], header[11], header[13], - header[15]] + header[15], + ] for line in f: labels = [] - fields = line.strip('\n').split(',') - image_path = os.path.join(data_path, fields[0]) + fields = line.strip("\n").split(",") + image_path = fields[0] + image_root_path = os.path.join(self.data_path, image_path) + # image_path = os.path.join(data_path, fields[0]) flg_enhance = False for index, value in enumerate(fields[5:]): if index == 5 or index == 8: labels.append(self.dict[1].get(value)) - if self.dict[1].get( - value) == '1' and \ - self.cfg.enhance_index.count(index) > 0: + if ( + self.dict[1].get(value) == "1" + and self.cfg.enhance_index.count(index) > 0 + ): flg_enhance = True elif index == 2 or index == 6 or index == 10: labels.append(self.dict[0].get(value)) - if self.dict[0].get( - value) == '1' and \ - self.cfg.enhance_index.count(index) > 0: + if ( + self.dict[0].get(value) == "1" + and self.cfg.enhance_index.count(index) > 0 + ): flg_enhance = True # labels = ([self.dict.get(n, n) for n in fields[5:]]) labels = list(map(int, labels)) self._image_paths.append(image_path) - assert os.path.exists(image_path), image_path + assert os.path.exists(image_root_path), image_path self._labels.append(labels) - if flg_enhance and self._mode == 'train': + if flg_enhance and self._mode == "train": for i in range(self.cfg.enhance_times): self._image_paths.append(image_path) self._labels.append(labels) @@ -58,9 +67,10 @@ def __len__(self): return self._num_image def __getitem__(self, idx): - image = cv2.imread(self._image_paths[idx], 0) + image_root_path = os.path.join(self.data_path, self._image_paths[idx]) + image = cv2.imread(image_root_path, 0) image = Image.fromarray(image) - if self._mode == 'train': + if self._mode == "train": image = GetTransforms(image, type=self.cfg.use_transforms_type) image = np.array(image) image = transform(image, self.cfg) @@ -68,11 +78,11 @@ def __getitem__(self, idx): path = self._image_paths[idx] - if self._mode == 'train' or self._mode == 'dev': + if self._mode == "train" or self._mode == "dev": return (image, labels) - elif self._mode == 'test': + elif self._mode == "test": return (image, path) - elif self._mode == 'heatmap': + elif self._mode == "heatmap": return (image, path, labels) else: - raise Exception('Unknown mode : {}'.format(self._mode)) + raise Exception("Unknown mode : {}".format(self._mode)) From 78e6b68a344ca3f19de51ec5f34e77a7714cfaa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 23 Aug 2021 18:52:40 -0500 Subject: [PATCH 10/11] Fix typo --- chexpert/mlcube/mlcube.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chexpert/mlcube/mlcube.yaml b/chexpert/mlcube/mlcube.yaml index afd403d..be6fefa 100644 --- a/chexpert/mlcube/mlcube.yaml +++ b/chexpert/mlcube/mlcube.yaml @@ -1,5 +1,5 @@ name: MLCommons Chexpert -description: MLCommons Chexpert examplefor inference with the Chexpert model. +description: MLCommons Chexpert example for inference with the Chexpert model. authors: - {name: "MLCommons Best Practices Working Group"} From fd43b3e008a385c872e3c387f38b214b88b97748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 24 Aug 2021 19:08:07 -0500 Subject: [PATCH 11/11] Add tqdm for inference task. Remove unused code --- chexpert/project/chexpert.py | 116 +++++++++++++++--------------- chexpert/project/requirements.txt | 1 + 2 files changed, 57 insertions(+), 60 deletions(-) diff --git a/chexpert/project/chexpert.py b/chexpert/project/chexpert.py index ae3bb62..793b414 100644 --- a/chexpert/project/chexpert.py +++ b/chexpert/project/chexpert.py @@ -6,6 +6,7 @@ import logging.config import json import time +from tqdm import tqdm from enum import Enum from typing import List from easydict import EasyDict as edict @@ -22,40 +23,18 @@ logger = logging.getLogger(__name__) + class Task(str, Enum): - DownloadData = 'download_data' - DownloadCkpt = 'download_ckpt' - Infer = 'infer' + DownloadData = "download_data" + DownloadCkpt = "download_ckpt" + Infer = "infer" + def create_directory(path: str) -> None: if not os.path.exists(path): os.makedirs(path, exist_ok=True) -def download_data(task_args: List[str]) -> None: - """ Task: download_data. - - Input parameters: - --data_dir - """ - logger.info(f"Starting '{Task.DownloadData}' task") - parser = argparse.ArgumentParser() - parser.add_argument('--data_dir', '--data-dir', type=str, default=None, help="Path to a dataset file.") - args = parser.parse_args(args=task_args) - - if args.data_dir is None: - raise ValueError("Data directory is not specified (did you use --data-dir=PATH?)") - os.makedirs(args.data_dir, exist_ok=True) - - if not args.data_dir.startswith("/"): - logger.warning("Data directory seems to be a relative path.") - - # TODO: check if data has already been downloaded. If so, return - - # TODO: actually download the data - return - - def infer(task_args: List[str]) -> None: """ Task: infer @@ -63,26 +42,32 @@ def infer(task_args: List[str]) -> None: --data_dir, --ckpt_dir, --out_dir """ parser = argparse.ArgumentParser() - parser.add_argument('--data_dir', '--data-dir', type=str, default=None, help="Dataset path.") - parser.add_argument('--model_dir', '--model-dir', type=str, default=None, help="Model location.") - parser.add_argument('--out_dir', '--out-dir', type=str, default=None, help="Model output directory.") + parser.add_argument( + "--data_dir", "--data-dir", type=str, default=None, help="Dataset path." + ) + parser.add_argument( + "--model_dir", "--model-dir", type=str, default=None, help="Model location." + ) + parser.add_argument( + "--out_dir", "--out-dir", type=str, default=None, help="Model output directory." + ) args = parser.parse_args(args=task_args) run(args) def get_pred(output, cfg): - if cfg.criterion == 'BCE' or cfg.criterion == "FL": + if cfg.criterion == "BCE" or cfg.criterion == "FL": for num_class in cfg.num_classes: assert num_class == 1 pred = torch.sigmoid(output.view(-1)).cpu().detach().numpy() - elif cfg.criterion == 'CE': + elif cfg.criterion == "CE": for num_class in cfg.num_classes: assert num_class >= 2 prob = F.softmax(output) pred = prob[:, 1].cpu().detach().numpy() else: - raise Exception('Unknown criterion : {}'.format(cfg.criterion)) + raise Exception("Unknown criterion : {}".format(cfg.criterion)) return pred @@ -94,16 +79,17 @@ def test_epoch(cfg, model, device, dataloader, out_csv_path): num_tasks = len(cfg.num_classes) test_header = [ - 'Path', - 'Cardiomegaly', - 'Edema', - 'Consolidation', - 'Atelectasis', - 'Pleural Effusion'] - - with open(out_csv_path, 'w') as f: - f.write(','.join(test_header) + '\n') - for step in range(steps): + "Path", + "Cardiomegaly", + "Edema", + "Consolidation", + "Atelectasis", + "Pleural Effusion", + ] + + with open(out_csv_path, "w") as f: + f.write(",".join(test_header) + "\n") + for step in tqdm(range(steps)): image, path = next(dataiter) image = image.to(device) output, __ = model(image) @@ -114,21 +100,24 @@ def test_epoch(cfg, model, device, dataloader, out_csv_path): pred[i] = get_pred(output[i], cfg) for i in range(batch_size): - batch = ','.join(map(lambda x: '{}'.format(x), pred[:, i])) - result = path[i] + ',' + batch - f.write(result + '\n') - logging.info('{}, Image : {}, Prob : {}'.format( - time.strftime("%Y-%m-%d %H:%M:%S"), path[i], batch)) + batch = ",".join(map(lambda x: "{}".format(x), pred[:, i])) + result = path[i] + "," + batch + f.write(result + "\n") + logging.info( + "{}, Image : {}, Prob : {}".format( + time.strftime("%Y-%m-%d %H:%M:%S"), path[i], batch + ) + ) def run(args): - ckpt_path = os.path.join(args.model_dir, 'model.pth') - config_path = os.path.join(args.model_dir, 'config.json') + ckpt_path = os.path.join(args.model_dir, "model.pth") + config_path = os.path.join(args.model_dir, "config.json") print(config_path) with open(config_path) as f: cfg = edict(json.load(f)) - device = torch.device('cuda' if torch.cuda.is_available() else "cpu") + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") ckpt = torch.load(ckpt_path, map_location=device) model = Classifier(cfg).to(device).eval() @@ -138,10 +127,11 @@ def run(args): in_csv_path = os.path.join(args.data_dir, "valid.csv") dataloader_test = DataLoader( - ImageDataset(in_csv_path, cfg, args.data_dir, mode='test'), + ImageDataset(in_csv_path, cfg, args.data_dir, mode="test"), batch_size=cfg.dev_batch_size, - drop_last=False, shuffle=False) - + drop_last=False, + shuffle=False, + ) test_epoch(cfg, model, device, dataloader_test, out_csv_path) @@ -151,7 +141,9 @@ def main(): chexpert.py task task_specific_parameters... """ parser = argparse.ArgumentParser() - parser.add_argument('--log_dir', '--log-dir', type=str, required=True, help="Logging directory.") + parser.add_argument( + "--log_dir", "--log-dir", type=str, required=True, help="Logging directory." + ) mlcube_args, task_args = parser.parse_known_args() os.makedirs(mlcube_args.log_dir, exist_ok=True) @@ -159,25 +151,29 @@ def main(): "version": 1, "disable_existing_loggers": True, "formatters": { - "standard": {"format": "%(asctime)s - %(name)s - %(threadName)s - %(levelname)s - %(message)s"}, + "standard": { + "format": "%(asctime)s - %(name)s - %(threadName)s - %(levelname)s - %(message)s" + }, }, "handlers": { "file_handler": { "class": "logging.FileHandler", "level": "INFO", "formatter": "standard", - "filename": os.path.join(mlcube_args.log_dir, f"mlcube_chexpert_infer.log") + "filename": os.path.join( + mlcube_args.log_dir, f"mlcube_chexpert_infer.log" + ), } }, "loggers": { "": {"level": "INFO", "handlers": ["file_handler"]}, "__main__": {"level": "NOTSET", "propagate": "yes"}, - "tensorflow": {"level": "NOTSET", "propagate": "yes"} - } + "tensorflow": {"level": "NOTSET", "propagate": "yes"}, + }, } logging.config.dictConfig(logger_config) infer(task_args) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/chexpert/project/requirements.txt b/chexpert/project/requirements.txt index c340706..84cb1fb 100644 --- a/chexpert/project/requirements.txt +++ b/chexpert/project/requirements.txt @@ -10,3 +10,4 @@ typer torch torchvision pandas +tqdm