From eb8b104f9b3efdb3cdce137f47c6889454cfac92 Mon Sep 17 00:00:00 2001 From: navneeth Date: Wed, 1 Jul 2020 16:33:13 +0900 Subject: [PATCH 1/3] 1. Normalization parameters of dataset in common location 2. Replacing explicit .cuda() with device selection code (.device()) --- evaluation.py | 322 ++++++++++++++++++++++++------------------------ explanations.py | 203 +++++++++++++++--------------- utils.py | 127 ++++++++++--------- 3 files changed, 326 insertions(+), 326 deletions(-) diff --git a/evaluation.py b/evaluation.py index 34ed5de..497e770 100644 --- a/evaluation.py +++ b/evaluation.py @@ -1,161 +1,161 @@ -from torch import nn -from tqdm import tqdm -from scipy.ndimage.filters import gaussian_filter - -from utils import * - -HW = 224 * 224 # image area -n_classes = 1000 - -def gkern(klen, nsig): - """Returns a Gaussian kernel array. - Convolution with it results in image blurring.""" - # create nxn zeros - inp = np.zeros((klen, klen)) - # set element at the middle to one, a dirac delta - inp[klen//2, klen//2] = 1 - # gaussian-smooth the dirac, resulting in a gaussian filter mask - k = gaussian_filter(inp, nsig) - kern = np.zeros((3, 3, klen, klen)) - kern[0, 0] = k - kern[1, 1] = k - kern[2, 2] = k - return torch.from_numpy(kern.astype('float32')) - -def auc(arr): - """Returns normalized Area Under Curve of the array.""" - return (arr.sum() - arr[0] / 2 - arr[-1] / 2) / (arr.shape[0] - 1) - -class CausalMetric(): - - def __init__(self, model, mode, step, substrate_fn): - r"""Create deletion/insertion metric instance. - - Args: - model (nn.Module): Black-box model being explained. - mode (str): 'del' or 'ins'. - step (int): number of pixels modified per one iteration. - substrate_fn (func): a mapping from old pixels to new pixels. - """ - assert mode in ['del', 'ins'] - self.model = model - self.mode = mode - self.step = step - self.substrate_fn = substrate_fn - - def single_run(self, img_tensor, explanation, verbose=0, save_to=None): - r"""Run metric on one image-saliency pair. - - Args: - img_tensor (Tensor): normalized image tensor. - explanation (np.ndarray): saliency map. - verbose (int): in [0, 1, 2]. - 0 - return list of scores. - 1 - also plot final step. - 2 - also plot every step and print 2 top classes. - save_to (str): directory to save every step plots to. - - Return: - scores (nd.array): Array containing scores at every step. - """ - pred = self.model(img_tensor.cuda()) - top, c = torch.max(pred, 1) - c = c.cpu().numpy()[0] - n_steps = (HW + self.step - 1) // self.step - - if self.mode == 'del': - title = 'Deletion game' - ylabel = 'Pixels deleted' - start = img_tensor.clone() - finish = self.substrate_fn(img_tensor) - elif self.mode == 'ins': - title = 'Insertion game' - ylabel = 'Pixels inserted' - start = self.substrate_fn(img_tensor) - finish = img_tensor.clone() - - scores = np.empty(n_steps + 1) - # Coordinates of pixels in order of decreasing saliency - salient_order = np.flip(np.argsort(explanation.reshape(-1, HW), axis=1), axis=-1) - for i in range(n_steps+1): - pred = self.model(start.cuda()) - pr, cl = torch.topk(pred, 2) - if verbose == 2: - print('{}: {:.3f}'.format(get_class_name(cl[0][0]), float(pr[0][0]))) - print('{}: {:.3f}'.format(get_class_name(cl[0][1]), float(pr[0][1]))) - scores[i] = pred[0, c] - # Render image if verbose, if it's the last step or if save is required. - if verbose == 2 or (verbose == 1 and i == n_steps) or save_to: - plt.figure(figsize=(10, 5)) - plt.subplot(121) - plt.title('{} {:.1f}%, P={:.4f}'.format(ylabel, 100 * i / n_steps, scores[i])) - plt.axis('off') - tensor_imshow(start[0]) - - plt.subplot(122) - plt.plot(np.arange(i+1) / n_steps, scores[:i+1]) - plt.xlim(-0.1, 1.1) - plt.ylim(0, 1.05) - plt.fill_between(np.arange(i+1) / n_steps, 0, scores[:i+1], alpha=0.4) - plt.title(title) - plt.xlabel(ylabel) - plt.ylabel(get_class_name(c)) - if save_to: - plt.savefig(save_to + '/{:03d}.png'.format(i)) - plt.close() - else: - plt.show() - if i < n_steps: - coords = salient_order[:, self.step * i:self.step * (i + 1)] - start.cpu().numpy().reshape(1, 3, HW)[0, :, coords] = finish.cpu().numpy().reshape(1, 3, HW)[0, :, coords] - return scores - - def evaluate(self, img_batch, exp_batch, batch_size): - r"""Efficiently evaluate big batch of images. - - Args: - img_batch (Tensor): batch of images. - exp_batch (np.ndarray): batch of explanations. - batch_size (int): number of images for one small batch. - - Returns: - scores (nd.array): Array containing scores at every step for every image. - """ - n_samples = img_batch.shape[0] - predictions = torch.FloatTensor(n_samples, n_classes) - assert n_samples % batch_size == 0 - for i in tqdm(range(n_samples // batch_size), desc='Predicting labels'): - preds = self.model(img_batch[i*batch_size:(i+1)*batch_size].cuda()).cpu() - predictions[i*batch_size:(i+1)*batch_size] = preds - top = np.argmax(predictions, -1) - n_steps = (HW + self.step - 1) // self.step - scores = np.empty((n_steps + 1, n_samples)) - salient_order = np.flip(np.argsort(exp_batch.reshape(-1, HW), axis=1), axis=-1) - r = np.arange(n_samples).reshape(n_samples, 1) - - substrate = torch.zeros_like(img_batch) - for j in tqdm(range(n_samples // batch_size), desc='Substrate'): - substrate[j*batch_size:(j+1)*batch_size] = self.substrate_fn(img_batch[j*batch_size:(j+1)*batch_size]) - - if self.mode == 'del': - caption = 'Deleting ' - start = img_batch.clone() - finish = substrate - elif self.mode == 'ins': - caption = 'Inserting ' - start = substrate - finish = img_batch.clone() - - # While not all pixels are changed - for i in tqdm(range(n_steps+1), desc=caption + 'pixels'): - # Iterate over batches - for j in range(n_samples // batch_size): - # Compute new scores - preds = self.model(start[j*batch_size:(j+1)*batch_size].cuda()) - preds = preds.cpu().numpy()[range(batch_size), top[j*batch_size:(j+1)*batch_size]] - scores[i, j*batch_size:(j+1)*batch_size] = preds - # Change specified number of most salient pixels to substrate pixels - coords = salient_order[:, self.step * i:self.step * (i + 1)] - start.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] = finish.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] - print('AUC: {}'.format(auc(scores.mean(1)))) - return scores +from torch import nn +from tqdm import tqdm +from scipy.ndimage.filters import gaussian_filter + +from utils import * + +HW = 224 * 224 # image area +n_classes = 1000 + +def gkern(klen, nsig): + """Returns a Gaussian kernel array. + Convolution with it results in image blurring.""" + # create nxn zeros + inp = np.zeros((klen, klen)) + # set element at the middle to one, a dirac delta + inp[klen//2, klen//2] = 1 + # gaussian-smooth the dirac, resulting in a gaussian filter mask + k = gaussian_filter(inp, nsig) + kern = np.zeros((3, 3, klen, klen)) + kern[0, 0] = k + kern[1, 1] = k + kern[2, 2] = k + return torch.from_numpy(kern.astype('float32')) + +def auc(arr): + """Returns normalized Area Under Curve of the array.""" + return (arr.sum() - arr[0] / 2 - arr[-1] / 2) / (arr.shape[0] - 1) + +class CausalMetric(): + + def __init__(self, model, mode, step, substrate_fn): + r"""Create deletion/insertion metric instance. + + Args: + model (nn.Module): Black-box model being explained. + mode (str): 'del' or 'ins'. + step (int): number of pixels modified per one iteration. + substrate_fn (func): a mapping from old pixels to new pixels. + """ + assert mode in ['del', 'ins'] + self.model = model + self.mode = mode + self.step = step + self.substrate_fn = substrate_fn + + def single_run(self, img_tensor, explanation, verbose=0, save_to=None): + r"""Run metric on one image-saliency pair. + + Args: + img_tensor (Tensor): normalized image tensor. + explanation (np.ndarray): saliency map. + verbose (int): in [0, 1, 2]. + 0 - return list of scores. + 1 - also plot final step. + 2 - also plot every step and print 2 top classes. + save_to (str): directory to save every step plots to. + + Return: + scores (nd.array): Array containing scores at every step. + """ + pred = self.model(img_tensor.cuda()) + top, c = torch.max(pred, 1) + c = c.cpu().numpy()[0] + n_steps = (HW + self.step - 1) // self.step + + if self.mode == 'del': + title = 'Deletion game' + ylabel = 'Pixels deleted' + start = img_tensor.clone() + finish = self.substrate_fn(img_tensor) + elif self.mode == 'ins': + title = 'Insertion game' + ylabel = 'Pixels inserted' + start = self.substrate_fn(img_tensor) + finish = img_tensor.clone() + + scores = np.empty(n_steps + 1) + # Coordinates of pixels in order of decreasing saliency + salient_order = np.flip(np.argsort(explanation.reshape(-1, HW), axis=1), axis=-1) + for i in range(n_steps+1): + pred = self.model(start.cuda()) + pr, cl = torch.topk(pred, 2) + if verbose == 2: + print('{}: {:.3f}'.format(get_class_name(cl[0][0]), float(pr[0][0]))) + print('{}: {:.3f}'.format(get_class_name(cl[0][1]), float(pr[0][1]))) + scores[i] = pred[0, c] + # Render image if verbose, if it's the last step or if save is required. + if verbose == 2 or (verbose == 1 and i == n_steps) or save_to: + plt.figure(figsize=(10, 5)) + plt.subplot(121) + plt.title('{} {:.1f}%, P={:.4f}'.format(ylabel, 100 * i / n_steps, scores[i])) + plt.axis('off') + tensor_imshow(start[0]) + + plt.subplot(122) + plt.plot(np.arange(i+1) / n_steps, scores[:i+1]) + plt.xlim(-0.1, 1.1) + plt.ylim(0, 1.05) + plt.fill_between(np.arange(i+1) / n_steps, 0, scores[:i+1], alpha=0.4) + plt.title(title) + plt.xlabel(ylabel) + plt.ylabel(get_class_name(c)) + if save_to: + plt.savefig(save_to + '/{:03d}.png'.format(i)) + plt.close() + else: + plt.show() + if i < n_steps: + coords = salient_order[:, self.step * i:self.step * (i + 1)] + start.cpu().numpy().reshape(1, 3, HW)[0, :, coords] = finish.cpu().numpy().reshape(1, 3, HW)[0, :, coords] + return scores + + def evaluate(self, img_batch, exp_batch, batch_size): + r"""Efficiently evaluate big batch of images. + + Args: + img_batch (Tensor): batch of images. + exp_batch (np.ndarray): batch of explanations. + batch_size (int): number of images for one small batch. + + Returns: + scores (nd.array): Array containing scores at every step for every image. + """ + n_samples = img_batch.shape[0] + predictions = torch.FloatTensor(n_samples, n_classes) + assert n_samples % batch_size == 0 + for i in tqdm(range(n_samples // batch_size), desc='Predicting labels'): + preds = self.model(img_batch[i*batch_size:(i+1)*batch_size].cuda()).cpu() + predictions[i*batch_size:(i+1)*batch_size] = preds + top = np.argmax(predictions, -1) + n_steps = (HW + self.step - 1) // self.step + scores = np.empty((n_steps + 1, n_samples)) + salient_order = np.flip(np.argsort(exp_batch.reshape(-1, HW), axis=1), axis=-1) + r = np.arange(n_samples).reshape(n_samples, 1) + + substrate = torch.zeros_like(img_batch) + for j in tqdm(range(n_samples // batch_size), desc='Substrate'): + substrate[j*batch_size:(j+1)*batch_size] = self.substrate_fn(img_batch[j*batch_size:(j+1)*batch_size]) + + if self.mode == 'del': + caption = 'Deleting ' + start = img_batch.clone() + finish = substrate + elif self.mode == 'ins': + caption = 'Inserting ' + start = substrate + finish = img_batch.clone() + + # While not all pixels are changed + for i in tqdm(range(n_steps+1), desc=caption + 'pixels'): + # Iterate over batches + for j in range(n_samples // batch_size): + # Compute new scores + preds = self.model(start[j*batch_size:(j+1)*batch_size].cuda()) + preds = preds.cpu().numpy()[range(batch_size), top[j*batch_size:(j+1)*batch_size]] + scores[i, j*batch_size:(j+1)*batch_size] = preds + # Change specified number of most salient pixels to substrate pixels + coords = salient_order[:, self.step * i:self.step * (i + 1)] + start.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] = finish.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] + print('AUC: {}'.format(auc(scores.mean(1)))) + return scores diff --git a/explanations.py b/explanations.py index 47cb5f8..bd8139d 100644 --- a/explanations.py +++ b/explanations.py @@ -1,100 +1,103 @@ -import numpy as np -import torch -import torch.nn as nn -from skimage.transform import resize -from tqdm import tqdm - - -class RISE(nn.Module): - def __init__(self, model, input_size, gpu_batch=100): - super(RISE, self).__init__() - self.model = model - self.input_size = input_size - self.gpu_batch = gpu_batch - - def generate_masks(self, N, s, p1, savepath='masks.npy'): - cell_size = np.ceil(np.array(self.input_size) / s) - up_size = (s + 1) * cell_size - - grid = np.random.rand(N, s, s) < p1 - grid = grid.astype('float32') - - self.masks = np.empty((N, *self.input_size)) - - for i in tqdm(range(N), desc='Generating filters'): - # Random shifts - x = np.random.randint(0, cell_size[0]) - y = np.random.randint(0, cell_size[1]) - # Linear upsampling and cropping - self.masks[i, :, :] = resize(grid[i], up_size, order=1, mode='reflect', - anti_aliasing=False)[x:x + self.input_size[0], y:y + self.input_size[1]] - self.masks = self.masks.reshape(-1, 1, *self.input_size) - np.save(savepath, self.masks) - self.masks = torch.from_numpy(self.masks).float() - self.masks = self.masks.cuda() - self.N = N - self.p1 = p1 - - def load_masks(self, filepath): - self.masks = np.load(filepath) - self.masks = torch.from_numpy(self.masks).float().cuda() - self.N = self.masks.shape[0] - - def forward(self, x): - N = self.N - _, _, H, W = x.size() - # Apply array of filters to the image - stack = torch.mul(self.masks, x.data) - - # p = nn.Softmax(dim=1)(model(stack)) processed in batches - p = [] - for i in range(0, N, self.gpu_batch): - p.append(self.model(stack[i:min(i + self.gpu_batch, N)])) - p = torch.cat(p) - # Number of classes - CL = p.size(1) - sal = torch.matmul(p.data.transpose(0, 1), self.masks.view(N, H * W)) - sal = sal.view((CL, H, W)) - sal = sal / N / self.p1 - return sal - - -class RISEBatch(RISE): - def forward(self, x): - # Apply array of filters to the image - N = self.N - B, C, H, W = x.size() - stack = torch.mul(self.masks.view(N, 1, H, W), x.data.view(B * C, H, W)) - stack = stack.view(B * N, C, H, W) - stack = stack - - #p = nn.Softmax(dim=1)(model(stack)) in batches - p = [] - for i in range(0, N*B, self.gpu_batch): - p.append(self.model(stack[i:min(i + self.gpu_batch, N*B)])) - p = torch.cat(p) - CL = p.size(1) - p = p.view(N, B, CL) - sal = torch.matmul(p.permute(1, 2, 0), self.masks.view(N, H * W)) - sal = sal.view(B, CL, H, W) - return sal - -# To process in batches -# def explain_all_batch(data_loader, explainer): -# n_batch = len(data_loader) -# b_size = data_loader.batch_size -# total = n_batch * b_size -# # Get all predicted labels first -# target = np.empty(total, 'int64') -# for i, (imgs, _) in enumerate(tqdm(data_loader, total=n_batch, desc='Predicting labels')): -# p, c = torch.max(nn.Softmax(1)(explainer.model(imgs.cuda())), dim=1) -# target[i * b_size:(i + 1) * b_size] = c -# image_size = imgs.shape[-2:] -# -# # Get saliency maps for all images in val loader -# explanations = np.empty((total, *image_size)) -# for i, (imgs, _) in enumerate(tqdm(data_loader, total=n_batch, desc='Explaining images')): -# saliency_maps = explainer(imgs.cuda()) -# explanations[i * b_size:(i + 1) * b_size] = saliency_maps[ -# range(b_size), target[i * b_size:(i + 1) * b_size]].data.cpu().numpy() -# return explanations +import numpy as np +import torch +import torch.nn as nn +from skimage.transform import resize +from tqdm import tqdm + +device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') +print('Using device :', device) + +class RISE(nn.Module): + def __init__(self, model, input_size, gpu_batch=100): + super(RISE, self).__init__() + self.model = model + self.input_size = input_size + self.gpu_batch = gpu_batch + + def generate_masks(self, N, s, p1, savepath='masks.npy'): + cell_size = np.ceil(np.array(self.input_size) / s) + up_size = (s + 1) * cell_size + + grid = np.random.rand(N, s, s) < p1 + grid = grid.astype('float32') + + self.masks = np.empty((N, *self.input_size)) + + for i in tqdm(range(N), desc='Generating filters'): + # Random shifts + x = np.random.randint(0, cell_size[0]) + y = np.random.randint(0, cell_size[1]) + # Linear upsampling and cropping + self.masks[i, :, :] = resize(grid[i], up_size, order=1, mode='reflect', + anti_aliasing=False)[x:x + self.input_size[0], y:y + self.input_size[1]] + self.masks = self.masks.reshape(-1, 1, *self.input_size) + np.save(savepath, self.masks) + self.masks = torch.from_numpy(self.masks).float() + self.masks = self.masks.to(device) + self.N = N + self.p1 = p1 + + def load_masks(self, filepath): + self.masks = np.load(filepath) + self.masks = torch.from_numpy(self.masks).float() + self.masks = self.masks.to(device) + self.N = self.masks.shape[0] + + def forward(self, x): + N = self.N + _, _, H, W = x.size() + # Apply array of filters to the image + stack = torch.mul(self.masks, x.data) + + # p = nn.Softmax(dim=1)(model(stack)) processed in batches + p = [] + for i in range(0, N, self.gpu_batch): + p.append(self.model(stack[i:min(i + self.gpu_batch, N)])) + p = torch.cat(p) + # Number of classes + CL = p.size(1) + sal = torch.matmul(p.data.transpose(0, 1), self.masks.view(N, H * W)) + sal = sal.view((CL, H, W)) + sal = sal / N / self.p1 + return sal + + +class RISEBatch(RISE): + def forward(self, x): + # Apply array of filters to the image + N = self.N + B, C, H, W = x.size() + stack = torch.mul(self.masks.view(N, 1, H, W), x.data.view(B * C, H, W)) + stack = stack.view(B * N, C, H, W) + stack = stack + + #p = nn.Softmax(dim=1)(model(stack)) in batches + p = [] + for i in range(0, N*B, self.gpu_batch): + p.append(self.model(stack[i:min(i + self.gpu_batch, N*B)])) + p = torch.cat(p) + CL = p.size(1) + p = p.view(N, B, CL) + sal = torch.matmul(p.permute(1, 2, 0), self.masks.view(N, H * W)) + sal = sal.view(B, CL, H, W) + return sal + +# To process in batches +# def explain_all_batch(data_loader, explainer): +# n_batch = len(data_loader) +# b_size = data_loader.batch_size +# total = n_batch * b_size +# # Get all predicted labels first +# target = np.empty(total, 'int64') +# for i, (imgs, _) in enumerate(tqdm(data_loader, total=n_batch, desc='Predicting labels')): +# p, c = torch.max(nn.Softmax(1)(explainer.model(imgs.cuda())), dim=1) +# target[i * b_size:(i + 1) * b_size] = c +# image_size = imgs.shape[-2:] +# +# # Get saliency maps for all images in val loader +# explanations = np.empty((total, *image_size)) +# for i, (imgs, _) in enumerate(tqdm(data_loader, total=n_batch, desc='Explaining images')): +# saliency_maps = explainer(imgs.cuda()) +# explanations[i * b_size:(i + 1) * b_size] = saliency_maps[ +# range(b_size), target[i * b_size:(i + 1) * b_size]].data.cpu().numpy() +# return explanations diff --git a/utils.py b/utils.py index 0318494..28bc51e 100644 --- a/utils.py +++ b/utils.py @@ -1,65 +1,62 @@ -import numpy as np -from matplotlib import pyplot as plt -import torch -from torch.utils.data.sampler import Sampler -from torchvision import transforms, datasets -from PIL import Image - - -# Dummy class to store arguments -class Dummy(): - pass - - -# Function that opens image from disk, normalizes it and converts to tensor -read_tensor = transforms.Compose([ - lambda x: Image.open(x), - transforms.Resize((224, 224)), - transforms.ToTensor(), - transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]), - lambda x: torch.unsqueeze(x, 0) -]) - - -# Plots image from tensor -def tensor_imshow(inp, title=None, **kwargs): - """Imshow for Tensor.""" - inp = inp.numpy().transpose((1, 2, 0)) - # Mean and std for ImageNet - mean = np.array([0.485, 0.456, 0.406]) - std = np.array([0.229, 0.224, 0.225]) - inp = std * inp + mean - inp = np.clip(inp, 0, 1) - plt.imshow(inp, **kwargs) - if title is not None: - plt.title(title) - - -# Given label number returns class name -def get_class_name(c): - labels = np.loadtxt('synset_words.txt', str, delimiter='\t') - return ' '.join(labels[c].split(',')[0].split()[1:]) - - -# Image preprocessing function -preprocess = transforms.Compose([ - transforms.Resize((224, 224)), - transforms.ToTensor(), - # Normalization for ImageNet - transforms.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]), - ]) - - -# Sampler for pytorch loader. Given range r loader will only -# return dataset[r] instead of whole dataset. -class RangeSampler(Sampler): - def __init__(self, r): - self.r = r - - def __iter__(self): - return iter(self.r) - - def __len__(self): - return len(self.r) +impo:wqrt numpy as np +from matplotlib import pyplot as plt +import torch +from torch.utils.data.sampler import Sampler +from torchvision import transforms, datasets +from PIL import Image + + +# Dummy class to store arguments +class Dummy(): + pass + +# Normalization params for Imagenet +means = [0.485, 0.456, 0.406] +stds = [0.229, 0.224, 0.225] + +# Function that opens image from disk, normalizes it and converts to tensor +read_tensor = transforms.Compose([ + lambda x: Image.open(x), + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(means, stds), + lambda x: torch.unsqueeze(x, 0) +]) + + +# Plots image from tensor +def tensor_imshow(inp, title=None, **kwargs): + """Imshow for Tensor.""" + inp = inp.numpy().transpose((1, 2, 0)) + inp = stds * inp + means + inp = np.clip(inp, 0, 1) + plt.imshow(inp, **kwargs) + if title is not None: + plt.title(title) + + +# Given label number returns class name +def get_class_name(c): + labels = np.loadtxt('synset_words.txt', str, delimiter='\t') + return ' '.join(labels[c].split(',')[0].split()[1:]) + + +# Image preprocessing function +preprocess = transforms.Compose([ + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(means, stds) + ]) + + +# Sampler for pytorch loader. Given range r loader will only +# return dataset[r] instead of whole dataset. +class RangeSampler(Sampler): + def __init__(self, r): + self.r = r + + def __iter__(self): + return iter(self.r) + + def __len__(self): + return len(self.r) From b70fffee7083798e7b0d7e226bebe64416468a68 Mon Sep 17 00:00:00 2001 From: navneeth Date: Wed, 1 Jul 2020 16:50:41 +0900 Subject: [PATCH 2/3] 1. Normalization parameters of dataset in common location 2. Replacing explicit .cuda() with device selection code (.device()) --- evaluation.py | 322 ++++++++++++++++++++++++------------------------ explanations.py | 206 +++++++++++++++---------------- utils.py | 124 +++++++++---------- 3 files changed, 326 insertions(+), 326 deletions(-) diff --git a/evaluation.py b/evaluation.py index 497e770..34ed5de 100644 --- a/evaluation.py +++ b/evaluation.py @@ -1,161 +1,161 @@ -from torch import nn -from tqdm import tqdm -from scipy.ndimage.filters import gaussian_filter - -from utils import * - -HW = 224 * 224 # image area -n_classes = 1000 - -def gkern(klen, nsig): - """Returns a Gaussian kernel array. - Convolution with it results in image blurring.""" - # create nxn zeros - inp = np.zeros((klen, klen)) - # set element at the middle to one, a dirac delta - inp[klen//2, klen//2] = 1 - # gaussian-smooth the dirac, resulting in a gaussian filter mask - k = gaussian_filter(inp, nsig) - kern = np.zeros((3, 3, klen, klen)) - kern[0, 0] = k - kern[1, 1] = k - kern[2, 2] = k - return torch.from_numpy(kern.astype('float32')) - -def auc(arr): - """Returns normalized Area Under Curve of the array.""" - return (arr.sum() - arr[0] / 2 - arr[-1] / 2) / (arr.shape[0] - 1) - -class CausalMetric(): - - def __init__(self, model, mode, step, substrate_fn): - r"""Create deletion/insertion metric instance. - - Args: - model (nn.Module): Black-box model being explained. - mode (str): 'del' or 'ins'. - step (int): number of pixels modified per one iteration. - substrate_fn (func): a mapping from old pixels to new pixels. - """ - assert mode in ['del', 'ins'] - self.model = model - self.mode = mode - self.step = step - self.substrate_fn = substrate_fn - - def single_run(self, img_tensor, explanation, verbose=0, save_to=None): - r"""Run metric on one image-saliency pair. - - Args: - img_tensor (Tensor): normalized image tensor. - explanation (np.ndarray): saliency map. - verbose (int): in [0, 1, 2]. - 0 - return list of scores. - 1 - also plot final step. - 2 - also plot every step and print 2 top classes. - save_to (str): directory to save every step plots to. - - Return: - scores (nd.array): Array containing scores at every step. - """ - pred = self.model(img_tensor.cuda()) - top, c = torch.max(pred, 1) - c = c.cpu().numpy()[0] - n_steps = (HW + self.step - 1) // self.step - - if self.mode == 'del': - title = 'Deletion game' - ylabel = 'Pixels deleted' - start = img_tensor.clone() - finish = self.substrate_fn(img_tensor) - elif self.mode == 'ins': - title = 'Insertion game' - ylabel = 'Pixels inserted' - start = self.substrate_fn(img_tensor) - finish = img_tensor.clone() - - scores = np.empty(n_steps + 1) - # Coordinates of pixels in order of decreasing saliency - salient_order = np.flip(np.argsort(explanation.reshape(-1, HW), axis=1), axis=-1) - for i in range(n_steps+1): - pred = self.model(start.cuda()) - pr, cl = torch.topk(pred, 2) - if verbose == 2: - print('{}: {:.3f}'.format(get_class_name(cl[0][0]), float(pr[0][0]))) - print('{}: {:.3f}'.format(get_class_name(cl[0][1]), float(pr[0][1]))) - scores[i] = pred[0, c] - # Render image if verbose, if it's the last step or if save is required. - if verbose == 2 or (verbose == 1 and i == n_steps) or save_to: - plt.figure(figsize=(10, 5)) - plt.subplot(121) - plt.title('{} {:.1f}%, P={:.4f}'.format(ylabel, 100 * i / n_steps, scores[i])) - plt.axis('off') - tensor_imshow(start[0]) - - plt.subplot(122) - plt.plot(np.arange(i+1) / n_steps, scores[:i+1]) - plt.xlim(-0.1, 1.1) - plt.ylim(0, 1.05) - plt.fill_between(np.arange(i+1) / n_steps, 0, scores[:i+1], alpha=0.4) - plt.title(title) - plt.xlabel(ylabel) - plt.ylabel(get_class_name(c)) - if save_to: - plt.savefig(save_to + '/{:03d}.png'.format(i)) - plt.close() - else: - plt.show() - if i < n_steps: - coords = salient_order[:, self.step * i:self.step * (i + 1)] - start.cpu().numpy().reshape(1, 3, HW)[0, :, coords] = finish.cpu().numpy().reshape(1, 3, HW)[0, :, coords] - return scores - - def evaluate(self, img_batch, exp_batch, batch_size): - r"""Efficiently evaluate big batch of images. - - Args: - img_batch (Tensor): batch of images. - exp_batch (np.ndarray): batch of explanations. - batch_size (int): number of images for one small batch. - - Returns: - scores (nd.array): Array containing scores at every step for every image. - """ - n_samples = img_batch.shape[0] - predictions = torch.FloatTensor(n_samples, n_classes) - assert n_samples % batch_size == 0 - for i in tqdm(range(n_samples // batch_size), desc='Predicting labels'): - preds = self.model(img_batch[i*batch_size:(i+1)*batch_size].cuda()).cpu() - predictions[i*batch_size:(i+1)*batch_size] = preds - top = np.argmax(predictions, -1) - n_steps = (HW + self.step - 1) // self.step - scores = np.empty((n_steps + 1, n_samples)) - salient_order = np.flip(np.argsort(exp_batch.reshape(-1, HW), axis=1), axis=-1) - r = np.arange(n_samples).reshape(n_samples, 1) - - substrate = torch.zeros_like(img_batch) - for j in tqdm(range(n_samples // batch_size), desc='Substrate'): - substrate[j*batch_size:(j+1)*batch_size] = self.substrate_fn(img_batch[j*batch_size:(j+1)*batch_size]) - - if self.mode == 'del': - caption = 'Deleting ' - start = img_batch.clone() - finish = substrate - elif self.mode == 'ins': - caption = 'Inserting ' - start = substrate - finish = img_batch.clone() - - # While not all pixels are changed - for i in tqdm(range(n_steps+1), desc=caption + 'pixels'): - # Iterate over batches - for j in range(n_samples // batch_size): - # Compute new scores - preds = self.model(start[j*batch_size:(j+1)*batch_size].cuda()) - preds = preds.cpu().numpy()[range(batch_size), top[j*batch_size:(j+1)*batch_size]] - scores[i, j*batch_size:(j+1)*batch_size] = preds - # Change specified number of most salient pixels to substrate pixels - coords = salient_order[:, self.step * i:self.step * (i + 1)] - start.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] = finish.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] - print('AUC: {}'.format(auc(scores.mean(1)))) - return scores +from torch import nn +from tqdm import tqdm +from scipy.ndimage.filters import gaussian_filter + +from utils import * + +HW = 224 * 224 # image area +n_classes = 1000 + +def gkern(klen, nsig): + """Returns a Gaussian kernel array. + Convolution with it results in image blurring.""" + # create nxn zeros + inp = np.zeros((klen, klen)) + # set element at the middle to one, a dirac delta + inp[klen//2, klen//2] = 1 + # gaussian-smooth the dirac, resulting in a gaussian filter mask + k = gaussian_filter(inp, nsig) + kern = np.zeros((3, 3, klen, klen)) + kern[0, 0] = k + kern[1, 1] = k + kern[2, 2] = k + return torch.from_numpy(kern.astype('float32')) + +def auc(arr): + """Returns normalized Area Under Curve of the array.""" + return (arr.sum() - arr[0] / 2 - arr[-1] / 2) / (arr.shape[0] - 1) + +class CausalMetric(): + + def __init__(self, model, mode, step, substrate_fn): + r"""Create deletion/insertion metric instance. + + Args: + model (nn.Module): Black-box model being explained. + mode (str): 'del' or 'ins'. + step (int): number of pixels modified per one iteration. + substrate_fn (func): a mapping from old pixels to new pixels. + """ + assert mode in ['del', 'ins'] + self.model = model + self.mode = mode + self.step = step + self.substrate_fn = substrate_fn + + def single_run(self, img_tensor, explanation, verbose=0, save_to=None): + r"""Run metric on one image-saliency pair. + + Args: + img_tensor (Tensor): normalized image tensor. + explanation (np.ndarray): saliency map. + verbose (int): in [0, 1, 2]. + 0 - return list of scores. + 1 - also plot final step. + 2 - also plot every step and print 2 top classes. + save_to (str): directory to save every step plots to. + + Return: + scores (nd.array): Array containing scores at every step. + """ + pred = self.model(img_tensor.cuda()) + top, c = torch.max(pred, 1) + c = c.cpu().numpy()[0] + n_steps = (HW + self.step - 1) // self.step + + if self.mode == 'del': + title = 'Deletion game' + ylabel = 'Pixels deleted' + start = img_tensor.clone() + finish = self.substrate_fn(img_tensor) + elif self.mode == 'ins': + title = 'Insertion game' + ylabel = 'Pixels inserted' + start = self.substrate_fn(img_tensor) + finish = img_tensor.clone() + + scores = np.empty(n_steps + 1) + # Coordinates of pixels in order of decreasing saliency + salient_order = np.flip(np.argsort(explanation.reshape(-1, HW), axis=1), axis=-1) + for i in range(n_steps+1): + pred = self.model(start.cuda()) + pr, cl = torch.topk(pred, 2) + if verbose == 2: + print('{}: {:.3f}'.format(get_class_name(cl[0][0]), float(pr[0][0]))) + print('{}: {:.3f}'.format(get_class_name(cl[0][1]), float(pr[0][1]))) + scores[i] = pred[0, c] + # Render image if verbose, if it's the last step or if save is required. + if verbose == 2 or (verbose == 1 and i == n_steps) or save_to: + plt.figure(figsize=(10, 5)) + plt.subplot(121) + plt.title('{} {:.1f}%, P={:.4f}'.format(ylabel, 100 * i / n_steps, scores[i])) + plt.axis('off') + tensor_imshow(start[0]) + + plt.subplot(122) + plt.plot(np.arange(i+1) / n_steps, scores[:i+1]) + plt.xlim(-0.1, 1.1) + plt.ylim(0, 1.05) + plt.fill_between(np.arange(i+1) / n_steps, 0, scores[:i+1], alpha=0.4) + plt.title(title) + plt.xlabel(ylabel) + plt.ylabel(get_class_name(c)) + if save_to: + plt.savefig(save_to + '/{:03d}.png'.format(i)) + plt.close() + else: + plt.show() + if i < n_steps: + coords = salient_order[:, self.step * i:self.step * (i + 1)] + start.cpu().numpy().reshape(1, 3, HW)[0, :, coords] = finish.cpu().numpy().reshape(1, 3, HW)[0, :, coords] + return scores + + def evaluate(self, img_batch, exp_batch, batch_size): + r"""Efficiently evaluate big batch of images. + + Args: + img_batch (Tensor): batch of images. + exp_batch (np.ndarray): batch of explanations. + batch_size (int): number of images for one small batch. + + Returns: + scores (nd.array): Array containing scores at every step for every image. + """ + n_samples = img_batch.shape[0] + predictions = torch.FloatTensor(n_samples, n_classes) + assert n_samples % batch_size == 0 + for i in tqdm(range(n_samples // batch_size), desc='Predicting labels'): + preds = self.model(img_batch[i*batch_size:(i+1)*batch_size].cuda()).cpu() + predictions[i*batch_size:(i+1)*batch_size] = preds + top = np.argmax(predictions, -1) + n_steps = (HW + self.step - 1) // self.step + scores = np.empty((n_steps + 1, n_samples)) + salient_order = np.flip(np.argsort(exp_batch.reshape(-1, HW), axis=1), axis=-1) + r = np.arange(n_samples).reshape(n_samples, 1) + + substrate = torch.zeros_like(img_batch) + for j in tqdm(range(n_samples // batch_size), desc='Substrate'): + substrate[j*batch_size:(j+1)*batch_size] = self.substrate_fn(img_batch[j*batch_size:(j+1)*batch_size]) + + if self.mode == 'del': + caption = 'Deleting ' + start = img_batch.clone() + finish = substrate + elif self.mode == 'ins': + caption = 'Inserting ' + start = substrate + finish = img_batch.clone() + + # While not all pixels are changed + for i in tqdm(range(n_steps+1), desc=caption + 'pixels'): + # Iterate over batches + for j in range(n_samples // batch_size): + # Compute new scores + preds = self.model(start[j*batch_size:(j+1)*batch_size].cuda()) + preds = preds.cpu().numpy()[range(batch_size), top[j*batch_size:(j+1)*batch_size]] + scores[i, j*batch_size:(j+1)*batch_size] = preds + # Change specified number of most salient pixels to substrate pixels + coords = salient_order[:, self.step * i:self.step * (i + 1)] + start.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] = finish.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] + print('AUC: {}'.format(auc(scores.mean(1)))) + return scores diff --git a/explanations.py b/explanations.py index bd8139d..4713274 100644 --- a/explanations.py +++ b/explanations.py @@ -1,103 +1,103 @@ -import numpy as np -import torch -import torch.nn as nn -from skimage.transform import resize -from tqdm import tqdm - -device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') -print('Using device :', device) - -class RISE(nn.Module): - def __init__(self, model, input_size, gpu_batch=100): - super(RISE, self).__init__() - self.model = model - self.input_size = input_size - self.gpu_batch = gpu_batch - - def generate_masks(self, N, s, p1, savepath='masks.npy'): - cell_size = np.ceil(np.array(self.input_size) / s) - up_size = (s + 1) * cell_size - - grid = np.random.rand(N, s, s) < p1 - grid = grid.astype('float32') - - self.masks = np.empty((N, *self.input_size)) - - for i in tqdm(range(N), desc='Generating filters'): - # Random shifts - x = np.random.randint(0, cell_size[0]) - y = np.random.randint(0, cell_size[1]) - # Linear upsampling and cropping - self.masks[i, :, :] = resize(grid[i], up_size, order=1, mode='reflect', - anti_aliasing=False)[x:x + self.input_size[0], y:y + self.input_size[1]] - self.masks = self.masks.reshape(-1, 1, *self.input_size) - np.save(savepath, self.masks) - self.masks = torch.from_numpy(self.masks).float() - self.masks = self.masks.to(device) - self.N = N - self.p1 = p1 - - def load_masks(self, filepath): - self.masks = np.load(filepath) - self.masks = torch.from_numpy(self.masks).float() - self.masks = self.masks.to(device) - self.N = self.masks.shape[0] - - def forward(self, x): - N = self.N - _, _, H, W = x.size() - # Apply array of filters to the image - stack = torch.mul(self.masks, x.data) - - # p = nn.Softmax(dim=1)(model(stack)) processed in batches - p = [] - for i in range(0, N, self.gpu_batch): - p.append(self.model(stack[i:min(i + self.gpu_batch, N)])) - p = torch.cat(p) - # Number of classes - CL = p.size(1) - sal = torch.matmul(p.data.transpose(0, 1), self.masks.view(N, H * W)) - sal = sal.view((CL, H, W)) - sal = sal / N / self.p1 - return sal - - -class RISEBatch(RISE): - def forward(self, x): - # Apply array of filters to the image - N = self.N - B, C, H, W = x.size() - stack = torch.mul(self.masks.view(N, 1, H, W), x.data.view(B * C, H, W)) - stack = stack.view(B * N, C, H, W) - stack = stack - - #p = nn.Softmax(dim=1)(model(stack)) in batches - p = [] - for i in range(0, N*B, self.gpu_batch): - p.append(self.model(stack[i:min(i + self.gpu_batch, N*B)])) - p = torch.cat(p) - CL = p.size(1) - p = p.view(N, B, CL) - sal = torch.matmul(p.permute(1, 2, 0), self.masks.view(N, H * W)) - sal = sal.view(B, CL, H, W) - return sal - -# To process in batches -# def explain_all_batch(data_loader, explainer): -# n_batch = len(data_loader) -# b_size = data_loader.batch_size -# total = n_batch * b_size -# # Get all predicted labels first -# target = np.empty(total, 'int64') -# for i, (imgs, _) in enumerate(tqdm(data_loader, total=n_batch, desc='Predicting labels')): -# p, c = torch.max(nn.Softmax(1)(explainer.model(imgs.cuda())), dim=1) -# target[i * b_size:(i + 1) * b_size] = c -# image_size = imgs.shape[-2:] -# -# # Get saliency maps for all images in val loader -# explanations = np.empty((total, *image_size)) -# for i, (imgs, _) in enumerate(tqdm(data_loader, total=n_batch, desc='Explaining images')): -# saliency_maps = explainer(imgs.cuda()) -# explanations[i * b_size:(i + 1) * b_size] = saliency_maps[ -# range(b_size), target[i * b_size:(i + 1) * b_size]].data.cpu().numpy() -# return explanations +import numpy as np +import torch +import torch.nn as nn +from skimage.transform import resize +from tqdm import tqdm + +device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') +print('Using device :', device) + +class RISE(nn.Module): + def __init__(self, model, input_size, gpu_batch=100): + super(RISE, self).__init__() + self.model = model + self.input_size = input_size + self.gpu_batch = gpu_batch + + def generate_masks(self, N, s, p1, savepath='masks.npy'): + cell_size = np.ceil(np.array(self.input_size) / s) + up_size = (s + 1) * cell_size + + grid = np.random.rand(N, s, s) < p1 + grid = grid.astype('float32') + + self.masks = np.empty((N, *self.input_size)) + + for i in tqdm(range(N), desc='Generating filters'): + # Random shifts + x = np.random.randint(0, cell_size[0]) + y = np.random.randint(0, cell_size[1]) + # Linear upsampling and cropping + self.masks[i, :, :] = resize(grid[i], up_size, order=1, mode='reflect', + anti_aliasing=False)[x:x + self.input_size[0], y:y + self.input_size[1]] + self.masks = self.masks.reshape(-1, 1, *self.input_size) + np.save(savepath, self.masks) + self.masks = torch.from_numpy(self.masks).float() + self.masks = self.masks.to(device) + self.N = N + self.p1 = p1 + + def load_masks(self, filepath): + self.masks = np.load(filepath) + self.masks = torch.from_numpy(self.masks).float() + self.masks = self.masks.to(device) + self.N = self.masks.shape[0] + + def forward(self, x): + N = self.N + _, _, H, W = x.size() + # Apply array of filters to the image + stack = torch.mul(self.masks, x.data) + + # p = nn.Softmax(dim=1)(model(stack)) processed in batches + p = [] + for i in range(0, N, self.gpu_batch): + p.append(self.model(stack[i:min(i + self.gpu_batch, N)])) + p = torch.cat(p) + # Number of classes + CL = p.size(1) + sal = torch.matmul(p.data.transpose(0, 1), self.masks.view(N, H * W)) + sal = sal.view((CL, H, W)) + sal = sal / N / self.p1 + return sal + + +class RISEBatch(RISE): + def forward(self, x): + # Apply array of filters to the image + N = self.N + B, C, H, W = x.size() + stack = torch.mul(self.masks.view(N, 1, H, W), x.data.view(B * C, H, W)) + stack = stack.view(B * N, C, H, W) + stack = stack + + #p = nn.Softmax(dim=1)(model(stack)) in batches + p = [] + for i in range(0, N*B, self.gpu_batch): + p.append(self.model(stack[i:min(i + self.gpu_batch, N*B)])) + p = torch.cat(p) + CL = p.size(1) + p = p.view(N, B, CL) + sal = torch.matmul(p.permute(1, 2, 0), self.masks.view(N, H * W)) + sal = sal.view(B, CL, H, W) + return sal + +# To process in batches +# def explain_all_batch(data_loader, explainer): +# n_batch = len(data_loader) +# b_size = data_loader.batch_size +# total = n_batch * b_size +# # Get all predicted labels first +# target = np.empty(total, 'int64') +# for i, (imgs, _) in enumerate(tqdm(data_loader, total=n_batch, desc='Predicting labels')): +# p, c = torch.max(nn.Softmax(1)(explainer.model(imgs.cuda())), dim=1) +# target[i * b_size:(i + 1) * b_size] = c +# image_size = imgs.shape[-2:] +# +# # Get saliency maps for all images in val loader +# explanations = np.empty((total, *image_size)) +# for i, (imgs, _) in enumerate(tqdm(data_loader, total=n_batch, desc='Explaining images')): +# saliency_maps = explainer(imgs.cuda()) +# explanations[i * b_size:(i + 1) * b_size] = saliency_maps[ +# range(b_size), target[i * b_size:(i + 1) * b_size]].data.cpu().numpy() +# return explanations diff --git a/utils.py b/utils.py index 28bc51e..b4b5c61 100644 --- a/utils.py +++ b/utils.py @@ -1,62 +1,62 @@ -impo:wqrt numpy as np -from matplotlib import pyplot as plt -import torch -from torch.utils.data.sampler import Sampler -from torchvision import transforms, datasets -from PIL import Image - - -# Dummy class to store arguments -class Dummy(): - pass - -# Normalization params for Imagenet -means = [0.485, 0.456, 0.406] -stds = [0.229, 0.224, 0.225] - -# Function that opens image from disk, normalizes it and converts to tensor -read_tensor = transforms.Compose([ - lambda x: Image.open(x), - transforms.Resize((224, 224)), - transforms.ToTensor(), - transforms.Normalize(means, stds), - lambda x: torch.unsqueeze(x, 0) -]) - - -# Plots image from tensor -def tensor_imshow(inp, title=None, **kwargs): - """Imshow for Tensor.""" - inp = inp.numpy().transpose((1, 2, 0)) - inp = stds * inp + means - inp = np.clip(inp, 0, 1) - plt.imshow(inp, **kwargs) - if title is not None: - plt.title(title) - - -# Given label number returns class name -def get_class_name(c): - labels = np.loadtxt('synset_words.txt', str, delimiter='\t') - return ' '.join(labels[c].split(',')[0].split()[1:]) - - -# Image preprocessing function -preprocess = transforms.Compose([ - transforms.Resize((224, 224)), - transforms.ToTensor(), - transforms.Normalize(means, stds) - ]) - - -# Sampler for pytorch loader. Given range r loader will only -# return dataset[r] instead of whole dataset. -class RangeSampler(Sampler): - def __init__(self, r): - self.r = r - - def __iter__(self): - return iter(self.r) - - def __len__(self): - return len(self.r) +import numpy as np +from matplotlib import pyplot as plt +import torch +from torch.utils.data.sampler import Sampler +from torchvision import transforms, datasets +from PIL import Image + + +# Dummy class to store arguments +class Dummy(): + pass + +# Normalization params for Imagenet +means = [0.485, 0.456, 0.406] +stds = [0.229, 0.224, 0.225] + +# Function that opens image from disk, normalizes it and converts to tensor +read_tensor = transforms.Compose([ + lambda x: Image.open(x), + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(means, stds), + lambda x: torch.unsqueeze(x, 0) +]) + + +# Plots image from tensor +def tensor_imshow(inp, title=None, **kwargs): + """Imshow for Tensor.""" + inp = inp.numpy().transpose((1, 2, 0)) + inp = stds * inp + means + inp = np.clip(inp, 0, 1) + plt.imshow(inp, **kwargs) + if title is not None: + plt.title(title) + + +# Given label number returns class name +def get_class_name(c): + labels = np.loadtxt('synset_words.txt', str, delimiter='\t') + return ' '.join(labels[c].split(',')[0].split()[1:]) + + +# Image preprocessing function +preprocess = transforms.Compose([ + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(means, stds) + ]) + + +# Sampler for pytorch loader. Given range r loader will only +# return dataset[r] instead of whole dataset. +class RangeSampler(Sampler): + def __init__(self, r): + self.r = r + + def __iter__(self): + return iter(self.r) + + def __len__(self): + return len(self.r) From 8aaae7d1c3b1db50bdacd0f2c3fcb207cd4a54b0 Mon Sep 17 00:00:00 2001 From: navneeth Date: Thu, 2 Jul 2020 16:15:38 +0900 Subject: [PATCH 3/3] Blackbox prediction on any model from pytorch zoo --- Saliency.py | 199 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 Saliency.py diff --git a/Saliency.py b/Saliency.py new file mode 100644 index 0000000..3479d77 --- /dev/null +++ b/Saliency.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +# coding: utf-8 + +# # Randomized Image Sampling for Explanations (RISE) + +import os +import argparse +from pathlib import Path +import numpy as np +from matplotlib import pyplot as plt +from tqdm import tqdm + +import torch +import torch.nn as nn +import torch.backends.cudnn as cudnn +import torch.utils.data + +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models + +from utils import (read_tensor, tensor_imshow, + get_class_name, preprocess, + RangeSampler) +from explanations import RISE + +cudnn.benchmark = True + +device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') +print('Using device :', device) + + +parser = argparse.ArgumentParser() + +# Directory with images split into class folders. +# Since we don't use ground truth labels for saliency all images can be +# moved to one class folder. +# e.g 'datasets/tiny-imagenet-200/val' +parser.add_argument('--datadir', required=True, + help='Directory with images (needs atleast 1 subfolder)') + +parser.add_argument('--model', required=False, type=str, default='resnet18', + help='type of architecture. string must match pytorch zoo') + +parser.add_argument('--workers', required=False, type=int, default=16, + help='Number of workers to load data') + +parser.add_argument('--gpu_batch', required=False, type=int, default=250, + help='Size of batches for GPU (use maximum that the GPU allows)') + + +args = parser.parse_args() +print(args) + + +# Sets the range of images to be explained for dataloader. +args.range = range(95, 105) +# Size of imput images. +args.input_size = (224, 224) + + +def load_model(model_name='resnet18', ptrained=True): + known_models = [x for x in dir(models)] + if model_name not in known_models: + raise ValueError('specified model doesnt exist in pytorch zoo') + + # This is equivalent to calling models.model_name(pretrained=True) + # e.g models.alexnet(pretrained=True) + model = getattr(models, model_name)(pretrained=ptrained) + + model.eval() + return model + + +def example(img, top_k=3): + saliency = explainer(img.to(device)).cpu().numpy() + + p, c = torch.topk(model(img.to(device)), k=top_k) + p, c = p[0], c[0] + + plt.figure(figsize=(10, 5*top_k)) + for k in range(top_k): + plt.subplot(top_k, 2, 2*k+1) + plt.axis('off') + plt.title('{:.2f}% {}'.format(100*p[k], get_class_name(c[k]))) + tensor_imshow(img[0]) + + plt.subplot(top_k, 2, 2*k+2) + plt.axis('off') + plt.title(get_class_name(c[k])) + tensor_imshow(img[0]) + sal = saliency[c[k]] + plt.imshow(sal, cmap='jet', alpha=0.5) + plt.colorbar(fraction=0.046, pad=0.04) + + plt.savefig('0-explain.png') + # plt.show() + plt.close() + + +# ## Explaining all images in dataloader +# Explaining the top predicted class for each image. + +def explain_all(data_loader, explainer): + # Get all predicted labels first + target = np.empty(len(data_loader), np.int) + for i, (img, _) in enumerate(tqdm(data_loader, total=len(data_loader), desc='Predicting labels')): + p, c = torch.max(model(img.to(device)), dim=-1) + target[i] = c[0] + + # Get saliency maps for all images in val loader + explanations = np.empty((len(data_loader), *args.input_size)) + for i, (img, _) in enumerate(tqdm(data_loader, total=len(data_loader), desc='Explaining images')): + saliency_maps = explainer(img.to(device)) + explanations[i] = saliency_maps[target[i]].cpu().numpy() + return explanations + + +if __name__ == '__main__': + # ## Prepare data + dataset = datasets.ImageFolder(args.datadir, preprocess) + + # This example only works with batch size 1. For larger batches see RISEBatch in explanations.py. + data_loader = torch.utils.data.DataLoader( + dataset, batch_size=1, shuffle=False, + num_workers=args.workers, pin_memory=True, sampler=RangeSampler(args.range)) + + print('Found {: >5} images belonging to {} classes.'.format(len(dataset), len(dataset.classes))) + print(' {: >5} images will be explained.'.format(len(data_loader) * data_loader.batch_size)) + + + # ## Load a black-box model for explanations from pytorch-zoo + # ## choose from any of + ''' + names = ['alexnet', 'vgg16', + 'resnet18', 'resnet34', 'resnet50', + 'squeezenet1_0', 'densenet161', 'inception_v3', + 'googlenet', 'shufflenet_v2_x1_0', 'mobilenet_v2'] + and more in https://pytorch.org/docs/stable/torchvision/models.html + ''' + model = load_model(args.model) + model = nn.Sequential(model, nn.Softmax(dim=1)) + model.to(device) + + for p in model.parameters(): + p.requires_grad = False + + # To use multiple GPUs + model = nn.DataParallel(model) + + # ## Create explainer instance + + explainer = RISE(model, args.input_size, args.gpu_batch) + + # Generate masks for RISE or use the saved ones. + maskspath = 'masks.npy' + generate_new = True + + if generate_new or not os.path.isfile(maskspath): + explainer.generate_masks(N=6000, s=8, p1=0.1, savepath=maskspath) + else: + explainer.load_masks(maskspath) + print('Masks are loaded.') + + # ## Explaining one instance + # Producing saliency maps for top $k$ predicted classes. + example(read_tensor('catdog.png'), 5) + + explanations = explain_all(data_loader, explainer) + + # Save explanations if needed. + # explanations.tofile('exp_{:05}-{:05}.npy'.format(args.range[0], args.range[-1])) + + for i, (img, _) in enumerate(data_loader): + p, c = torch.max(model(img.to(device)), dim=-1) + p, c = p[0].item(), c[0].item() + + prob = torch.softmax(model(img.to(device)), dim=-1) + pred_prob = prob[0][c] + + plt.figure(figsize=(10, 5)) + plt.suptitle('RISE Explanation for model {}'.format(args.model)) + + plt.subplot(121) + plt.axis('off') + plt.title('{:.2f}% {}'.format(100*p, get_class_name(c))) + tensor_imshow(img[0]) + + plt.subplot(122) + plt.axis('off') + plt.title(get_class_name(c)) + tensor_imshow(img[0]) + sal = explanations[i] + plt.imshow(sal, cmap='jet', alpha=0.5) + # plt.colorbar(fraction=0.046, pad=0.04) + + plt.savefig('{}-explain-{}.png'.format(i+1, args.model)) + # plt.show() + plt.close()