fix: resolve multiple open issues (#11, #19, #20, #7/#12/#16/#23)

README.md

@@ -57,7 +57,12 @@ Download the official training dataset based on the instructions in [TecoGAN-Ten
     <img src="results/gvt72_preview.gif" width="640" />
 </p>
 
-You can get them at :arrow_double_down: [百度网盘](https://pan.baidu.com/s/1lKyLJ5u6lrrXejyljao0Mw) (提取码:8tqc) and put them into :file_folder: [Datasets](data).
+~~You can get them at :arrow_double_down: [百度网盘](https://pan.baidu.com/s/1lKyLJ5u6lrrXejyljao0Mw) (提取码:8tqc) and put them into :file_folder: [Datasets](data).~~
+
+> **Note**: The original Baidu Netdisk download link is no longer available. You can prepare the testing datasets manually:
+> - **Vid4**: A widely-used VSR benchmark. You can find it in many VSR repos such as [BasicSR](https://github.com/xinntao/BasicSR).
+> - **ToS3**: Three video sequences from the TecoGAN paper.
+> - Ensure GT images are in PNG format under `data/<dataset>/GT/<sequence>/` and LR images under `data/<dataset>/Gaussian4xLR/<sequence>/`.
 The following shows the structure of the above three datasets.
 ```tex
 data

codes/main.py

@@ -173,10 +173,12 @@ def test(opt):
             logger.info('Testing on {}: {}'.format(dataset_idx, ds_name))
 
             # define metric calculator
-            try:
-                metric_calculator = MetricCalculator(opt)
-            except:
-                print('No metirc need to compute!')
+            metric_calculator = None
+            if opt.get('metric'):
+                try:
+                    metric_calculator = MetricCalculator(opt)
+                except Exception as e:
+                    logger.warning('Failed to create MetricCalculator: {}'.format(e))
 
             # create data loader
             test_loader = create_dataloader(opt, dataset_idx=dataset_idx)
@@ -199,25 +201,24 @@ def test(opt):
                     data_utils.save_sequence(res_seq_dir, hr_seq, frm_idx, to_bgr=True)
 
                 # compute metrics for the current sequence
-                true_seq_dir = osp.join(opt['dataset'][dataset_idx]['gt_seq_dir'], seq_idx)
-                try:
-                    metric_calculator.compute_sequence_metrics(seq_idx, true_seq_dir, '', pred_seq=hr_seq)
-                except:
-                    print('No metirc need to compute!')
+                if metric_calculator is not None:
+                    true_seq_dir = osp.join(opt['dataset'][dataset_idx]['gt_seq_dir'], seq_idx)
+                    try:
+                        metric_calculator.compute_sequence_metrics(seq_idx, true_seq_dir, '', pred_seq=hr_seq)
+                    except Exception as e:
+                        logger.warning('Failed to compute metrics for {}: {}'.format(seq_idx, e))
 
             # save/print metrics
-            try:
-                if opt['test'].get('save_json'):
-                    # save results to json file
-                    json_path = osp.join(
-                        opt['test']['json_dir'], '{}_avg.json'.format(ds_name))
-                    metric_calculator.save_results(model_idx, json_path, override=True)
-                else:
-                    # print directly
-                    metric_calculator.display_results()
-
-            except:
-                print('No metirc need to save!')
+            if metric_calculator is not None:
+                try:
+                    if opt['test'].get('save_json'):
+                        json_path = osp.join(
+                            opt['test']['json_dir'], '{}_avg.json'.format(ds_name))
+                        metric_calculator.save_results(model_idx, json_path, override=True)
+                    else:
+                        metric_calculator.display_results()
+                except Exception as e:
+                    logger.warning('Failed to save/display metrics: {}'.format(e))
 
             logger.info('-' * 40)
 

codes/metrics/LPIPS/models/__init__.py

@@ -3,160 +3,34 @@
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
-from skimage.metrics import structural_similarity as ssim
-import torch
-from torch.autograd import Variable
+from .util import (
+    normalize_tensor, l2, psnr, dssim, rgb2lab, tensor2np, np2tensor,
+    tensor2tensorlab, tensorlab2tensor, tensor2im, im2tensor, tensor2vec,
+    voc_ap,
+)
 
-from metrics.LPIPS.models import dist_model
+import torch
 
 
 class PerceptualLoss(torch.nn.Module):
-    def __init__(self, model='net-lin', net='alex', colorspace='rgb', spatial=False, use_gpu=True, gpu_ids=[0], version='0.1'): # VGG using our perceptually-learned weights (LPIPS metric)
-    # def __init__(self, model='net', net='vgg', use_gpu=True): # "default" way of using VGG as a perceptual loss
+    def __init__(self, model='net-lin', net='alex', colorspace='rgb',
+                 spatial=False, use_gpu=True, gpu_ids=[0], version='0.1'):
         super(PerceptualLoss, self).__init__()
+        from .dist_model import DistModel
+
         print('Setting up Perceptual loss...')
         self.use_gpu = use_gpu
         self.spatial = spatial
         self.gpu_ids = gpu_ids
-        self.model = dist_model.DistModel()
-        self.model.initialize(model=model, net=net, use_gpu=use_gpu, colorspace=colorspace, spatial=self.spatial, gpu_ids=gpu_ids, version=version)
-        print('...[%s] initialized'%self.model.name())
+        self.model = DistModel()
+        self.model.initialize(
+            model=model, net=net, use_gpu=use_gpu, colorspace=colorspace,
+            spatial=self.spatial, gpu_ids=gpu_ids, version=version)
+        print('...[%s] initialized' % self.model.name())
         print('...Done')
 
     def forward(self, pred, target, normalize=False):
-        """
-        Pred and target are Variables.
-        If normalize is True, assumes the images are between [0,1] and then scales them between [-1,+1]
-        If normalize is False, assumes the images are already between [-1,+1]
-
-        Inputs pred and target are Nx3xHxW
-        Output pytorch Variable N long
-        """
-
         if normalize:
-            target = 2 * target  - 1
-            pred = 2 * pred  - 1
-
+            target = 2 * target - 1
+            pred = 2 * pred - 1
         return self.model.forward(target, pred)
-
-def normalize_tensor(in_feat,eps=1e-10):
-    norm_factor = torch.sqrt(torch.sum(in_feat**2,dim=1,keepdim=True))
-    return in_feat/(norm_factor+eps)
-
-def l2(p0, p1, range=255.):
-    return .5*np.mean((p0 / range - p1 / range)**2)
-
-def psnr(p0, p1, peak=255.):
-    return 10*np.log10(peak**2/np.mean((1.*p0-1.*p1)**2))
-
-def dssim(p0, p1, range=255.):
-    return (1 - ssim(p0, p1, data_range=range, multichannel=True)) / 2.
-
-def rgb2lab(in_img,mean_cent=False):
-    from skimage import color
-    img_lab = color.rgb2lab(in_img)
-    if(mean_cent):
-        img_lab[:,:,0] = img_lab[:,:,0]-50
-    return img_lab
-
-def tensor2np(tensor_obj):
-    # change dimension of a tensor object into a numpy array
-    return tensor_obj[0].cpu().float().numpy().transpose((1,2,0))
-
-def np2tensor(np_obj):
-     # change dimenion of np array into tensor array
-    return torch.Tensor(np_obj[:, :, :, np.newaxis].transpose((3, 2, 0, 1)))
-
-def tensor2tensorlab(image_tensor,to_norm=True,mc_only=False):
-    # image tensor to lab tensor
-    from skimage import color
-
-    img = tensor2im(image_tensor)
-    img_lab = color.rgb2lab(img)
-    if(mc_only):
-        img_lab[:,:,0] = img_lab[:,:,0]-50
-    if(to_norm and not mc_only):
-        img_lab[:,:,0] = img_lab[:,:,0]-50
-        img_lab = img_lab/100.
-
-    return np2tensor(img_lab)
-
-def tensorlab2tensor(lab_tensor,return_inbnd=False):
-    from skimage import color
-    import warnings
-    warnings.filterwarnings("ignore")
-
-    lab = tensor2np(lab_tensor)*100.
-    lab[:,:,0] = lab[:,:,0]+50
-
-    rgb_back = 255.*np.clip(color.lab2rgb(lab.astype('float')),0,1)
-    if(return_inbnd):
-        # convert back to lab, see if we match
-        lab_back = color.rgb2lab(rgb_back.astype('uint8'))
-        mask = 1.*np.isclose(lab_back,lab,atol=2.)
-        mask = np2tensor(np.prod(mask,axis=2)[:,:,np.newaxis])
-        return (im2tensor(rgb_back),mask)
-    else:
-        return im2tensor(rgb_back)
-
-
-def rgb2lab(input):
-    from skimage import color
-    return color.rgb2lab(input / 255.)
-
-def tensor2im(image_tensor, imtype=np.uint8, cent=1., factor=255./2.):
-    image_numpy = image_tensor[0].cpu().float().numpy()
-    image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + cent) * factor
-    return image_numpy.astype(imtype)
-
-def im2tensor(image, imtype=np.uint8, cent=1., factor=255./2.):
-    return torch.Tensor((image / factor - cent)
-                        [:, :, :, np.newaxis].transpose((3, 2, 0, 1)))
-
-def tensor2vec(vector_tensor):
-    return vector_tensor.data.cpu().numpy()[:, :, 0, 0]
-
-def voc_ap(rec, prec, use_07_metric=False):
-    """ ap = voc_ap(rec, prec, [use_07_metric])
-    Compute VOC AP given precision and recall.
-    If use_07_metric is true, uses the
-    VOC 07 11 point method (default:False).
-    """
-    if use_07_metric:
-        # 11 point metric
-        ap = 0.
-        for t in np.arange(0., 1.1, 0.1):
-            if np.sum(rec >= t) == 0:
-                p = 0
-            else:
-                p = np.max(prec[rec >= t])
-            ap = ap + p / 11.
-    else:
-        # correct AP calculation
-        # first append sentinel values at the end
-        mrec = np.concatenate(([0.], rec, [1.]))
-        mpre = np.concatenate(([0.], prec, [0.]))
-
-        # compute the precision envelope
-        for i in range(mpre.size - 1, 0, -1):
-            mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
-
-        # to calculate area under PR curve, look for points
-        # where X axis (recall) changes value
-        i = np.where(mrec[1:] != mrec[:-1])[0]
-
-        # and sum (\Delta recall) * prec
-        ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
-    return ap
-
-def tensor2im(image_tensor, imtype=np.uint8, cent=1., factor=255./2.):
-# def tensor2im(image_tensor, imtype=np.uint8, cent=1., factor=1.):
-    image_numpy = image_tensor[0].cpu().float().numpy()
-    image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + cent) * factor
-    return image_numpy.astype(imtype)
-
-def im2tensor(image, imtype=np.uint8, cent=1., factor=255./2.):
-# def im2tensor(image, imtype=np.uint8, cent=1., factor=1.):
-    return torch.Tensor((image / factor - cent)
-                        [:, :, :, np.newaxis].transpose((3, 2, 0, 1)))

codes/metrics/LPIPS/models/dist_model.py

@@ -19,7 +19,7 @@
 # from IPython import embed
 
 from . import networks_basic as networks
-import models as util
+from . import util
 
 class DistModel(BaseModel):
     def name(self):

codes/metrics/LPIPS/models/networks_basic.py

@@ -12,7 +12,7 @@
 # from IPython import embed
 from . import pretrained_networks as pn
 
-import metrics.LPIPS.models as util
+from . import util
 
 def spatial_average(in_tens, keepdim=True):
     return in_tens.mean([2,3],keepdim=keepdim)

codes/metrics/LPIPS/models/util.py

@@ -0,0 +1,114 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+from skimage.metrics import structural_similarity as ssim
+import torch
+
+
+def normalize_tensor(in_feat, eps=1e-10):
+    norm_factor = torch.sqrt(torch.sum(in_feat**2, dim=1, keepdim=True))
+    return in_feat / (norm_factor + eps)
+
+
+def l2(p0, p1, range=255.):
+    return .5 * np.mean((p0 / range - p1 / range)**2)
+
+
+def psnr(p0, p1, peak=255.):
+    return 10 * np.log10(peak**2 / np.mean((1. * p0 - 1. * p1)**2))
+
+
+def dssim(p0, p1, range=255.):
+    return (1 - ssim(p0, p1, data_range=range, multichannel=True)) / 2.
+
+
+def rgb2lab(in_img, mean_cent=False):
+    from skimage import color
+    img_lab = color.rgb2lab(in_img)
+    if mean_cent:
+        img_lab[:, :, 0] = img_lab[:, :, 0] - 50
+    return img_lab
+
+
+def tensor2np(tensor_obj):
+    return tensor_obj[0].cpu().float().numpy().transpose((1, 2, 0))
+
+
+def np2tensor(np_obj):
+    return torch.Tensor(np_obj[:, :, :, np.newaxis].transpose((3, 2, 0, 1)))
+
+
+def tensor2tensorlab(image_tensor, to_norm=True, mc_only=False):
+    from skimage import color
+
+    img = tensor2im(image_tensor)
+    img_lab = color.rgb2lab(img)
+    if mc_only:
+        img_lab[:, :, 0] = img_lab[:, :, 0] - 50
+    if to_norm and not mc_only:
+        img_lab[:, :, 0] = img_lab[:, :, 0] - 50
+        img_lab = img_lab / 100.
+
+    return np2tensor(img_lab)
+
+
+def tensorlab2tensor(lab_tensor, return_inbnd=False):
+    from skimage import color
+    import warnings
+    warnings.filterwarnings("ignore")
+
+    lab = tensor2np(lab_tensor) * 100.
+    lab[:, :, 0] = lab[:, :, 0] + 50
+
+    rgb_back = 255. * np.clip(color.lab2rgb(lab.astype('float')), 0, 1)
+    if return_inbnd:
+        lab_back = color.rgb2lab(rgb_back.astype('uint8'))
+        mask = 1. * np.isclose(lab_back, lab, atol=2.)
+        mask = np2tensor(np.prod(mask, axis=2)[:, :, np.newaxis])
+        return (im2tensor(rgb_back), mask)
+    else:
+        return im2tensor(rgb_back)
+
+
+def tensor2im(image_tensor, imtype=np.uint8, cent=1., factor=255. / 2.):
+    image_numpy = image_tensor[0].cpu().float().numpy()
+    image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + cent) * factor
+    return image_numpy.astype(imtype)
+
+
+def im2tensor(image, imtype=np.uint8, cent=1., factor=255. / 2.):
+    return torch.Tensor((image / factor - cent)
+                        [:, :, :, np.newaxis].transpose((3, 2, 0, 1)))
+
+
+def tensor2vec(vector_tensor):
+    return vector_tensor.data.cpu().numpy()[:, :, 0, 0]
+
+
+def voc_ap(rec, prec, use_07_metric=False):
+    """ ap = voc_ap(rec, prec, [use_07_metric])
+    Compute VOC AP given precision and recall.
+    If use_07_metric is true, uses the
+    VOC 07 11 point method (default:False).
+    """
+    if use_07_metric:
+        ap = 0.
+        for t in np.arange(0., 1.1, 0.1):
+            if np.sum(rec >= t) == 0:
+                p = 0
+            else:
+                p = np.max(prec[rec >= t])
+            ap = ap + p / 11.
+    else:
+        mrec = np.concatenate(([0.], rec, [1.]))
+        mpre = np.concatenate(([0.], prec, [0.]))
+
+        for i in range(mpre.size - 1, 0, -1):
+            mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
+
+        i = np.where(mrec[1:] != mrec[:-1])[0]
+
+        ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
+    return ap