utils.py

import torch
import scipy.sparse as sp
import math
import numpy as np

from pathlib import Path

def move_to_cuda(maybe_tensor, device):
    if torch.is_tensor(maybe_tensor):
        return maybe_tensor.cuda(device)
    elif isinstance(maybe_tensor, dict):
        return {
            key: move_to_cuda(value, device)
            for key, value in maybe_tensor.items()
        }
    elif isinstance(maybe_tensor, list):
        return [move_to_cuda(x, device) for x in maybe_tensor]
    else:
        return maybe_tensor

def reformat_text_data(text_tokens):
    text_data = {}
    for k,v in text_tokens[0].items():
        for curr_dict in text_tokens:
            if text_data.get(k, None) is None:
                text_data[k] = [curr_dict[k].tolist()]
            else:
                text_data[k].append(curr_dict[k].tolist())

        text_data[k] = torch.tensor(text_data[k])

    return text_data

def save_model( save_path, args, model):
    output_dir = Path(save_path)
    # epoch_name = str(epoch)
    checkpoint_paths = [output_dir / ('best.pth')]
    for checkpoint_path in checkpoint_paths:
        to_save = {
            'model': model.state_dict(),
            'args': args
        }

        torch.save(to_save, checkpoint_path)

def compute_f_by_tensor(input, target, mask):
    input = input.view(-1).tolist()
    target = target.view(-1).tolist()
    mask = mask.view(-1).tolist()
    tp, fp, tn, fn = 0., 0., 0., 0.
    for i, t, m in zip(input, target, mask):
        if m == 1:
            continue
        else:
            if i == 1:
                if t == 1:
                    tp +=1
                else:
                    fp +=1
            else:
                if t == 1:
                    fn +=1
                else:
                    tn +=1
    if tp == 0:
        return 0., 0., 0.

    P = tp / (tp + fp)
    R = tp / (tp + fn)
    F = 2*P*R/(P+R)
    return P, R, F

def gelu_fast(x):
    if not hasattr(gelu_fast, "_a"):
        gelu_fast._a = math.sqrt(2 / math.pi)
    return 0.5 * x * (1 + torch.tanh(gelu_fast._a * (x + 0.044715 * torch.pow(x, 3))))

def gelu(x: torch.Tensor) -> torch.Tensor:
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))


def label_smoothed_nll_loss(log_probs, target, eps):
    #log_probs: N x C
    #target: N
    nll_loss = -log_probs.gather(dim=-1, index=target.unsqueeze(1)).squeeze(1)
    if eps == 0.:
        return nll_loss
    smooth_loss = -log_probs.sum(dim=-1)
    eps_i = eps / log_probs.size(-1)
    loss = (1. - eps) * nll_loss + eps_i * smooth_loss
    return loss

def encode_onehot(labels):
    classes = set(labels)
    classes_dict = {c: np.identity(len(classes))[i, :] for i, c in
                    enumerate(classes)}
    labels_onehot = np.array(list(map(classes_dict.get, labels)),
                             dtype=np.int32)
    return labels_onehot


def normalize(mx):
    """Row-normalize sparse matrix"""
    rowsum = np.array(mx.sum(1))
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = sp.diags(r_inv)
    mx = r_mat_inv.dot(mx)
    return mx


def accuracy(output, labels):
    preds = output.max(1)[1].type_as(labels)
    correct = preds.eq(labels).double()
    correct = correct.sum()
    return correct / len(labels)


def sparse_mx_to_torch_sparse_tensor(sparse_mx):
    """Convert a scipy sparse matrix to a torch sparse tensor."""
    sparse_mx = sparse_mx.tocoo().astype(np.float32)
    indices = torch.from_numpy(
        np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64))
    values = torch.from_numpy(sparse_mx.data)
    shape = torch.Size(sparse_mx.shape)
    return torch.sparse.FloatTensor(indices, values, shape)


import torch

class EarlyStopping:
    """Early stops the training if validation loss doesn't improve after a given patience."""
    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt'):
        """
        Args:
            patience (int): How long to wait after last time validation loss improved.
                            Default: 7
            verbose (bool): If True, prints a message for each validation loss improvement.
                            Default: False
            delta (float): Minimum change in the monitored quantity to qualify as an improvement.
                            Default: 0
            path (str): Path for the checkpoint to be saved to.
                            Default: 'checkpoint.pt'
        """
        self.patience = patience
        self.verbose = verbose
        self.counter = 0
        self.best_score = None
        self.early_stop = False
        self.val_loss_min = np.Inf
        self.delta = delta
        self.path = path

    def __call__(self, val_loss, model):

        score = -val_loss

        if self.best_score is None:
            self.best_score = score
            self.save_checkpoint(val_loss, model)
        elif score < self.best_score + self.delta:
            self.counter += 1
            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
            if self.counter >= self.patience:
                self.early_stop = True
        else:
            self.best_score = score
            self.save_checkpoint(val_loss, model)
            self.counter = 0

    def save_checkpoint(self, val_loss, model):
        '''Saves model when validation loss decrease.'''
        if self.verbose:
            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')
        torch.save(model.state_dict(), self.path)
        self.val_loss_min = val_loss