Hyperparam optimization with augmented data #1

[hbandukw]

DATA: AUGMENTED / BALANCED
METRIC: ACCURACY
HYPERPARAMS:
[I 2025-01-24 19:55:40,455] Trial 27 finished with value: 0.904019688269073 and parameters: {'learning_rate': 8.300432875328772e-05, 'num_trainable_layers': 2, 'dropout_rate': 0.3847406475130443, 'batch_size': 32, 'step_size': 9, 'gamma': 0.5951936405857416, 'epochs': 5}. Best is trial 27 with value: 0.904019688269073.

RESULTS: Epoch 5 - Train Loss: 0.6058, Val Loss: 1.1301, Val Accuracy: 0.6579
Final Model Accuracy: 0.6579

see comment below for differences in results

[hbandukw]

Training model with the best hyperparams to validate results

import pandas as pd

# aug 
train1 = pd.read_csv('/kaggle/input/balanced/train1.csv')
train2 = pd.read_csv('/kaggle/input/balanced/train2.csv')
train3 = pd.read_csv('/kaggle/input/balanced/train3.csv')
train4 = pd.read_csv('/kaggle/input/balanced/train4.csv')

datasets = [train1, train2, train3, train4]

# Extract quotes and labels using list comprehension
texts = [ds['quote'] for ds in datasets]
labels = [ds['numeric_label'] for ds in datasets]

# Concatenate all texts and labels using pandas.concat
train1234_texts = pd.concat(texts, ignore_index=True)
train1234_labels = pd.concat(labels, ignore_index=True)

train1234_texts = train1234_texts.to_list()
train1234_labels = train1234_labels.to_list()

test = pd.read_csv("/kaggle/input/balanced/test.csv")

test_texts = test['quote'].to_list()
test_labels = test["numeric_label"].to_list()

import os
import pandas as pd
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
from transformers import DistilBertTokenizer, DistilBertForSequenceClassification, DistilBertConfig
from sklearn.metrics import f1_score, confusion_matrix, balanced_accuracy_score, precision_score, recall_score, accuracy_score
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
from torch.optim import AdamW, lr_scheduler
import random

# Set seed for reproducibility
torch.manual_seed(42)
np.random.seed(42)
random.seed(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

# Training params
MAX_LENGTH = 365
TRAIN_BATCH_SIZE = 32
VAL_BATCH_SIZE = 32
LEARNING_RATE = 8.300432875328772e-05
STEP_SIZE = 9
GAMMA = 0.5951936405857416
EPOCHS = 5   
#GRAD_CLIP = 0.5835870650159247  
#PATIENCE = 3  # early stopping: Number of epochs to wait for improvement
#MIN_DELTA = 0.01 # early stopping: Minimum loss improvement required
DROPOUT_RATE = 0.3847406475130443
NUM_TRAINABLE_LAYERS = 2


# Ensure output directory exists
output_dir = "/kaggle/working/output"
os.makedirs(output_dir, exist_ok=True)

# Load and prepare data
#df = pd.read_parquet("/kaggle/input/climatetext/train.parquet")
#df['label_int'] = df['label'].str.split("_").str[0].astype('int')

# Create dictionary for label names
#label_dict = df[['label_int', 'label']].drop_duplicates().set_index('label_int')['label'].to_dict()
label_dict = {
    0: "0_not_relevant",
    1: "1_not_happening",
    2: "2_not_human",
    3: "3_not_bad",
    4: "4_solutions_harmful_unnecessary",
    5: "5_science_unreliable",
    6: "6_proponents_biased",
    7: "7_fossil_fuels_needed"
}


# Initialize tokenizer and model
tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased', do_lower_case=True)

# Initialize model configuration with custom dropout rate
config = DistilBertConfig.from_pretrained('distilbert-base-uncased', 
                                          num_labels=8, 
                                          dropout=DROPOUT_RATE,
                                          attention_dropout=DROPOUT_RATE)

# Load the model with the updated configuration
model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', config=config)

# Freeze all layers except the last `num_trainable_layers`
for name, param in model.distilbert.named_parameters():
    param.requires_grad = False

# Unfreeze the last `num_trainable_layers`
for layer_idx in range(6 - NUM_TRAINABLE_LAYERS, 6):  # DistilBERT has 6 layers
    for name, param in model.distilbert.transformer.layer[layer_idx].named_parameters():
        param.requires_grad = True

# Ensure classification head is trainable
for name, param in model.classifier.named_parameters():
    param.requires_grad = True

# Dataset and DataLoader preparation
class QuotesDataset(Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx], dtype=torch.long)
        return item

    def __len__(self):
        return len(self.labels)

def encode_data(tokenizer, texts, labels, max_length):
    encodings = tokenizer(texts, truncation=True, padding='max_length', max_length=max_length, return_tensors='pt')
    return QuotesDataset(encodings, labels)

train_dataset = encode_data(tokenizer, train1234_texts, train1234_labels, MAX_LENGTH)
val_dataset = encode_data(tokenizer, test_texts, test_labels, MAX_LENGTH)

#train_dataset = encode_data(tokenizer, X_train, y_train, MAX_LENGTH)
#val_dataset = encode_data(tokenizer, X_val, y_val, MAX_LENGTH)

train_loader = DataLoader(train_dataset, batch_size=TRAIN_BATCH_SIZE, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=VAL_BATCH_SIZE, shuffle=False)

# Initialize optimizer
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
optimizer = AdamW(model.parameters(), lr=LEARNING_RATE)
scheduler = lr_scheduler.StepLR(optimizer, step_size=STEP_SIZE, gamma=GAMMA)

# Plotting functions
def plot_accuracies(training_accuracies, validation_accuracies):
    plt.figure(figsize=(8, 6))
    plt.plot(training_accuracies, label='Training Accuracy')
    plt.plot(validation_accuracies, label='Validation Accuracy')
    plt.title('Training and Validation Accuracy')
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    plt.legend()
    plt.savefig(f"{output_dir}/accuracy_plot.png")
    plt.close()

def plot_confusion_matrix(cm, class_labels, epoch, output_dir):
    plt.figure(figsize=(10, 8))
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_labels, yticklabels=class_labels)
    plt.title(f'Confusion Matrix for Epoch {epoch + 1}')
    plt.ylabel('True Label')
    plt.xlabel('Predicted Label')
    plt.savefig(os.path.join(output_dir, f'confusion_matrix_epoch_{epoch + 1}.png'))
    plt.close()

# Training and validation loop
train_losses = []
val_losses = []
train_accuracies = []
val_accuracies = []
metrics_df = pd.DataFrame()

for epoch in range(EPOCHS):
    model.train()
    train_loss = 0
    correct_train = 0
    total_train = 0

    for batch in train_loader:
        optimizer.zero_grad()
        batch = {k: v.to(device) for k, v in batch.items()}
        outputs = model(**batch)
        loss = outputs.loss
        loss.backward()

        # Gradient clipping
        #torch.nn.utils.clip_grad_norm_(model.parameters(), GRAD_CLIP)  # Clip gradients to stabilize training

        optimizer.step()  
        train_loss += loss.item()

        predictions = torch.argmax(outputs.logits, dim=-1)
        correct_train += (predictions == batch['labels']).sum().item()
        total_train += batch['labels'].size(0)

    scheduler.step()
    train_losses.append(train_loss / len(train_loader))
    train_accuracies.append(correct_train / total_train)

    model.eval()
    val_loss = 0
    correct_val = 0
    total_val = 0

    # Clear predictions and true labels before each validation loop
    all_predictions, all_true_labels = [], []  # Reset for each epoch

    with torch.no_grad():
        for batch in val_loader:
            batch = {k: v.to(device) for k, v in batch.items()}
            outputs = model(**batch)
            val_loss += outputs.loss.item()
            predictions = torch.argmax(outputs.logits, dim=-1)
            all_predictions.extend(predictions.cpu().numpy())
            all_true_labels.extend(batch['labels'].cpu().numpy())

            correct_val += (predictions == batch['labels']).sum().item()
            total_val += batch['labels'].size(0)

    val_losses.append(val_loss / len(val_loader))
    val_accuracies.append(correct_val / total_val)

    # Compute confusion matrix
    cm = confusion_matrix(all_true_labels, all_predictions)
    class_labels = [label_dict.get(i, f'Class {i}') for i in range(len(np.unique(all_true_labels)))]

    # Save confusion matrix only for the best validation loss
    plot_confusion_matrix(cm, class_labels, epoch, output_dir)

    # Print training progress per epoch
    print(f"Epoch {epoch + 1} - Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}, Val Accuracy: {val_accuracies[-1]:.4f}")

    # Early stopping check
    #if (best_val_loss - val_loss) > MIN_DELTA:  # Significant improvement
    #    best_val_loss = val_loss
    #    no_improvement_count = 0

        # Save the best model
        #torch.save(model.state_dict(), f"{output_dir}/best_model.pth")
    #else:  # No significant improvement
     #   no_improvement_count += 1
    #    if no_improvement_count >= PATIENCE:
     #       print(f"Early stopping at epoch {epoch + 1}")
      #      break

    # Calculate metrics
    balanced_acc = balanced_accuracy_score(all_true_labels, all_predictions)
    average_f1 = f1_score(all_true_labels, all_predictions, average='macro')
    weighted_f1 = f1_score(all_true_labels, all_predictions, average='weighted')
    f1_scores_per_class = f1_score(all_true_labels, all_predictions, average=None)
    precision_per_class = precision_score(all_true_labels, all_predictions, average=None, zero_division=0)
    recall_per_class = recall_score(all_true_labels, all_predictions, average=None, zero_division=0)

    # Append metrics to DataFrame with class labels
    epoch_metrics = {
        "Epoch": epoch + 1,
        "Train Loss": train_losses[-1],
        "Validation Loss": val_losses[-1],
        "Train Accuracy": train_accuracies[-1],
        "Validation Accuracy": val_accuracies[-1],
        "Balanced Accuracy": balanced_acc,
        "Average F1": average_f1,
        "Weighted F1": weighted_f1,
    }
    epoch_metrics.update({f"{label_dict[i]} F1": f1_scores_per_class[i] for i in range(len(f1_scores_per_class))})
    epoch_metrics.update({f"{label_dict[i]} Precision": precision_per_class[i] for i in range(len(precision_per_class))})
    epoch_metrics.update({f"{label_dict[i]} Recall": recall_per_class[i] for i in range(len(recall_per_class))})

    metrics_df = pd.concat([metrics_df, pd.DataFrame([epoch_metrics])], ignore_index=True)

# Save metrics to CSV and plot accuracies
metrics_df.to_csv(f"{output_dir}/training_metrics.csv", index=False)
plot_accuracies(train_accuracies, val_accuracies)

# Calculate final model accuracy
final_accuracy = accuracy_score(all_true_labels, all_predictions)
print(f"Final Model Accuracy: {final_accuracy:.4f}")

# Save the trained model
model_path = f"{output_dir}/trained_model.pth"
torch.save(model.state_dict(), model_path)
print(f"Model saved to {model_path}")

[hbandukw]

First Script (Dynamic Dropout Adjustment)
After loading the model, the script changes the dropout rate for each layer directly. Dropout is applied in two parts of the model:

Attention Mechanism: It randomly ignores some attention scores.
Feed-Forward Layers: It randomly ignores some neuron outputs.
The script dynamically sets the dropout rate like this:

for layer in model.distilbert.transformer.layer:
    layer.attention.dropout.p = dropout_rate  # Change dropout in attention
    layer.ffn.dropout.p = dropout_rate        # Change dropout in feed-forward
This means all layers, even frozen ones (those you are not training), will use this new dropout rate.

Second Script (Static Dropout Adjustment)
Dropout rates are set only once when the model is initialized:

config = DistilBertConfig.from_pretrained(
    'distilbert-base-uncased', 
    num_labels=8, 
    dropout=DROPOUT_RATE, 
    attention_dropout=DROPOUT_RATE
)

This applies the same dropout rate across the whole model and doesn’t touch it again during training.

[rafecchang]

So my understanding is that dynamic dropout adjustment involves randomness that might not be able to replicate- is this correct?

[rafecchang]

We also place scheduler at the different parts of the training/validation loop. Maybe that's what's causing this.

[hbandukw]

changes

I changed where I placed the scheduler (see below! I have indicated the change with "## HERE IS THE CHANGE")

results:

Epoch 5 - Train Loss: 0.6058, Val Loss: 1.1301, Val Accuracy: 0.6579
Final Model Accuracy: 0.6579

code

import os
import pandas as pd
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
from transformers import DistilBertTokenizer, DistilBertForSequenceClassification, DistilBertConfig
from sklearn.metrics import f1_score, confusion_matrix, balanced_accuracy_score, precision_score, recall_score, accuracy_score
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
from torch.optim import AdamW, lr_scheduler
import random

# Set seed for reproducibility
torch.manual_seed(42)
np.random.seed(42)
random.seed(42)
#torch.backends.cudnn.deterministic = True
#torch.backends.cudnn.benchmark = False

# Training params
MAX_LENGTH = 365
TRAIN_BATCH_SIZE = 32
VAL_BATCH_SIZE = 32
LEARNING_RATE = 8.300432875328772e-05
STEP_SIZE = 9
GAMMA = 0.5951936405857416
EPOCHS = 5   
#GRAD_CLIP = 0.5835870650159247  
#PATIENCE = 3  # early stopping: Number of epochs to wait for improvement
#MIN_DELTA = 0.01 # early stopping: Minimum loss improvement required
DROPOUT_RATE = 0.3847406475130443
NUM_TRAINABLE_LAYERS = 2


# Ensure output directory exists
output_dir = "/kaggle/working/output"
os.makedirs(output_dir, exist_ok=True)

# Load and prepare data
#df = pd.read_parquet("/kaggle/input/climatetext/train.parquet")
#df['label_int'] = df['label'].str.split("_").str[0].astype('int')

# Create dictionary for label names
#label_dict = df[['label_int', 'label']].drop_duplicates().set_index('label_int')['label'].to_dict()
label_dict = {
    0: "0_not_relevant",
    1: "1_not_happening",
    2: "2_not_human",
    3: "3_not_bad",
    4: "4_solutions_harmful_unnecessary",
    5: "5_science_unreliable",
    6: "6_proponents_biased",
    7: "7_fossil_fuels_needed"
}


# Initialize tokenizer and model
tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased', do_lower_case=True)

# Initialize model configuration with custom dropout rate
config = DistilBertConfig.from_pretrained('distilbert-base-uncased', 
                                          num_labels=8, 
                                          dropout=DROPOUT_RATE,
                                          attention_dropout=DROPOUT_RATE)

# Load the model with the updated configuration
model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', config=config)
 
# Freeze all layers except the last `num_trainable_layers`
for name, param in model.distilbert.named_parameters():
    param.requires_grad = False

# Unfreeze the last `num_trainable_layers`
for layer_idx in range(6 - NUM_TRAINABLE_LAYERS, 6):  # DistilBERT has 6 layers
    for name, param in model.distilbert.transformer.layer[layer_idx].named_parameters():
        param.requires_grad = True

# Ensure classification head is trainable
for name, param in model.classifier.named_parameters():
    param.requires_grad = True

# Dataset and DataLoader preparation
class QuotesDataset(Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx], dtype=torch.long)
        return item

    def __len__(self):
        return len(self.labels)

def encode_data(tokenizer, texts, labels, max_length):
    encodings = tokenizer(texts, truncation=True, padding='max_length', max_length=max_length, return_tensors='pt')
    return QuotesDataset(encodings, labels)

train_dataset = encode_data(tokenizer, train1234_texts, train1234_labels, MAX_LENGTH)
val_dataset = encode_data(tokenizer, test_texts, test_labels, MAX_LENGTH)

#train_dataset = encode_data(tokenizer, X_train, y_train, MAX_LENGTH)
#val_dataset = encode_data(tokenizer, X_val, y_val, MAX_LENGTH)

train_loader = DataLoader(train_dataset, batch_size=TRAIN_BATCH_SIZE, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=VAL_BATCH_SIZE, shuffle=False)

# Initialize optimizer
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
optimizer = AdamW(model.parameters(), lr=LEARNING_RATE)
scheduler = lr_scheduler.StepLR(optimizer, step_size=STEP_SIZE, gamma=GAMMA)

# Plotting functions
def plot_accuracies(training_accuracies, validation_accuracies):
    plt.figure(figsize=(8, 6))
    plt.plot(training_accuracies, label='Training Accuracy')
    plt.plot(validation_accuracies, label='Validation Accuracy')
    plt.title('Training and Validation Accuracy')
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    plt.legend()
    plt.savefig(f"{output_dir}/accuracy_plot.png")
    plt.close()

def plot_confusion_matrix(cm, class_labels, epoch, output_dir):
    plt.figure(figsize=(10, 8))
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_labels, yticklabels=class_labels)
    plt.title(f'Confusion Matrix for Epoch {epoch + 1}')
    plt.ylabel('True Label')
    plt.xlabel('Predicted Label')
    plt.savefig(os.path.join(output_dir, f'confusion_matrix_epoch_{epoch + 1}.png'))
    plt.close()

# Training and validation loop
train_losses = []
val_losses = []
train_accuracies = []
val_accuracies = []
metrics_df = pd.DataFrame()

for epoch in range(EPOCHS):
    model.train()
    train_loss = 0
    correct_train = 0
    total_train = 0

    for batch in train_loader:
        optimizer.zero_grad()
        batch = {k: v.to(device) for k, v in batch.items()}
        outputs = model(**batch)
        loss = outputs.loss
        loss.backward()

        # Gradient clipping
        #torch.nn.utils.clip_grad_norm_(model.parameters(), GRAD_CLIP)  # Clip gradients to stabilize training

        train_loss += loss.item()

        predictions = torch.argmax(outputs.logits, dim=-1)
        correct_train += (predictions == batch['labels']).sum().item()
        total_train += batch['labels'].size(0)
        optimizer.step()

    

    
    train_losses.append(train_loss / len(train_loader))
    train_accuracies.append(correct_train / total_train)

    model.eval()
    val_loss = 0
    correct_val = 0
    total_val = 0

    # Clear predictions and true labels before each validation loop
    all_predictions, all_true_labels = [], []  # Reset for each epoch

    with torch.no_grad():
        for batch in val_loader:
            batch = {k: v.to(device) for k, v in batch.items()}
            outputs = model(**batch)
            val_loss += outputs.loss.item()
            predictions = torch.argmax(outputs.logits, dim=-1)
            all_predictions.extend(predictions.cpu().numpy())
            all_true_labels.extend(batch['labels'].cpu().numpy())

            correct_val += (predictions == batch['labels']).sum().item()
            total_val += batch['labels'].size(0)

    val_losses.append(val_loss / len(val_loader))
    val_accuracies.append(correct_val / total_val)

    # Compute confusion matrix
    cm = confusion_matrix(all_true_labels, all_predictions)
    class_labels = [label_dict.get(i, f'Class {i}') for i in range(len(np.unique(all_true_labels)))]

    # Save confusion matrix only for the best validation loss
    plot_confusion_matrix(cm, class_labels, epoch, output_dir)

    # Print training progress per epoch
    print(f"Epoch {epoch + 1} - Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}, Val Accuracy: {val_accuracies[-1]:.4f}")

    scheduler.step()  ## HERE IS THE CHANGE
    
    
    # Early stopping check
    #if (best_val_loss - val_loss) > MIN_DELTA:  # Significant improvement
    #    best_val_loss = val_loss
    #    no_improvement_count = 0

        # Save the best model
        #torch.save(model.state_dict(), f"{output_dir}/best_model.pth")
    #else:  # No significant improvement
     #   no_improvement_count += 1
    #    if no_improvement_count >= PATIENCE:
     #       print(f"Early stopping at epoch {epoch + 1}")
      #      break

    # Calculate metrics
    balanced_acc = balanced_accuracy_score(all_true_labels, all_predictions)
    average_f1 = f1_score(all_true_labels, all_predictions, average='macro')
    weighted_f1 = f1_score(all_true_labels, all_predictions, average='weighted')
    f1_scores_per_class = f1_score(all_true_labels, all_predictions, average=None)
    precision_per_class = precision_score(all_true_labels, all_predictions, average=None, zero_division=0)
    recall_per_class = recall_score(all_true_labels, all_predictions, average=None, zero_division=0)

    # Append metrics to DataFrame with class labels
    epoch_metrics = {
        "Epoch": epoch + 1,
        "Train Loss": train_losses[-1],
        "Validation Loss": val_losses[-1],
        "Train Accuracy": train_accuracies[-1],
        "Validation Accuracy": val_accuracies[-1],
        "Balanced Accuracy": balanced_acc,
        "Average F1": average_f1,
        "Weighted F1": weighted_f1,
    }
    epoch_metrics.update({f"{label_dict[i]} F1": f1_scores_per_class[i] for i in range(len(f1_scores_per_class))})
    epoch_metrics.update({f"{label_dict[i]} Precision": precision_per_class[i] for i in range(len(precision_per_class))})
    epoch_metrics.update({f"{label_dict[i]} Recall": recall_per_class[i] for i in range(len(recall_per_class))})

    metrics_df = pd.concat([metrics_df, pd.DataFrame([epoch_metrics])], ignore_index=True)

      

# Save metrics to CSV and plot accuracies
metrics_df.to_csv(f"{output_dir}/training_metrics.csv", index=False)
plot_accuracies(train_accuracies, val_accuracies)

# Calculate final model accuracy
final_accuracy = accuracy_score(all_true_labels, all_predictions)
print(f"Final Model Accuracy: {final_accuracy:.4f}")

# Save the trained model
model_path = f"{output_dir}/trained_model.pth"
torch.save(model.state_dict(), model_path)
print(f"Model saved to {model_path}")