CBAM + Basic CNN
In [ ]:
from google.colab import drive
drive.mount('/content/drive')
Mounted at /content/drive
In [ ]:
!pip install timm grad-cam --quiet
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0.0/44.5 kB ? eta -:--:-- ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 44.5/44.5 kB 2.7 MB/s eta 0:00:00 Installing build dependencies ... done Getting requirements to build wheel ... done Preparing metadata (pyproject.toml) ... done Building wheel for grad-cam (pyproject.toml) ... done
In [ ]:
import os
import zipfile
from google.colab import drive
# ============================================================
# CONFIGURATION
# ============================================================
ZIP_PATH = "/content/drive/MyDrive/Colab Notebooks/final-thesis/dataset_split_224.zip"
EXTRACT_ROOT = "/content"
# ============================================================
# CHECK ZIP FILE
# ============================================================
if not os.path.exists(ZIP_PATH):
raise FileNotFoundError(
f"❌ ZIP file not found:\n{ZIP_PATH}"
)
print("ZIP file found:")
print(ZIP_PATH)
# ============================================================
# EXTRACT
# ============================================================
print("\nExtracting dataset...")
with zipfile.ZipFile(
ZIP_PATH,
"r"
) as zip_ref:
zip_ref.extractall(
EXTRACT_ROOT
)
# ============================================================
# VERIFY
# ============================================================
DATASET_DIR = "/content/dataset_split_224"
print("\n" + "=" * 70)
print("DATASET EXTRACTION COMPLETED")
print("=" * 70)
if os.path.exists(DATASET_DIR):
print(f"✓ Dataset ready:")
print(DATASET_DIR)
for split in ["train", "val", "test"]:
split_path = os.path.join(
DATASET_DIR,
split
)
if os.path.exists(split_path):
class_count = len(
[
d for d in os.listdir(split_path)
if os.path.isdir(
os.path.join(split_path, d)
)
]
)
image_count = sum(
len(files)
for _, _, files in os.walk(split_path)
)
print(
f"✓ {split:5s} : "
f"{image_count} images | "
f"{class_count} classes"
)
else:
print(
f"⚠️ {split} folder not found"
)
else:
print("❌ Extraction failed.")
print("=" * 70)
ZIP file found: /content/drive/MyDrive/Colab Notebooks/final-thesis/dataset_split_224.zip Extracting dataset... ====================================================================== DATASET EXTRACTION COMPLETED ====================================================================== ✓ Dataset ready: /content/dataset_split_224 ✓ train : 7308 images | 28 classes ✓ val : 382 images | 28 classes ✓ test : 421 images | 28 classes ======================================================================
In [ ]:
# ============================================================
# CELL 1 — CBAM + Basic CNN: Configuration, Model & DataLoader
# From-scratch training | T4 Optimized | Reproducible | Resume-Friendly
#
# NOTE:
# Letterboxing has ALREADY been applied to the complete dataset.
# Therefore, NO additional resizing, cropping or letterboxing.
#
# This model is NOT pretrained (custom CNN), so:
# - there is NO timm config -> normalization is set below (NORM_MODE)
# - there is NO Stage 1 / Stage 2 -> one single training stage
#
# This cell defines the model architecture AND all shared paths, so
# Cell 2 (training) and Cell 3 (evaluation) can run independently.
# ============================================================
# ============================================================
# 1. IMPORTS
# ============================================================
import os
import time
import copy
import json
import random
import cv2
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from torchvision import datasets
# Avoid CPU thread oversubscription inside DataLoader workers
cv2.setNumThreads(0)
# ============================================================
# 2. REPRODUCIBILITY
# ============================================================
SEED = 42
random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
if torch.cuda.is_available():
torch.cuda.manual_seed(SEED)
torch.cuda.manual_seed_all(SEED)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
def seed_worker(worker_id):
"""Make every DataLoader worker reproducible."""
worker_seed = torch.initial_seed() % 2**32
np.random.seed(worker_seed)
random.seed(worker_seed)
loader_generator = torch.Generator()
loader_generator.manual_seed(SEED)
# ============================================================
# 3. DEVICE CONFIGURATION
# ============================================================
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("\n" + "=" * 70)
print("DEVICE CONFIGURATION")
print("=" * 70)
print(f"Device : {device}")
if torch.cuda.is_available():
gpu_name = torch.cuda.get_device_name(0)
gpu_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3
print(f"GPU : {gpu_name}")
print(f"CUDA Memory : {gpu_memory:.2f} GB")
print("=" * 70)
# ============================================================
# 4. DATASET PATHS
# ============================================================
train_dir = "/content/dataset_split_224/train"
val_dir = "/content/dataset_split_224/val"
test_dir = "/content/dataset_split_224/test"
# ============================================================
# 5. HYPERPARAMETERS
# ============================================================
IMG_SIZE = 224
BATCH_SIZE = 16
# Single-stage training from scratch (no pretrained weights)
EPOCHS = 60
PATIENCE = 12
# DataLoader
NUM_WORKERS = 2
# Optimizer (AdamW + cosine decay)
LR = 1e-3
WEIGHT_DECAY = 1e-2
# Optional light augmentation (train split only). Default OFF so the
# experiment stays identical to the other models; set True to enable
# random horizontal flip (only if flipping keeps labels valid).
# NOTE: a from-scratch CNN overfits quickly without augmentation.
USE_AUGMENTATION = False
# Mixed precision (T4 supports fp16, NOT bf16)
USE_AMP = torch.cuda.is_available()
# Normalization:
# "imagenet" -> ImageNet mean/std (fast, fine because the first conv is followed by BatchNorm)
# "dataset" -> mean/std computed once from the TRAIN split (cached on Drive)
NORM_MODE = "imagenet"
# ============================================================
# 5b. MODEL CONFIGURATION
# ============================================================
USE_CBAM = True # False -> plain Basic CNN (ablation)
CNN_CHANNELS = (32, 64, 128, 256) # channels of the 4 conv stages
CBAM_REDUCTION = 16
DROPOUT = 0.3
DISPLAY_NAME = "CBAM-BasicCNN" if USE_CBAM else "BasicCNN"
MODEL_NAME = "cbam_basic_cnn" if USE_CBAM else "basic_cnn"
ARCH_CONFIG = {
"use_cbam": USE_CBAM,
"channels": tuple(CNN_CHANNELS),
"cbam_reduction": CBAM_REDUCTION,
"dropout": DROPOUT,
}
# ============================================================
# 6. OUTPUT / CHECKPOINT DIRECTORIES & SHARED PATHS
# ============================================================
OUTPUT_DIR = (
"/content/drive/MyDrive/Colab Notebooks/"
f"final-thesis/Model_Comparison_Results_{DISPLAY_NAME}"
)
os.makedirs(OUTPUT_DIR, exist_ok=True)
CHECKPOINT_DIR = os.path.join(OUTPUT_DIR, "checkpoints")
os.makedirs(CHECKPOINT_DIR, exist_ok=True)
# Shared by Cell 2 (training) and Cell 3 (evaluation)
LATEST_CHECKPOINT = os.path.join(CHECKPOINT_DIR, "latest_checkpoint.pt")
BEST_CHECKPOINT = os.path.join(CHECKPOINT_DIR, "best_checkpoint.pt")
FINAL_MODEL_PATH = os.path.join(OUTPUT_DIR, f"{DISPLAY_NAME}_best_macro_f1.pt")
HISTORY_PATH = os.path.join(OUTPUT_DIR, f"{DISPLAY_NAME}_training_history.json")
STATS_PATH = os.path.join(OUTPUT_DIR, "dataset_stats.json")
print("\n" + "=" * 70)
print("OUTPUT CONFIGURATION")
print("=" * 70)
print(f"Model : {DISPLAY_NAME}")
print(f"Output directory : {OUTPUT_DIR}")
print(f"Checkpoint directory : {CHECKPOINT_DIR}")
print("=" * 70)
# ============================================================
# 7. CLASS NAMES (taken from ImageFolder so they always match
# the labels the Dataset will produce)
# ============================================================
_train_probe = datasets.ImageFolder(train_dir)
class_names = list(_train_probe.classes)
num_classes = len(class_names)
if num_classes == 0:
raise ValueError(f"No class folders found in:\n{train_dir}")
train_targets = np.array(_train_probe.targets)
print("\n" + "=" * 70)
print("CLASS INFORMATION")
print("=" * 70)
print(f"Number of classes : {num_classes}")
# ============================================================
# 8. CALCULATE CLASS WEIGHTS ("balanced": N / (n_classes * count))
# Computed from the SAME file list ImageFolder uses.
# ============================================================
class_counts = np.bincount(train_targets, minlength=num_classes)
total_train = int(class_counts.sum())
class_weights = np.ones(num_classes, dtype=np.float32)
for i in range(num_classes):
if class_counts[i] > 0:
class_weights[i] = total_train / (num_classes * class_counts[i])
else:
print(f"⚠️ Class '{class_names[i]}' has 0 training images.")
class_weights_tensor = torch.tensor(
class_weights, dtype=torch.float32, device=device
)
print("\nClass weights:")
print(f"{'ID':<4}| {'Class':<30} | {'Count':>6} | Weight")
for i, cls in enumerate(class_names):
print(f"{i:02d} | {cls:<30} | {class_counts[i]:>6} | {class_weights[i]:.4f}")
print("=" * 70)
# ============================================================
# 9. NORMALIZATION (no timm config for a custom model)
# ============================================================
def compute_dataset_stats(root_dir, cache_path):
"""Per-channel mean/std of the TRAIN split (RGB, 0-1). Cached as JSON."""
if os.path.exists(cache_path):
with open(cache_path, "r") as f:
stats = json.load(f)
return np.array(stats["mean"], dtype=np.float32), np.array(stats["std"], dtype=np.float32)
probe = datasets.ImageFolder(root_dir)
channel_sum = np.zeros(3, dtype=np.float64)
channel_sq_sum = np.zeros(3, dtype=np.float64)
pixel_count = 0
for i, (path, _) in enumerate(probe.samples, start=1):
img = cv2.imread(path)
if img is None:
raise RuntimeError(f"Could not read image:\n{path}")
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float64) / 255.0
channel_sum += img.sum(axis=(0, 1))
channel_sq_sum += (img ** 2).sum(axis=(0, 1))
pixel_count += img.shape[0] * img.shape[1]
if i % 2000 == 0:
print(f" stats: {i}/{len(probe.samples)} images")
mean = channel_sum / pixel_count
std = np.sqrt(np.maximum(channel_sq_sum / pixel_count - mean ** 2, 1e-12))
with open(cache_path, "w") as f:
json.dump({"mean": mean.tolist(), "std": std.tolist()}, f, indent=4)
return mean.astype(np.float32), std.astype(np.float32)
print("\n" + "=" * 70)
print("NORMALIZATION")
print("=" * 70)
if NORM_MODE == "dataset":
_mean, _std = compute_dataset_stats(train_dir, STATS_PATH)
elif NORM_MODE == "imagenet":
_mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
_std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
else:
raise ValueError("NORM_MODE must be 'imagenet' or 'dataset'.")
MODEL_MEAN = _mean.reshape(1, 1, 3)
MODEL_STD = _std.reshape(1, 1, 3)
print(f"Mode : {NORM_MODE}")
print(f"Normalization mean : {MODEL_MEAN.flatten().tolist()}")
print(f"Normalization std : {MODEL_STD.flatten().tolist()}")
print("✓ Additional resize/crop/letterbox disabled.")
print("=" * 70)
# ============================================================
# 10. MODEL DEFINITION — CBAM + BASIC CNN
# ============================================================
class ChannelAttention(nn.Module):
"""CBAM channel attention: shared MLP over avg-pooled and max-pooled descriptors."""
def __init__(self, channels, reduction=16):
super().__init__()
hidden = max(channels // reduction, 4)
self.mlp = nn.Sequential(
nn.Conv2d(channels, hidden, kernel_size=1, bias=False),
nn.ReLU(inplace=True),
nn.Conv2d(hidden, channels, kernel_size=1, bias=False),
)
def forward(self, x):
avg_out = self.mlp(F.adaptive_avg_pool2d(x, 1))
max_out = self.mlp(F.adaptive_max_pool2d(x, 1))
return torch.sigmoid(avg_out + max_out)
class SpatialAttention(nn.Module):
"""CBAM spatial attention: 7x7 conv over channel-wise avg and max maps."""
def __init__(self, kernel_size=7):
super().__init__()
self.conv = nn.Conv2d(2, 1, kernel_size=kernel_size,
padding=kernel_size // 2, bias=False)
def forward(self, x):
avg_map = x.mean(dim=1, keepdim=True)
max_map = x.max(dim=1, keepdim=True)[0]
return torch.sigmoid(self.conv(torch.cat([avg_map, max_map], dim=1)))
class CBAM(nn.Module):
"""Convolutional Block Attention Module (channel -> spatial)."""
def __init__(self, channels, reduction=16, kernel_size=7):
super().__init__()
self.channel_attention = ChannelAttention(channels, reduction)
self.spatial_attention = SpatialAttention(kernel_size)
def forward(self, x):
x = x * self.channel_attention(x)
x = x * self.spatial_attention(x)
return x
def conv_bn_relu(in_ch, out_ch):
return nn.Sequential(
nn.Conv2d(in_ch, out_ch, kernel_size=3, padding=1, bias=False),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True),
)
class CBAMBasicCNN(nn.Module):
"""
4 conv stages: [conv-bn-relu x2 (+ CBAM)], MaxPool(2) between stages,
then global average pooling -> dropout -> linear classifier.
224 -> 224 -> 112 -> 56 -> 28 (last stage works on 28x28 maps).
"""
def __init__(self, num_classes, channels=(32, 64, 128, 256),
use_cbam=True, reduction=16, dropout=0.3):
super().__init__()
stages = []
in_ch = 3
for out_ch in channels:
layers = [conv_bn_relu(in_ch, out_ch), conv_bn_relu(out_ch, out_ch)]
if use_cbam:
layers.append(CBAM(out_ch, reduction))
stages.append(nn.Sequential(*layers))
in_ch = out_ch
self.stages = nn.ModuleList(stages)
self.pool = nn.MaxPool2d(2)
self.gap = nn.AdaptiveAvgPool2d(1)
self.dropout = nn.Dropout(dropout)
self.fc = nn.Linear(in_ch, num_classes)
self._init_weights()
def _init_weights(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
elif isinstance(m, nn.BatchNorm2d):
nn.init.ones_(m.weight)
nn.init.zeros_(m.bias)
elif isinstance(m, nn.Linear):
nn.init.normal_(m.weight, 0.0, 0.01)
nn.init.zeros_(m.bias)
def get_classifier(self):
return self.fc
def forward(self, x):
last = len(self.stages) - 1
for i, stage in enumerate(self.stages):
x = stage(x)
if i < last:
x = self.pool(x)
x = self.gap(x).flatten(1)
x = self.dropout(x)
return self.fc(x)
def build_model():
return CBAMBasicCNN(
num_classes=num_classes,
channels=ARCH_CONFIG["channels"],
use_cbam=ARCH_CONFIG["use_cbam"],
reduction=ARCH_CONFIG["cbam_reduction"],
dropout=ARCH_CONFIG["dropout"],
)
# ---- Architecture sanity check (CPU, no gradients)
print("\n" + "=" * 70)
print("MODEL ARCHITECTURE CHECK")
print("=" * 70)
_check_model = build_model().eval()
with torch.no_grad():
_out = _check_model(torch.zeros(2, 3, IMG_SIZE, IMG_SIZE))
n_params = sum(p.numel() for p in _check_model.parameters())
print(f"Model : {DISPLAY_NAME}")
print(f"Architecture : {ARCH_CONFIG}")
print(f"Output shape : {tuple(_out.shape)} (expected (2, {num_classes}))")
print(f"Total parameters : {n_params:,}")
if tuple(_out.shape) != (2, num_classes):
raise ValueError("Unexpected model output shape.")
del _check_model, _out
print("=" * 70)
# ============================================================
# 11. DATASET DEFINITION
# ============================================================
class PreprocessedImageDataset(Dataset):
def __init__(self, root_dir, img_size=224, augment=False):
self.dataset = datasets.ImageFolder(root_dir)
self.targets = self.dataset.targets
self.img_size = img_size
self.augment = augment
self.mean = MODEL_MEAN
self.std = MODEL_STD
def __len__(self):
return len(self.dataset)
def __getitem__(self, idx):
img_path, label = self.dataset.samples[idx]
image = cv2.imread(img_path)
if image is None:
raise RuntimeError(f"Could not read image:\n{img_path}")
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
h, w = image.shape[:2]
if h != self.img_size or w != self.img_size:
raise ValueError(
f"Unexpected image size {w}x{h} for:\n{img_path}\n"
f"Expected preprocessed size: {self.img_size}x{self.img_size}"
)
# Optional augmentation (train only)
if self.augment and random.random() < 0.5:
image = np.ascontiguousarray(image[:, ::-1])
image = image.astype(np.float32) / 255.0
image = (image - self.mean) / self.std
image = np.ascontiguousarray(np.transpose(image, (2, 0, 1)))
image_tensor = torch.from_numpy(image).float()
label_tensor = torch.tensor(label, dtype=torch.long)
return image_tensor, label_tensor
# ============================================================
# 12. BUILD DATALOADERS
# ============================================================
def build_dataloaders():
train_ds = PreprocessedImageDataset(train_dir, IMG_SIZE, augment=USE_AUGMENTATION)
val_ds = PreprocessedImageDataset(val_dir, IMG_SIZE, augment=False)
test_ds = PreprocessedImageDataset(test_dir, IMG_SIZE, augment=False)
expected_class_to_idx = {name: idx for idx, name in enumerate(class_names)}
for split_name, ds in [("Train", train_ds), ("Validation", val_ds), ("Test", test_ds)]:
if ds.dataset.class_to_idx != expected_class_to_idx:
raise ValueError(f"{split_name} class mapping differs from training mapping.")
pin_memory = (device.type == "cuda")
persistent = (NUM_WORKERS > 0)
common = dict(
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=pin_memory,
persistent_workers=persistent,
worker_init_fn=seed_worker,
)
train_loader = DataLoader(train_ds, shuffle=True, generator=loader_generator, **common)
val_loader = DataLoader(val_ds, shuffle=False, **common)
test_loader = DataLoader(test_ds, shuffle=False, **common)
return train_loader, val_loader, test_loader
train_loader, val_loader, test_loader = build_dataloaders()
# ============================================================
# 13. DATASET SUMMARY
# ============================================================
print("\n" + "=" * 70)
print("DATASET SUMMARY")
print("=" * 70)
print(f"Train images : {len(train_loader.dataset)}")
print(f"Validation : {len(val_loader.dataset)}")
print(f"Test images : {len(test_loader.dataset)}")
print(f"\nTrain batches : {len(train_loader)}")
print(f"Val batches : {len(val_loader)}")
print(f"Test batches : {len(test_loader)}")
print(f"\nBatch size : {BATCH_SIZE}")
print(f"Image size : {IMG_SIZE} x {IMG_SIZE}")
print(f"AMP enabled : {USE_AMP}")
print(f"Workers : {NUM_WORKERS}")
print(f"Augmentation : {USE_AUGMENTATION}")
print("\nPreprocessing : Already letterboxed")
print(f"Normalization : {NORM_MODE}")
print("=" * 70)
# ============================================================
# 14. VERIFY ONE BATCH
# ============================================================
print("\n" + "=" * 70)
print("DATA PIPELINE VERIFICATION")
print("=" * 70)
sample_images, sample_labels = next(iter(train_loader))
print(f"Batch image shape : {tuple(sample_images.shape)}")
print(f"Batch label shape : {tuple(sample_labels.shape)}")
print(f"Image dtype : {sample_images.dtype}")
print(f"Label dtype : {sample_labels.dtype}")
print(f"Pixel range : [{sample_images.min():.3f}, {sample_images.max():.3f}]")
if tuple(sample_images.shape[1:]) == (3, IMG_SIZE, IMG_SIZE):
print("\n✓ Data pipeline verification passed.")
else:
print("\n⚠️ Sample shape differs from expected:")
print(f"Expected: (B, 3, {IMG_SIZE}, {IMG_SIZE})")
print("=" * 70)
======================================================================
DEVICE CONFIGURATION
======================================================================
Device : cpu
======================================================================
======================================================================
OUTPUT CONFIGURATION
======================================================================
Model : CBAM-BasicCNN
Output directory : /content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN
Checkpoint directory : /content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/checkpoints
======================================================================
======================================================================
CLASS INFORMATION
======================================================================
Number of classes : 28
Class weights:
ID | Class | Count | Weight
00 | Bottle_Gourd_Flower_Blossom_Blight | 172 | 1.5174
01 | Bottle_Gourd_Flower_Healthy | 196 | 1.3316
02 | Bottle_Gourd_Fruit_Borer | 240 | 1.0875
03 | Bottle_Gourd_Fruit_Healthy | 244 | 1.0697
04 | Bottle_Gourd_Leaf_Healthy | 276 | 0.9457
05 | Bottle_Gourd_Leaf_Wilt | 236 | 1.1059
06 | Eggplant_Fruit_Borer | 172 | 1.5174
07 | Eggplant_Fruit_Healthy | 288 | 0.9062
08 | Eggplant_Leaf_Healthy | 620 | 0.4210
09 | Eggplant_Leaf_Mosaic_Virus | 288 | 0.9062
10 | Eggplant_Leaf_Wilt | 200 | 1.3050
11 | Okra_Leaf_Healthy | 212 | 1.2311
12 | Okra_Leaf_Yellow_Vein_Mosaic_Virus | 244 | 1.0697
13 | Papaya_Fruit_Anthracnose | 248 | 1.0524
14 | Papaya_Fruit_Healthy | 240 | 1.0875
15 | Papaya_Leaf_Curl_Virus | 248 | 1.0524
16 | Papaya_Leaf_Healthy | 336 | 0.7768
17 | Papaya_Leaf_Mosaic_Virus | 180 | 1.4500
18 | Papaya_Leaf_Spot | 296 | 0.8818
19 | Pointed_Gourd_Fruit_Healthy | 216 | 1.2083
20 | Pointed_Gourd_Fruit_Rot | 144 | 1.8125
21 | Pointed_Gourd_Leaf_Healthy | 316 | 0.8259
22 | Pointed_Gourd_Leaf_Potassium_Deficiency | 232 | 1.1250
23 | Teasel_Gourd_Fruit_Healthy | 372 | 0.7016
24 | Teasel_Gourd_Fruit_Rot | 336 | 0.7768
25 | Teasel_Gourd_Leaf_Blight | 168 | 1.5536
26 | Teasel_Gourd_Leaf_Healthy | 416 | 0.6274
27 | Teasel_Gourd_Leaf_Spot | 172 | 1.5174
======================================================================
======================================================================
NORMALIZATION
======================================================================
Mode : imagenet
Normalization mean : [0.48500001430511475, 0.4560000002384186, 0.4059999883174896]
Normalization std : [0.2290000021457672, 0.2240000069141388, 0.22499999403953552]
✓ Additional resize/crop/letterbox disabled.
======================================================================
======================================================================
MODEL ARCHITECTURE CHECK
======================================================================
Model : CBAM-BasicCNN
Architecture : {'use_cbam': True, 'channels': (32, 64, 128, 256), 'cbam_reduction': 16, 'dropout': 0.3}
Output shape : (2, 28) (expected (2, 28))
Total parameters : 1,191,812
======================================================================
======================================================================
DATASET SUMMARY
======================================================================
Train images : 7308
Validation : 382
Test images : 421
Train batches : 457
Val batches : 24
Test batches : 27
Batch size : 16
Image size : 224 x 224
AMP enabled : False
Workers : 2
Augmentation : False
Preprocessing : Already letterboxed
Normalization : imagenet
======================================================================
======================================================================
DATA PIPELINE VERIFICATION
======================================================================
Batch image shape : (16, 3, 224, 224)
Batch label shape : (16,)
Image dtype : torch.float32
Label dtype : torch.int64
Pixel range : [-2.118, 2.640]
✓ Data pipeline verification passed.
======================================================================
In [ ]:
# ============================================================
# CELL 2 — CBAM + Basic CNN: From-Scratch Training
# Keras-Style Live Progress + AMP + Checkpoint + Resume
# + Early Stopping + Best Model
#
# Single training stage (no pretrained weights -> no freezing).
#
# DEPENDENCY NOTE:
# Needs CELL 1 (config, model definition, dataloaders, shared paths).
# ============================================================
import os
import sys
import time
import json
from contextlib import nullcontext
import torch
import torch.nn as nn
import torch.nn.functional as F
from sklearn.metrics import f1_score
print("\n" + "=" * 80)
print(f"🚀 Running From-Scratch Training — {DISPLAY_NAME}")
print("=" * 80)
# ============================================================
# 1. MODEL INITIALIZATION
# ============================================================
model = build_model().to(device)
n_params = sum(p.numel() for p in model.parameters())
print(f"\n✓ Model built: {DISPLAY_NAME}")
print(f"✓ Architecture: {ARCH_CONFIG}")
print(f"✓ Input size: {IMG_SIZE} × {IMG_SIZE}")
print(f"✓ Trainable parameters: {n_params:,}")
# ============================================================
# 2. LOSS FUNCTION
# - `criterion` (class-weighted) is used for TRAINING only.
# - Reported train/val loss is the plain (unweighted) CE so the
# two curves are comparable and exact per-sample averages.
# ============================================================
criterion = nn.CrossEntropyLoss(weight=class_weights_tensor)
# ============================================================
# 3. AMP GRAD SCALER & AUTOCAST CONTEXT
# ============================================================
try:
scaler = torch.amp.GradScaler("cuda", enabled=USE_AMP)
except Exception:
scaler = torch.cuda.amp.GradScaler(enabled=USE_AMP)
def autocast_context():
if USE_AMP:
return torch.autocast(device_type="cuda", dtype=torch.float16)
return nullcontext()
# ============================================================
# 4. LIVE PROGRESS DISPLAY (Keras-Style)
# ============================================================
def print_progress(current, total, start_time, train_loss, train_acc, bar_length=24):
progress = current / total if total > 0 else 0.0
filled = int(bar_length * progress)
bar = "━" * filled + "░" * (bar_length - filled)
elapsed = time.time() - start_time
remaining = (elapsed / current) * (total - current) if current > 0 else 0.0
sys.stdout.write(
f"\r{current}/{total} {bar} {progress * 100:5.1f}% "
f"- {elapsed:.0f}s - ETA {remaining:.0f}s "
f"- accuracy: {train_acc:.4f} - loss: {train_loss:.4f}"
)
sys.stdout.flush()
# ============================================================
# 5. VALIDATION FUNCTION
# ============================================================
def validate_model(model, val_loader):
model.eval()
loss_sum = 0.0
correct = 0
total = 0
y_true, y_pred = [], []
with torch.no_grad():
for images, labels in val_loader:
images = images.to(device, non_blocking=True)
labels = labels.to(device, non_blocking=True)
with autocast_context():
outputs = model(images)
outputs = outputs.float()
loss_sum += F.cross_entropy(outputs, labels, reduction="sum").item()
preds = outputs.argmax(dim=1)
correct += (preds == labels).sum().item()
total += labels.size(0)
y_true.extend(labels.cpu().numpy())
y_pred.extend(preds.cpu().numpy())
val_loss = loss_sum / max(total, 1)
val_acc = correct / max(total, 1)
macro_f1 = f1_score(y_true, y_pred, average="macro", zero_division=0)
return val_loss, val_acc, macro_f1
# ============================================================
# 6. CHECKPOINTING (best-checkpoint is model-only to keep
# Drive writes small and fast)
# ============================================================
ATOMIC_SAVE = False # keep False on Google Drive (prevents duplicate files)
def save_checkpoint(
path, model, optimizer, scheduler, scaler,
epoch, best_macro_f1, epochs_no_improve, history,
include_training_state=True
):
checkpoint = {
"model_state_dict": model.state_dict(),
"epoch": epoch,
"stage": 1,
"best_macro_f1": best_macro_f1,
"epochs_no_improve": epochs_no_improve,
"history": history,
"class_names": class_names,
"num_classes": num_classes,
"img_size": IMG_SIZE,
"batch_size": BATCH_SIZE,
"seed": SEED,
"model_name": MODEL_NAME,
"arch_config": ARCH_CONFIG,
"norm_mode": NORM_MODE,
"norm_mean": MODEL_MEAN.flatten().tolist(),
"norm_std": MODEL_STD.flatten().tolist(),
}
if include_training_state:
checkpoint["optimizer_state_dict"] = optimizer.state_dict()
checkpoint["scheduler_state_dict"] = scheduler.state_dict()
checkpoint["scaler_state_dict"] = scaler.state_dict()
if ATOMIC_SAVE:
# Temp file + rename. NOT recommended on Google Drive: Drive can keep
# the old file and create a 2nd file with the same name.
tmp_path = path + ".tmp"
torch.save(checkpoint, tmp_path)
os.replace(tmp_path, path)
else:
# Direct overwrite -> Drive keeps exactly ONE file per name
torch.save(checkpoint, path)
# ============================================================
# 7. OPTIMIZER & SCHEDULER
# ============================================================
optimizer = torch.optim.AdamW(
model.parameters(),
lr=LR,
weight_decay=WEIGHT_DECAY
)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer,
T_max=EPOCHS
)
# ============================================================
# 8. CHECKPOINT COMPATIBILITY & RESUME STATE LOADING
# ============================================================
start_epoch = 0
best_macro_f1 = 0.0
epochs_no_improve = 0
history = {"stage1": [], "stage2": []} # only "stage1" is used (kept for Cell 3 compatibility)
if os.path.exists(LATEST_CHECKPOINT):
# Load on CPU (saves GPU memory)
resume_ckpt = torch.load(LATEST_CHECKPOINT, map_location="cpu", weights_only=False)
compatible = (
resume_ckpt.get("model_name") == MODEL_NAME
and resume_ckpt.get("class_names") == class_names
and resume_ckpt.get("img_size") == IMG_SIZE
and resume_ckpt.get("arch_config") == ARCH_CONFIG
)
if not compatible:
raise RuntimeError(
"Incompatible checkpoint detected. Back up or remove "
f"latest_checkpoint.pt and best_checkpoint.pt from {CHECKPOINT_DIR}."
)
print("\n" + "=" * 80)
print("🔄 CHECKPOINT FOUND — RESUMING TRAINING")
print("=" * 80)
model.load_state_dict(resume_ckpt["model_state_dict"])
optimizer.load_state_dict(resume_ckpt["optimizer_state_dict"])
scheduler.load_state_dict(resume_ckpt["scheduler_state_dict"])
scaler.load_state_dict(resume_ckpt["scaler_state_dict"])
best_macro_f1 = resume_ckpt.get("best_macro_f1", 0.0)
epochs_no_improve = resume_ckpt.get("epochs_no_improve", 0)
history = resume_ckpt.get("history", {"stage1": [], "stage2": []})
history.setdefault("stage1", [])
history.setdefault("stage2", [])
saved_epoch = resume_ckpt["epoch"]
start_epoch = saved_epoch + 1
print(f"Saved epoch : {saved_epoch + 1}")
print(f"Best Macro F1 : {best_macro_f1:.4f}")
print(f"No improvement : {epochs_no_improve}")
print(f"\n→ Resuming from Epoch {start_epoch + 1}")
print("✓ Optimizer/scheduler/scaler restored.")
del resume_ckpt
else:
print("\n✓ No checkpoint found. Starting training from scratch.")
# ============================================================
# 9. EPOCH LOGIC
# ============================================================
def train_one_epoch(epoch):
model.train()
epoch_start = time.time()
loss_sum, correct, seen, skipped = 0.0, 0, 0, 0
n_batches = len(train_loader)
print(f"\nEpoch {epoch + 1}/{EPOCHS}")
for batch_idx, (images, labels) in enumerate(train_loader, start=1):
images = images.to(device, non_blocking=True)
labels = labels.to(device, non_blocking=True)
optimizer.zero_grad(set_to_none=True)
with autocast_context():
outputs = model(images)
outputs = outputs.float()
loss = criterion(outputs, labels)
if not torch.isfinite(loss):
skipped += 1
continue
scaler.scale(loss).backward()
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
scaler.step(optimizer)
scaler.update()
with torch.no_grad():
loss_sum += F.cross_entropy(outputs.detach(), labels, reduction="sum").item()
correct += (outputs.argmax(dim=1) == labels).sum().item()
seen += labels.size(0)
print_progress(
batch_idx, n_batches, epoch_start,
loss_sum / seen, correct / seen
)
print()
if skipped:
print(f"⚠️ Skipped {skipped} batch(es) with non-finite loss.")
if seen == 0:
raise RuntimeError("All batches were skipped (non-finite loss).")
return loss_sum / seen, correct / seen, epoch_start
def run_training(first_epoch):
"""Runs all epochs. Returns True if early stopping fired."""
global best_macro_f1, epochs_no_improve
n_batches = len(train_loader)
for epoch in range(first_epoch, EPOCHS):
train_loss, train_acc, epoch_start = train_one_epoch(epoch)
# LR actually used during this epoch (read BEFORE scheduler.step)
current_lr = optimizer.param_groups[0]["lr"]
scheduler.step()
val_loss, val_acc, macro_f1 = validate_model(model, val_loader)
epoch_time = time.time() - epoch_start
step_ms = epoch_time / n_batches * 1000
print(
f"{n_batches}/{n_batches} ━━━━━━━━━━━━━━━━━━━━━━━━ "
f"{epoch_time:.0f}s {step_ms:.0f}ms/step "
f"- accuracy: {train_acc:.4f} - loss: {train_loss:.4f} "
f"- val_accuracy: {val_acc:.4f} - val_loss: {val_loss:.4f} "
f"- val_macro_f1: {macro_f1:.4f}"
)
history["stage1"].append({
"epoch": epoch + 1,
"train_loss": train_loss,
"train_acc": train_acc,
"val_loss": val_loss,
"val_acc": val_acc,
"macro_f1": macro_f1,
"lr": current_lr,
"time_sec": epoch_time,
})
if macro_f1 > best_macro_f1:
best_macro_f1 = macro_f1
epochs_no_improve = 0
print(f"🏆 New Best Macro F1: {best_macro_f1:.4f}")
save_checkpoint(
BEST_CHECKPOINT, model, optimizer, scheduler, scaler,
epoch, best_macro_f1, epochs_no_improve, history,
include_training_state=False
)
else:
epochs_no_improve += 1
print(f"⏳ No improvement: {epochs_no_improve}/{PATIENCE}")
save_checkpoint(
LATEST_CHECKPOINT, model, optimizer, scheduler, scaler,
epoch, best_macro_f1, epochs_no_improve, history
)
with open(HISTORY_PATH, "w") as f:
json.dump(history, f, indent=4)
if epochs_no_improve >= PATIENCE:
print("\n" + "=" * 80)
print("🛑 EARLY STOPPING TRIGGERED")
print(f"No Macro F1 improvement for {PATIENCE} epochs.")
print("=" * 80)
return True
return False
# ============================================================
# 10. TRAINING
# ============================================================
print("\n" + "=" * 80)
print("🟩 TRAINING FROM SCRATCH (single stage)")
print("=" * 80)
print(f"Epochs : {EPOCHS} (patience {PATIENCE})")
print(f"LR : {LR}")
print(f"Weight decay : {WEIGHT_DECAY}")
if epochs_no_improve >= PATIENCE:
print("\n🛑 Early stopping was already triggered in a previous run. Skipping training.")
elif start_epoch >= EPOCHS:
print("\n✓ All epochs were already completed in a previous run. Skipping training.")
else:
run_training(start_epoch)
if torch.cuda.is_available():
torch.cuda.empty_cache()
# ============================================================
# 11. FINAL MODEL & ARTIFACT SAVING
# ============================================================
print("\n" + "=" * 80)
print("🏆 LOADING BEST MODEL & SAVING ARTIFACTS")
print("=" * 80)
if not os.path.exists(BEST_CHECKPOINT):
raise FileNotFoundError(f"Best checkpoint not found:\n{BEST_CHECKPOINT}")
best_ckpt = torch.load(BEST_CHECKPOINT, map_location="cpu", weights_only=False)
model.load_state_dict(best_ckpt["model_state_dict"])
best_macro_f1 = best_ckpt["best_macro_f1"]
del best_ckpt
print(f"✓ Best validation Macro F1: {best_macro_f1:.4f}")
torch.save(model.state_dict(), FINAL_MODEL_PATH)
print(f"✓ Final best model saved:\n{FINAL_MODEL_PATH}")
with open(HISTORY_PATH, "w") as f:
json.dump(history, f, indent=4)
print(f"✓ Training history saved:\n{HISTORY_PATH}")
# ============================================================
# 12. TRAINING SUMMARY
# ============================================================
print("\n" + "=" * 80)
print("✨ TRAINING COMPLETE")
print("=" * 80)
print(f"Epochs completed : {len(history['stage1'])}")
print(f"Best Validation Macro F1 : {best_macro_f1:.4f}")
print(f"Parameters : {n_params:,}")
print(f"AMP enabled : {USE_AMP}")
print(f"\nBest checkpoint : {BEST_CHECKPOINT}")
print(f"Latest checkpoint : {LATEST_CHECKPOINT}")
print(f"Training history : {HISTORY_PATH}")
print("=" * 80)
================================================================================
🚀 Running From-Scratch Training — CBAM-BasicCNN
================================================================================
✓ Model built: CBAM-BasicCNN
✓ Architecture: {'use_cbam': True, 'channels': (32, 64, 128, 256), 'cbam_reduction': 16, 'dropout': 0.3}
✓ Input size: 224 × 224
✓ Trainable parameters: 1,191,812
✓ No checkpoint found. Starting training from scratch.
================================================================================
🟩 TRAINING FROM SCRATCH (single stage)
================================================================================
Epochs : 60 (patience 12)
LR : 0.001
Weight decay : 0.01
Epoch 1/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 37s - ETA 0s - accuracy: 0.3853 - loss: 1.9930
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 38s 82ms/step - accuracy: 0.3853 - loss: 1.9930 - val_accuracy: 0.5000 - val_loss: 1.4972 - val_macro_f1: 0.4453
🏆 New Best Macro F1: 0.4453
Epoch 2/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.6062 - loss: 1.1807
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.6062 - loss: 1.1807 - val_accuracy: 0.1885 - val_loss: 5.1433 - val_macro_f1: 0.0986
⏳ No improvement: 1/12
Epoch 3/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 43s - ETA 0s - accuracy: 0.6812 - loss: 0.9298
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 44s 96ms/step - accuracy: 0.6812 - loss: 0.9298 - val_accuracy: 0.7094 - val_loss: 0.7335 - val_macro_f1: 0.7019
🏆 New Best Macro F1: 0.7019
Epoch 4/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.7295 - loss: 0.8023
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.7295 - loss: 0.8023 - val_accuracy: 0.7382 - val_loss: 0.7465 - val_macro_f1: 0.7144
🏆 New Best Macro F1: 0.7144
Epoch 5/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.7663 - loss: 0.6947
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.7663 - loss: 0.6947 - val_accuracy: 0.7775 - val_loss: 0.6702 - val_macro_f1: 0.7636
🏆 New Best Macro F1: 0.7636
Epoch 6/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.7945 - loss: 0.6157
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.7945 - loss: 0.6157 - val_accuracy: 0.7618 - val_loss: 0.6450 - val_macro_f1: 0.7372
⏳ No improvement: 1/12
Epoch 7/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.8131 - loss: 0.5392
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.8131 - loss: 0.5392 - val_accuracy: 0.7382 - val_loss: 0.9182 - val_macro_f1: 0.7338
⏳ No improvement: 2/12
Epoch 8/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.8306 - loss: 0.4889
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.8306 - loss: 0.4889 - val_accuracy: 0.7225 - val_loss: 0.8174 - val_macro_f1: 0.6932
⏳ No improvement: 3/12
Epoch 9/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.8532 - loss: 0.4279
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.8532 - loss: 0.4279 - val_accuracy: 0.8115 - val_loss: 0.5798 - val_macro_f1: 0.7922
🏆 New Best Macro F1: 0.7922
Epoch 10/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.8704 - loss: 0.3860
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.8704 - loss: 0.3860 - val_accuracy: 0.8010 - val_loss: 0.6441 - val_macro_f1: 0.7923
🏆 New Best Macro F1: 0.7923
Epoch 11/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.8820 - loss: 0.3537
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.8820 - loss: 0.3537 - val_accuracy: 0.7880 - val_loss: 0.8020 - val_macro_f1: 0.7534
⏳ No improvement: 1/12
Epoch 12/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.8926 - loss: 0.3173
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.8926 - loss: 0.3173 - val_accuracy: 0.8534 - val_loss: 0.5017 - val_macro_f1: 0.8411
🏆 New Best Macro F1: 0.8411
Epoch 13/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9082 - loss: 0.2636
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9082 - loss: 0.2636 - val_accuracy: 0.7565 - val_loss: 0.7578 - val_macro_f1: 0.7371
⏳ No improvement: 1/12
Epoch 14/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9172 - loss: 0.2485
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9172 - loss: 0.2485 - val_accuracy: 0.8665 - val_loss: 0.5228 - val_macro_f1: 0.8543
🏆 New Best Macro F1: 0.8543
Epoch 15/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9252 - loss: 0.2260
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9252 - loss: 0.2260 - val_accuracy: 0.8115 - val_loss: 0.6590 - val_macro_f1: 0.8008
⏳ No improvement: 1/12
Epoch 16/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9267 - loss: 0.2154
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9267 - loss: 0.2154 - val_accuracy: 0.7906 - val_loss: 0.8810 - val_macro_f1: 0.7671
⏳ No improvement: 2/12
Epoch 17/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 37s - ETA 0s - accuracy: 0.9383 - loss: 0.1788
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 38s 84ms/step - accuracy: 0.9383 - loss: 0.1788 - val_accuracy: 0.8508 - val_loss: 0.5359 - val_macro_f1: 0.8388
⏳ No improvement: 3/12
Epoch 18/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 39s - ETA 0s - accuracy: 0.9502 - loss: 0.1489
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 40s 87ms/step - accuracy: 0.9502 - loss: 0.1489 - val_accuracy: 0.8482 - val_loss: 0.6149 - val_macro_f1: 0.8313
⏳ No improvement: 4/12
Epoch 19/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9514 - loss: 0.1483
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9514 - loss: 0.1483 - val_accuracy: 0.8927 - val_loss: 0.4286 - val_macro_f1: 0.8859
🏆 New Best Macro F1: 0.8859
Epoch 20/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 41s - ETA 0s - accuracy: 0.9573 - loss: 0.1280
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 42s 93ms/step - accuracy: 0.9573 - loss: 0.1280 - val_accuracy: 0.8717 - val_loss: 0.5312 - val_macro_f1: 0.8568
⏳ No improvement: 1/12
Epoch 21/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 36s - ETA 0s - accuracy: 0.9553 - loss: 0.1370
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 80ms/step - accuracy: 0.9553 - loss: 0.1370 - val_accuracy: 0.8691 - val_loss: 0.4945 - val_macro_f1: 0.8584
⏳ No improvement: 2/12
Epoch 22/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9620 - loss: 0.1144
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9620 - loss: 0.1144 - val_accuracy: 0.8953 - val_loss: 0.3919 - val_macro_f1: 0.8880
🏆 New Best Macro F1: 0.8880
Epoch 23/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 37s - ETA 0s - accuracy: 0.9628 - loss: 0.1189
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 37s 82ms/step - accuracy: 0.9628 - loss: 0.1189 - val_accuracy: 0.8743 - val_loss: 0.5206 - val_macro_f1: 0.8637
⏳ No improvement: 1/12
Epoch 24/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9702 - loss: 0.0939
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9702 - loss: 0.0939 - val_accuracy: 0.8927 - val_loss: 0.3806 - val_macro_f1: 0.8792
⏳ No improvement: 2/12
Epoch 25/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9709 - loss: 0.0856
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9709 - loss: 0.0856 - val_accuracy: 0.9005 - val_loss: 0.4098 - val_macro_f1: 0.8907
🏆 New Best Macro F1: 0.8907
Epoch 26/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 36s - ETA 0s - accuracy: 0.9761 - loss: 0.0745
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 37s 80ms/step - accuracy: 0.9761 - loss: 0.0745 - val_accuracy: 0.8953 - val_loss: 0.4778 - val_macro_f1: 0.8849
⏳ No improvement: 1/12
Epoch 27/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9781 - loss: 0.0688
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9781 - loss: 0.0688 - val_accuracy: 0.8822 - val_loss: 0.5144 - val_macro_f1: 0.8712
⏳ No improvement: 2/12
Epoch 28/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 36s - ETA 0s - accuracy: 0.9826 - loss: 0.0464
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 80ms/step - accuracy: 0.9826 - loss: 0.0464 - val_accuracy: 0.8953 - val_loss: 0.4755 - val_macro_f1: 0.8861
⏳ No improvement: 3/12
Epoch 29/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9874 - loss: 0.0425
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9874 - loss: 0.0425 - val_accuracy: 0.8822 - val_loss: 0.5175 - val_macro_f1: 0.8677
⏳ No improvement: 4/12
Epoch 30/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9874 - loss: 0.0390
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9874 - loss: 0.0390 - val_accuracy: 0.8901 - val_loss: 0.4968 - val_macro_f1: 0.8760
⏳ No improvement: 5/12
Epoch 31/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9856 - loss: 0.0465
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9856 - loss: 0.0465 - val_accuracy: 0.8691 - val_loss: 0.7002 - val_macro_f1: 0.8546
⏳ No improvement: 6/12
Epoch 32/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9871 - loss: 0.0430
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9871 - loss: 0.0430 - val_accuracy: 0.8901 - val_loss: 0.4679 - val_macro_f1: 0.8785
⏳ No improvement: 7/12
Epoch 33/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9929 - loss: 0.0224
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9929 - loss: 0.0224 - val_accuracy: 0.8979 - val_loss: 0.5330 - val_macro_f1: 0.8868
⏳ No improvement: 8/12
Epoch 34/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9910 - loss: 0.0266
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9910 - loss: 0.0266 - val_accuracy: 0.8979 - val_loss: 0.5338 - val_macro_f1: 0.8917
🏆 New Best Macro F1: 0.8917
Epoch 35/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9912 - loss: 0.0244
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9912 - loss: 0.0244 - val_accuracy: 0.8953 - val_loss: 0.4973 - val_macro_f1: 0.8829
⏳ No improvement: 1/12
Epoch 36/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9926 - loss: 0.0226
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 79ms/step - accuracy: 0.9926 - loss: 0.0226 - val_accuracy: 0.8953 - val_loss: 0.6018 - val_macro_f1: 0.8878
⏳ No improvement: 2/12
Epoch 37/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9941 - loss: 0.0212
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9941 - loss: 0.0212 - val_accuracy: 0.9162 - val_loss: 0.5077 - val_macro_f1: 0.9038
🏆 New Best Macro F1: 0.9038
Epoch 38/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9966 - loss: 0.0096
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9966 - loss: 0.0096 - val_accuracy: 0.9005 - val_loss: 0.4440 - val_macro_f1: 0.8838
⏳ No improvement: 1/12
Epoch 39/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9975 - loss: 0.0109
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9975 - loss: 0.0109 - val_accuracy: 0.8979 - val_loss: 0.4983 - val_macro_f1: 0.8824
⏳ No improvement: 2/12
Epoch 40/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9984 - loss: 0.0073
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9984 - loss: 0.0073 - val_accuracy: 0.9058 - val_loss: 0.4998 - val_macro_f1: 0.8974
⏳ No improvement: 3/12
Epoch 41/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9974 - loss: 0.0093
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9974 - loss: 0.0093 - val_accuracy: 0.8901 - val_loss: 0.5378 - val_macro_f1: 0.8788
⏳ No improvement: 4/12
Epoch 42/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9973 - loss: 0.0093
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9973 - loss: 0.0093 - val_accuracy: 0.9136 - val_loss: 0.5658 - val_macro_f1: 0.9050
🏆 New Best Macro F1: 0.9050
Epoch 43/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9975 - loss: 0.0080
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9975 - loss: 0.0080 - val_accuracy: 0.9005 - val_loss: 0.6009 - val_macro_f1: 0.8881
⏳ No improvement: 1/12
Epoch 44/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9979 - loss: 0.0068
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9979 - loss: 0.0068 - val_accuracy: 0.9058 - val_loss: 0.4945 - val_macro_f1: 0.8953
⏳ No improvement: 2/12
Epoch 45/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9985 - loss: 0.0040
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9985 - loss: 0.0040 - val_accuracy: 0.9162 - val_loss: 0.4912 - val_macro_f1: 0.9109
🏆 New Best Macro F1: 0.9109
Epoch 46/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9988 - loss: 0.0038
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9988 - loss: 0.0038 - val_accuracy: 0.9241 - val_loss: 0.5135 - val_macro_f1: 0.9137
🏆 New Best Macro F1: 0.9137
Epoch 47/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9986 - loss: 0.0039
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9986 - loss: 0.0039 - val_accuracy: 0.9188 - val_loss: 0.4597 - val_macro_f1: 0.9085
⏳ No improvement: 1/12
Epoch 48/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9992 - loss: 0.0026
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9992 - loss: 0.0026 - val_accuracy: 0.9188 - val_loss: 0.4618 - val_macro_f1: 0.9079
⏳ No improvement: 2/12
Epoch 49/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9988 - loss: 0.0037
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9988 - loss: 0.0037 - val_accuracy: 0.9162 - val_loss: 0.4686 - val_macro_f1: 0.9052
⏳ No improvement: 3/12
Epoch 50/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9992 - loss: 0.0029
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9992 - loss: 0.0029 - val_accuracy: 0.9241 - val_loss: 0.4484 - val_macro_f1: 0.9179
🏆 New Best Macro F1: 0.9179
Epoch 51/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9995 - loss: 0.0016
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9995 - loss: 0.0016 - val_accuracy: 0.9162 - val_loss: 0.4902 - val_macro_f1: 0.9089
⏳ No improvement: 1/12
Epoch 52/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9999 - loss: 0.0008
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9999 - loss: 0.0008 - val_accuracy: 0.9241 - val_loss: 0.4411 - val_macro_f1: 0.9188
🏆 New Best Macro F1: 0.9188
Epoch 53/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9996 - loss: 0.0011
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9996 - loss: 0.0011 - val_accuracy: 0.9162 - val_loss: 0.4333 - val_macro_f1: 0.9102
⏳ No improvement: 1/12
Epoch 54/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9997 - loss: 0.0013
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9997 - loss: 0.0013 - val_accuracy: 0.9188 - val_loss: 0.4428 - val_macro_f1: 0.9129
⏳ No improvement: 2/12
Epoch 55/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9999 - loss: 0.0009
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9999 - loss: 0.0009 - val_accuracy: 0.9293 - val_loss: 0.4180 - val_macro_f1: 0.9234
🏆 New Best Macro F1: 0.9234
Epoch 56/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9997 - loss: 0.0010
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9997 - loss: 0.0010 - val_accuracy: 0.9215 - val_loss: 0.4221 - val_macro_f1: 0.9162
⏳ No improvement: 1/12
Epoch 57/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9997 - loss: 0.0008
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9997 - loss: 0.0008 - val_accuracy: 0.9267 - val_loss: 0.4278 - val_macro_f1: 0.9204
⏳ No improvement: 2/12
Epoch 58/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9999 - loss: 0.0008
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9999 - loss: 0.0008 - val_accuracy: 0.9188 - val_loss: 0.4384 - val_macro_f1: 0.9131
⏳ No improvement: 3/12
Epoch 59/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9999 - loss: 0.0005
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9999 - loss: 0.0005 - val_accuracy: 0.9215 - val_loss: 0.4313 - val_macro_f1: 0.9141
⏳ No improvement: 4/12
Epoch 60/60
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 100.0% - 35s - ETA 0s - accuracy: 0.9992 - loss: 0.0020
457/457 ━━━━━━━━━━━━━━━━━━━━━━━━ 36s 78ms/step - accuracy: 0.9992 - loss: 0.0020 - val_accuracy: 0.9241 - val_loss: 0.4343 - val_macro_f1: 0.9185
⏳ No improvement: 5/12
================================================================================
🏆 LOADING BEST MODEL & SAVING ARTIFACTS
================================================================================
✓ Best validation Macro F1: 0.9234
✓ Final best model saved:
/content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/CBAM-BasicCNN_best_macro_f1.pt
✓ Training history saved:
/content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/CBAM-BasicCNN_training_history.json
================================================================================
✨ TRAINING COMPLETE
================================================================================
Epochs completed : 60
Best Validation Macro F1 : 0.9234
Parameters : 1,191,812
AMP enabled : True
Best checkpoint : /content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/checkpoints/best_checkpoint.pt
Latest checkpoint : /content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/checkpoints/latest_checkpoint.pt
Training history : /content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/CBAM-BasicCNN_training_history.json
================================================================================
In [ ]:
import os
import json
try:
with open(HISTORY_PATH, 'r') as f:
data = json.load(f)
grand_total_time = 0
grand_total_epochs = 0
print("--- Stage-by-Stage Training Time Report ---")
# Loop through each stage (e.g., stage1, stage2)
for stage_name, epochs_list in data.items():
if not epochs_list:
continue
stage_time = sum(epoch.get("time_sec", 0) for epoch in epochs_list)
stage_epochs = len(epochs_list)
stage_avg = stage_time / stage_epochs if stage_epochs > 0 else 0
grand_total_time += stage_time
grand_total_epochs += stage_epochs
# Print individual stage report
print(f"\n[{stage_name.upper()}]")
print(f" - Epochs: {stage_epochs}")
print(f" - Total Time: {stage_time:.2f} seconds ({stage_time/60:.2f} minutes)")
print(f" - Avg Time per Epoch: {stage_avg:.2f} seconds")
# Calculate overall grand total in hours, minutes, seconds
hours = int(grand_total_time // 3600)
minutes = int((grand_total_time % 3600) // 60)
seconds = grand_total_time % 3600 % 60
print("\n" + "="*40)
print("--- OVERALL TRAINING SUMMARY ---")
print(f"Total Epochs Across All Stages: {grand_total_epochs}")
if hours > 0:
print(f"Grand Total Time: {hours} hours, {minutes} minutes, {seconds:.2f} seconds")
else:
print(f"Grand Total Time: {minutes} minutes, {seconds:.2f} seconds ({grand_total_time:.2f} total seconds)")
print("="*40)
except FileNotFoundError:
print(f"Error: File not found at {HISTORY_PATH}. Please check your path variables.")
--- Stage-by-Stage Training Time Report --- [STAGE1] - Epochs: 60 - Total Time: 2180.52 seconds (36.34 minutes) - Avg Time per Epoch: 36.34 seconds ======================================== --- OVERALL TRAINING SUMMARY --- Total Epochs Across All Stages: 60 Grand Total Time: 36 minutes, 20.52 seconds (2180.52 total seconds) ========================================
In [ ]:
# ============================================================
# CELL 3 — CBAM + Basic CNN: FINAL TEST EVALUATION
# Independent / Resume-Friendly
#
# DEPENDENCY NOTE:
# Depends ONLY on CELL 1 (config, model definition, shared paths, dataloaders).
# Does NOT require CELL 2 in the same session — loads weights
# directly from best_checkpoint.pt (or the final .pt file).
# ============================================================
import os
import sys
import json
import math
import random
import subprocess
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.ticker import MaxNLocator
import torch
from sklearn.metrics import (
classification_report,
confusion_matrix,
roc_curve,
precision_recall_curve,
auc,
roc_auc_score,
f1_score,
accuracy_score,
balanced_accuracy_score,
matthews_corrcoef,
average_precision_score
)
try:
import pytorch_grad_cam
except ImportError:
subprocess.check_call([sys.executable, "-m", "pip", "install", "-q", "grad-cam"])
from pytorch_grad_cam import GradCAMPlusPlus
from pytorch_grad_cam.utils.image import show_cam_on_image
from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
# ============================================================
# 1. EVALUATION DIRECTORY SETUP
# ============================================================
EVAL_DIR = os.path.join(OUTPUT_DIR, "Evaluation")
os.makedirs(EVAL_DIR, exist_ok=True)
n_classes = len(class_names)
print("=" * 80)
print(f"FINAL TEST EVALUATION (Independent) — {DISPLAY_NAME}")
print("=" * 80)
# ============================================================
# 2. LOAD TRAINING HISTORY (if available)
# ============================================================
print("\n" + "=" * 80)
print("LOADING TRAINING HISTORY")
print("=" * 80)
history = {"stage1": [], "stage2": []}
if os.path.exists(HISTORY_PATH):
with open(HISTORY_PATH, "r") as f:
history = json.load(f)
print(f"✓ Training history loaded from:\n{HISTORY_PATH}")
else:
print("⚠️ Training history file not found. Curves will be skipped.")
stage1_history = history.get("stage1", [])
stage2_history = history.get("stage2", [])
# ============================================================
# 3. TRAINING CURVES
# ============================================================
rows = []
for r in stage1_history:
rows.append({"stage": 1, "epoch": int(r["epoch"]), **{k: r.get(k) for k in ("train_loss", "val_loss", "train_acc", "val_acc")}})
stage1_completed = len(stage1_history)
for r in stage2_history:
rows.append({"stage": 2, "epoch": stage1_completed + int(r["epoch"]), **{k: r.get(k) for k in ("train_loss", "val_loss", "train_acc", "val_acc")}})
history_df = pd.DataFrame(rows)
# Ensure epochs strictly start from 1 and are sequential
if len(history_df) > 0:
history_df = history_df.sort_values("epoch").reset_index(drop=True)
history_df["epoch"] = range(1, len(history_df) + 1)
def plot_curve(train_col, val_col, ylabel, title, fname, ylim=None):
fig, ax = plt.subplots(figsize=(12, 7))
ax.plot(history_df["epoch"], history_df[train_col], marker="o", label=f"Training {ylabel}")
ax.plot(history_df["epoch"], history_df[val_col], marker="s", label=f"Validation {ylabel}")
if stage1_completed > 0 and len(stage2_history) > 0:
# Adjust vertical line position based on the new sequential index
ax.axvline(stage1_completed + 0.5, color="gray", linestyle=":", label="Stage 1 → Stage 2")
ax.set_title(f"{title} — {DISPLAY_NAME}", fontsize=16, fontweight="bold")
ax.set_xlabel("Epoch")
total_epochs = len(history_df)
# Set X-axis ticks: if epochs are very many, show ticks at smart intervals to prevent overlap
if total_epochs > 20:
# Show every 2nd or 5th epoch if there are too many epochs
step = max(1, total_epochs // 15)
xticks = list(range(1, total_epochs + 1, step))
if total_epochs not in xticks:
xticks.append(total_epochs) # Make sure the last epoch is always shown
ax.set_xticks(xticks)
else:
ax.set_xticks(range(1, total_epochs + 1))
ax.set_xlim(0.8, total_epochs + 0.2)
# Rotate x-axis labels slightly if they feel crowded
plt.xticks(rotation=0) # Change to 45 if you still find them close
ax.set_ylabel(ylabel)
if ylim:
ax.set_ylim(*ylim)
ax.grid(linestyle="--", alpha=0.4)
ax.legend()
fig.savefig(os.path.join(EVAL_DIR, fname), dpi=300, bbox_inches="tight")
plt.show()
plt.close(fig)
if len(history_df) > 0:
plot_curve("train_loss", "val_loss", "Loss", "Training vs Validation Loss",
f"{DISPLAY_NAME}_Training_vs_Validation_Loss.png")
if not history_df["val_acc"].isna().all():
plot_curve("train_acc", "val_acc", "Accuracy", "Training vs Validation Accuracy",
f"{DISPLAY_NAME}_Training_vs_Validation_Accuracy.png", ylim=(0, 1.05))
# ============================================================
# 4. MODEL INITIALIZATION & CHECKPOINT LOADING
# ============================================================
print("\n" + "=" * 80)
print(f"INITIALIZING MODEL ({DISPLAY_NAME}) & LOADING CHECKPOINT")
print("=" * 80)
model = build_model()
best_val_f1 = float("nan")
if os.path.exists(BEST_CHECKPOINT):
ckpt = torch.load(BEST_CHECKPOINT, map_location="cpu", weights_only=False)
model.load_state_dict(ckpt["model_state_dict"])
best_val_f1 = ckpt.get("best_macro_f1", float("nan"))
del ckpt
print("✓ Loaded model weights from best_checkpoint.pt")
print(f"Saved Validation Macro F1: {best_val_f1:.4f}")
elif os.path.exists(FINAL_MODEL_PATH):
model.load_state_dict(torch.load(FINAL_MODEL_PATH, map_location="cpu", weights_only=True))
print("✓ Loaded model weights from final model path.")
else:
raise FileNotFoundError(
f"No saved model found at:\n1) {BEST_CHECKPOINT}\n2) {FINAL_MODEL_PATH}\n"
"Please ensure training was run at least once."
)
model = model.to(device)
model.eval()
# ============================================================
# 5. TEST INFERENCE
# ============================================================
print("\n" + "=" * 80)
print("RUNNING TEST INFERENCE")
print("=" * 80)
y_true, y_pred, y_probs = [], [], []
test_ds = test_loader.dataset
image_paths = [s[0] for s in test_ds.dataset.samples] # same order as loader (shuffle=False)
with torch.no_grad():
for images, labels in test_loader:
images = images.to(device, non_blocking=True)
if device.type == "cuda":
with torch.autocast(device_type="cuda", dtype=torch.float16):
outputs = model(images)
else:
outputs = model(images)
probs = torch.softmax(outputs.float(), dim=1)
y_true.extend(labels.numpy())
y_pred.extend(probs.argmax(dim=1).cpu().numpy())
y_probs.extend(probs.cpu().numpy())
y_true = np.asarray(y_true)
y_pred = np.asarray(y_pred)
y_probs = np.asarray(y_probs)
print(f"✓ Test samples evaluated: {len(y_true)}")
# ============================================================
# 6. METRICS
# ============================================================
accuracy = accuracy_score(y_true, y_pred)
balanced_acc = balanced_accuracy_score(y_true, y_pred)
macro_f1 = f1_score(y_true, y_pred, average="macro", zero_division=0)
weighted_f1 = f1_score(y_true, y_pred, average="weighted", zero_division=0)
micro_f1 = f1_score(y_true, y_pred, average="micro", zero_division=0)
mcc = matthews_corrcoef(y_true, y_pred)
# One-hot labels (works for any n_classes, incl. 2)
y_true_bin = np.eye(n_classes, dtype=int)[y_true]
support = y_true_bin.sum(axis=0)
# Per-class ROC-AUC / PR-AUC (NaN if a class has no positives or no negatives in test)
per_class_roc_auc, per_class_pr_auc = [], []
for i in range(n_classes):
if 0 < support[i] < len(y_true):
per_class_roc_auc.append(roc_auc_score(y_true_bin[:, i], y_probs[:, i]))
per_class_pr_auc.append(average_precision_score(y_true_bin[:, i], y_probs[:, i]))
else:
per_class_roc_auc.append(np.nan)
per_class_pr_auc.append(np.nan)
per_class_roc_auc = np.array(per_class_roc_auc)
per_class_pr_auc = np.array(per_class_pr_auc)
valid = ~np.isnan(per_class_roc_auc)
# Computed manually so a class missing from the test set cannot crash sklearn
macro_roc_auc = float(np.nanmean(per_class_roc_auc))
weighted_roc_auc = float(np.average(per_class_roc_auc[valid], weights=support[valid]))
micro_roc_auc = float(roc_auc_score(y_true_bin.ravel(), y_probs.ravel()))
macro_pr_auc = float(np.nanmean(per_class_pr_auc))
print("\n" + "=" * 80)
print("TEST RESULTS SUMMARY")
print("=" * 80)
print(f"Accuracy : {accuracy:.4f}")
print(f"Balanced Accuracy : {balanced_acc:.4f}")
print(f"Macro F1 : {macro_f1:.4f}")
print(f"Weighted F1 : {weighted_f1:.4f}")
print(f"Micro F1 : {micro_f1:.4f}")
print(f"Macro ROC-AUC : {macro_roc_auc:.4f}")
print(f"Weighted ROC-AUC : {weighted_roc_auc:.4f}")
print(f"Micro ROC-AUC : {micro_roc_auc:.4f}")
print(f"Macro PR-AUC : {macro_pr_auc:.4f}")
print(f"MCC : {mcc:.4f}")
# ---- Save overall metrics
overall = {
"model": DISPLAY_NAME,
"val_macro_f1_best": best_val_f1,
"test_accuracy": accuracy,
"test_balanced_accuracy": balanced_acc,
"test_macro_f1": macro_f1,
"test_weighted_f1": weighted_f1,
"test_micro_f1": micro_f1,
"test_macro_roc_auc": macro_roc_auc,
"test_weighted_roc_auc": weighted_roc_auc,
"test_micro_roc_auc": micro_roc_auc,
"test_macro_pr_auc": macro_pr_auc,
"test_mcc": mcc,
"test_samples": int(len(y_true)),
"parameters": int(sum(p.numel() for p in model.parameters())),
}
pd.DataFrame([overall]).to_csv(os.path.join(EVAL_DIR, f"{DISPLAY_NAME}_overall_metrics.csv"), index=False)
# ---- Classification report & per-class metrics
report_dict = classification_report(
y_true, y_pred, labels=np.arange(n_classes),
target_names=class_names, zero_division=0, output_dict=True
)
pd.DataFrame(report_dict).transpose().to_csv(
os.path.join(EVAL_DIR, f"{DISPLAY_NAME}_classification_report.csv")
)
per_class_df = pd.DataFrame([{
"class": cls,
"precision": report_dict[cls]["precision"],
"recall": report_dict[cls]["recall"],
"f1_score": report_dict[cls]["f1-score"],
"support": report_dict[cls]["support"],
"roc_auc": per_class_roc_auc[i],
"pr_auc": per_class_pr_auc[i],
} for i, cls in enumerate(class_names)])
per_class_df.to_csv(os.path.join(EVAL_DIR, f"{DISPLAY_NAME}_per_class_metrics.csv"), index=False)
# ---- Per-image predictions (useful for error analysis)
pred_df = pd.DataFrame({
"image_path": image_paths,
"true_label": [class_names[i] for i in y_true],
"pred_label": [class_names[i] for i in y_pred],
"confidence": y_probs.max(axis=1),
"correct": (y_true == y_pred),
})
pred_df.to_csv(os.path.join(EVAL_DIR, f"{DISPLAY_NAME}_test_predictions.csv"), index=False)
print(f"✓ Misclassified test images: {int((~pred_df['correct']).sum())}")
# ============================================================
# 7. CONFUSION MATRICES (raw + row-normalized)
# ============================================================
cm = confusion_matrix(y_true, y_pred, labels=np.arange(n_classes))
cm_norm = cm / np.clip(cm.sum(axis=1, keepdims=True), 1, None)
for data, fmt, name, title in [
(cm, "d", "confusion_matrix", "Confusion Matrix"),
(cm_norm, ".2f", "confusion_matrix_normalized", "Normalized Confusion Matrix"),
]:
plt.figure(figsize=(18, 15))
sns.heatmap(data, annot=(n_classes <= 30), fmt=fmt, cmap="Blues",
xticklabels=class_names, yticklabels=class_names)
plt.title(f"{title} — {DISPLAY_NAME}", fontsize=16, fontweight="bold")
plt.xlabel("Predicted")
plt.ylabel("True")
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.savefig(os.path.join(EVAL_DIR, f"{DISPLAY_NAME}_{name}.png"), dpi=300, bbox_inches="tight")
plt.show()
plt.close()
# ============================================================
# 8. ROC & PRECISION-RECALL CURVES
# ============================================================
plt.figure(figsize=(14, 11))
for i in range(n_classes):
if not valid[i]:
continue
fpr, tpr, _ = roc_curve(y_true_bin[:, i], y_probs[:, i])
plt.plot(fpr, tpr, linewidth=1.7, label=f"{class_names[i]} (AUC={auc(fpr, tpr):.3f})")
plt.plot([0, 1], [0, 1], "k--", linewidth=1.5)
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title(f"Multiclass ROC Curves — {DISPLAY_NAME}", fontsize=15, fontweight="bold")
plt.legend(loc="center left", bbox_to_anchor=(1.02, 0.5), fontsize=8)
plt.tight_layout()
plt.savefig(os.path.join(EVAL_DIR, f"{DISPLAY_NAME}_ROC_AUC_curves.png"), dpi=300, bbox_inches="tight")
plt.show()
plt.close()
plt.figure(figsize=(14, 11))
for i in range(n_classes):
if support[i] == 0:
continue
prec, rec, _ = precision_recall_curve(y_true_bin[:, i], y_probs[:, i])
plt.plot(rec, prec, linewidth=1.7, label=f"{class_names[i]} (AP={per_class_pr_auc[i]:.3f})")
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.title(f"Multiclass Precision-Recall Curves — {DISPLAY_NAME}", fontsize=15, fontweight="bold")
plt.legend(loc="center left", bbox_to_anchor=(1.02, 0.5), fontsize=8)
plt.tight_layout()
plt.savefig(os.path.join(EVAL_DIR, f"{DISPLAY_NAME}_PR_curves.png"), dpi=300, bbox_inches="tight")
plt.show()
plt.close()
# ============================================================
# 9. FOREGROUND-RESTRICTED EIGENCAM (CBAM + Basic CNN, uses MODEL_MEAN / MODEL_STD from Cell 1)
# The principal component is computed ONLY over object positions (padding excluded), so the map
# shows variation INSIDE the object instead of just "object vs padding".
# CNN feature maps are [B, C, H, W] -> NO reshape_transform.
# ============================================================
import cv2
import math
import random
import numpy as np
import torch
from pytorch_grad_cam import EigenCAM
from pytorch_grad_cam.utils.image import show_cam_on_image
LAYER_NAME = "stages.3.2" # try "stages.3.1", "stages.2.2" too
FG_THRESH = 0.05 # padding is black
NUM_RUNS = 3
class FgEigenCAM(EigenCAM):
"""EigenCAM whose SVD uses only foreground positions. Set cam.fg (HxW bool, image size) before each call.
Batch size must be 1."""
fg = None
def get_cam_image(self, input_tensor, target_layer, target_category,
activations, grads, eigen_smooth):
acts = activations[0] # [C, h, w]
C, h, w = acts.shape
m = cv2.resize(self.fg.astype(np.float32), (w, h), interpolation=cv2.INTER_AREA) > 0.5
if m.sum() < 4: # tiny object: fall back to plain EigenCAM
return super().get_cam_image(input_tensor, target_layer, target_category,
activations, grads, eigen_smooth)
X = np.nan_to_num(acts.reshape(C, -1).T) # [h*w, C]
mk = m.ravel()
Xc = X - X[mk].mean(axis=0, keepdims=True) # center with object-only mean
_, _, Vt = np.linalg.svd(Xc[mk], full_matrices=False)
proj = Xc @ Vt[0] # [h*w]
# orient the arbitrary sign: hot = positions with larger activation energy inside the object
energy = np.linalg.norm(X, axis=1)
if np.corrcoef(proj[mk], energy[mk])[0, 1] < 0:
proj = -proj
proj = proj.reshape(h, w)
proj = proj - proj[m].min() # non-negative inside the object
proj[~m] = 0.0 # padding stays cold
return proj[None]
model.eval()
target_layer = model.get_submodule(LAYER_NAME)
_shapes = []
_h = target_layer.register_forward_hook(lambda m, i, o: _shapes.append(tuple(o.shape)))
with torch.no_grad():
model(test_ds[0][0].unsqueeze(0).to(device))
_h.remove()
print(f"FgEigenCAM target layer: {LAYER_NAME} | output shape: {_shapes[0] if _shapes else '?'}")
# Class -> test indices, WITHOUT decoding every image
test_targets = np.array(test_ds.targets)
class_to_indices = {i: np.where(test_targets == i)[0].tolist() for i in range(n_classes)}
mean_sq = MODEL_MEAN.squeeze()
std_sq = MODEL_STD.squeeze()
N_COLS = 4
n_rows = math.ceil(n_classes / N_COLS)
cam = FgEigenCAM(model=model, target_layers=[target_layer])
try:
for run_id in range(1, NUM_RUNS + 1):
print(f"\nGenerating FgEigenCAM grid image {run_id}/{NUM_RUNS}...")
rng = random.Random(SEED + run_id)
fig, axes = plt.subplots(n_rows, N_COLS, figsize=(5 * N_COLS, 4.8 * n_rows))
axes = np.atleast_1d(axes).flatten()
for class_idx in range(n_classes):
ax = axes[class_idx]
if not class_to_indices[class_idx]:
ax.axis("off")
continue
chosen_idx = rng.choice(class_to_indices[class_idx])
img_tensor, true_label = test_ds[chosen_idx]
input_tensor = img_tensor.unsqueeze(0).to(device)
with torch.no_grad():
probs = torch.softmax(model(input_tensor).float(), dim=1)
pred_label = int(probs.argmax(dim=1).item())
confidence = float(probs[0, pred_label].item())
img_np = img_tensor.numpy().transpose(1, 2, 0) * std_sq + mean_sq
img_np = np.clip(img_np, 0, 1).astype(np.float32)
cam.fg = img_np.max(axis=2) > FG_THRESH
grayscale_cam = cam(input_tensor=input_tensor, targets=None)[0]
cam_image = show_cam_on_image(img_np, grayscale_cam, use_rgb=True)
is_correct = (pred_label == int(true_label))
ax.imshow(cam_image)
ax.set_title(
f"[{'✓' if is_correct else '✗'}] {class_names[class_idx]}\n"
f"Pred: {class_names[pred_label]} ({confidence:.1%})",
color="green" if is_correct else "red", fontsize=9, fontweight="bold")
ax.axis("off")
for idx in range(n_classes, len(axes)):
axes[idx].axis("off")
plt.tight_layout()
save_path = os.path.join(EVAL_DIR, f"{DISPLAY_NAME}_FgEigenCAM_{LAYER_NAME.replace('.', '_')}_Grid_{run_id}.png")
plt.savefig(save_path, dpi=300, bbox_inches="tight")
plt.show()
plt.close()
print(f"✓ Grid {run_id} saved: {save_path}")
finally:
cam.activations_and_grads.release()
if torch.cuda.is_available():
torch.cuda.empty_cache()
print("\n" + "=" * 80)
print("✨ INDEPENDENT EVALUATION COMPLETED SUCCESSFULLY!")
print(f"📁 Artifacts saved in: {EVAL_DIR}")
print("=" * 80)
================================================================================ FINAL TEST EVALUATION (Independent) — CBAM-BasicCNN ================================================================================ ================================================================================ LOADING TRAINING HISTORY ================================================================================ ✓ Training history loaded from: /content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/CBAM-BasicCNN_training_history.json
================================================================================ INITIALIZING MODEL (CBAM-BasicCNN) & LOADING CHECKPOINT ================================================================================ ✓ Loaded model weights from best_checkpoint.pt Saved Validation Macro F1: 0.9234 ================================================================================ RUNNING TEST INFERENCE ================================================================================ ✓ Test samples evaluated: 421 ================================================================================ TEST RESULTS SUMMARY ================================================================================ Accuracy : 0.9430 Balanced Accuracy : 0.9374 Macro F1 : 0.9386 Weighted F1 : 0.9423 Micro F1 : 0.9430 Macro ROC-AUC : 0.9987 Weighted ROC-AUC : 0.9988 Micro ROC-AUC : 0.9990 Macro PR-AUC : 0.9787 MCC : 0.9408 ✓ Misclassified test images: 24
FgEigenCAM target layer: stages.3.2 | output shape: (1, 256, 28, 28) Generating FgEigenCAM grid image 1/3...
✓ Grid 1 saved: /content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/Evaluation/CBAM-BasicCNN_FgEigenCAM_stages_3_2_Grid_1.png Generating FgEigenCAM grid image 2/3...
✓ Grid 2 saved: /content/drive/MyDrive/Colab Notebooks/final-thesis/Model_Comparison_Results_CBAM-BasicCNN/Evaluation/CBAM-BasicCNN_FgEigenCAM_stages_3_2_Grid_2.png Generating FgEigenCAM grid image 3/3...
