Augmentation
In [ ]:
from google.colab import drive
drive.mount('/content/drive')
Mounted at /content/drive
Segmented Dataset Load & Preview
In [ ]:
!zip -r -q /content/dataset.zip "/content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul"
In [ ]:
!ls -lh /content/dataset.zip
-rw-r--r-- 1 root root 4.4G Oct 14 13:02 /content/dataset.zip
In [ ]:
import zipfile
import os
ZIP_PATH = '/content/dataset.zip' # Verify your path
def show_zip_tree(zip_path):
if not os.path.exists(zip_path):
print(f"File not found: {zip_path}")
return
folder_counts = {}
grand_total = 0
with zipfile.ZipFile(zip_path, 'r') as z:
namelist = z.namelist()
# Check total entries in zip
if not namelist:
print("The zip file is empty!")
return
for name in namelist:
if name.endswith('/') or name.startswith('__MACOSX'):
continue
grand_total += 1
# Extract folder path
folder = os.path.dirname(name)
folder = folder if folder else "(Root)"
folder_counts[folder] = folder_counts.get(folder, 0) + 1
print(f"📁 {os.path.basename(zip_path)}/\n" + "─" * 40)
for folder in sorted(folder_counts.keys()):
print(f"├── 📂 {folder} --> ({folder_counts[folder]} files)")
print("─" * 40)
print(f"Grand Total Files: {grand_total}\n" + "─" * 40)
show_zip_tree(ZIP_PATH)
📁 dataset.zip/ ──────────────────────────────────────── ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Bottle_Gourd_Flower_Blossom_Blight --> (62 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Bottle_Gourd_Flower_Healthy --> (71 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Bottle_Gourd_Fruit_Borer --> (87 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Bottle_Gourd_Fruit_Healthy --> (88 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Bottle_Gourd_Leaf_Healthy --> (99 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Bottle_Gourd_Leaf_Wilt --> (85 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Eggplant_Fruit_Borer --> (62 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Eggplant_Fruit_Healthy --> (104 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Eggplant_Leaf_Healthy --> (222 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Eggplant_Leaf_Mosaic_Virus --> (104 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Eggplant_Leaf_Wilt --> (72 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Okra_Leaf_Healthy --> (77 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Okra_Leaf_Yellow_Vein_Mosaic_Virus --> (88 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Papaya_Fruit_Anthracnose --> (89 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Papaya_Fruit_Healthy --> (87 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Papaya_Leaf_Curl_Virus --> (90 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Papaya_Leaf_Healthy --> (120 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Papaya_Leaf_Mosaic_Virus --> (65 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Papaya_Leaf_Spot --> (106 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Pointed_Gourd_Fruit_Healthy --> (78 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Pointed_Gourd_Fruit_Rot --> (52 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Pointed_Gourd_Leaf_Healthy --> (113 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Pointed_Gourd_Leaf_Potassium_Deficiency --> (83 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Teasel_Gourd_Fruit_Healthy --> (134 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Teasel_Gourd_Fruit_Rot --> (120 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Teasel_Gourd_Leaf_Blight --> (61 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Teasel_Gourd_Leaf_Healthy --> (149 files) ├── 📂 content/drive/MyDrive/Segmented_Dataset_Cleaned_Ratul/Teasel_Gourd_Leaf_Spot --> (62 files) ──────────────────────────────────────── Grand Total Files: 2630 ────────────────────────────────────────
Data Splitting (70:15:15 Distribution)
In [ ]:
import os
import zipfile
import random
from collections import defaultdict
from tqdm.notebook import tqdm
# 1. Zip file path ebong output directory
ZIP_PATH = '/content/dataset.zip'
OUTPUT_DIR = '/content/Dataset_Split_70_15_15'
# Split ratio
TRAIN_RATIO = 0.70
VAL_RATIO = 0.15
TEST_RATIO = 0.15
# Reproducibility
RANDOM_SEED = 42
random.seed(RANDOM_SEED)
VALID_EXTS = ('.jpg', '.jpeg', '.png', '.bmp', '.webp', '.tiff', '.tif')
if not os.path.exists(ZIP_PATH):
raise FileNotFoundError(f"'{ZIP_PATH}' file ti pawa jayni!")
# 2. Zip file scan kore class onujayi file organize kora
class_files = defaultdict(list)
print("Scanning ZIP file...")
with zipfile.ZipFile(ZIP_PATH, 'r') as zf:
for member in zf.infolist():
if member.is_dir() or member.filename.startswith('__MACOSX/'):
continue
if member.filename.lower().endswith(VALID_EXTS):
parts = member.filename.strip('/').split('/')
if len(parts) >= 2:
class_name = parts[-2]
class_files[class_name].append(member)
classes = sorted(list(class_files.keys()))
print(f"Total detected classes: {len(classes)}")
# 3. Shuffle kore train/val/test-e extract kora
summary = {'train': 0, 'val': 0, 'test': 0}
with zipfile.ZipFile(ZIP_PATH, 'r') as zf:
with tqdm(total=len(classes), desc="Splitting Classes", ncols=100, unit="class") as pbar:
for cls in classes:
items = class_files[cls]
random.shuffle(items)
total_items = len(items)
train_count = int(total_items * TRAIN_RATIO)
val_count = int(total_items * VAL_RATIO)
splits = {
'train': items[:train_count],
'val': items[train_count:train_count + val_count],
'test': items[train_count + val_count:]
}
for split_name, split_items in splits.items():
target_dir = os.path.join(OUTPUT_DIR, split_name, cls)
os.makedirs(target_dir, exist_ok=True)
for member in split_items:
filename = os.path.basename(member.filename)
dest_path = os.path.join(target_dir, filename)
with zf.open(member) as src, open(dest_path, 'wb') as dst:
dst.write(src.read())
summary[split_name] += len(split_items)
pbar.set_postfix({"Class": cls[:12], "Total": total_items})
pbar.update(1)
# 4. Summary display
grand_total = summary['train'] + summary['val'] + summary['test']
print("\n" + "=" * 45)
print("Dataset Splitting Completed!")
print("=" * 45)
print(f"Train Images : {summary['train']} ({summary['train']/grand_total*100:.1f}%)")
print(f"Validation Images : {summary['val']} ({summary['val']/grand_total*100:.1f}%)")
print(f"Test Images : {summary['test']} ({summary['test']/grand_total*100:.1f}%)")
print("-" * 45)
print(f"Grand Total Images: {grand_total}")
print(f"Output Directory : {OUTPUT_DIR}")
print("=" * 45)
Scanning ZIP file... Total detected classes: 28
============================================= Dataset Splitting Completed! ============================================= Train Images : 1827 (69.5%) Validation Images : 382 (14.5%) Test Images : 421 (16.0%) --------------------------------------------- Grand Total Images: 2630 Output Directory : /content/Dataset_Split_70_15_15 =============================================
Training Set Augmentation (1,827 → 7,308 Samples)
In [ ]:
import os
import cv2
import shutil
import random
import numpy as np
import albumentations as A
from tqdm.notebook import tqdm
from google.colab import drive
# 1. Drive and Paths
if not os.path.exists('/content/drive'):
drive.mount('/content/drive')
DATASET_DIR = '/content/Dataset_Split_70_15_15'
TRAIN_DIR = os.path.join(DATASET_DIR, 'train')
AUGMENTED_TRAIN_DIR = os.path.join(DATASET_DIR, 'augmented_train')
os.makedirs(AUGMENTED_TRAIN_DIR, exist_ok=True)
# Reproducibility
RANDOM_SEED = 42
random.seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)
VALID_EXTS = ('.jpg', '.jpeg', '.png', '.bmp', '.webp', '.tiff', '.tif')
# 2. Final Safe Pipeline (CLAHE completely removed)
safe_augmentor = A.Compose([
# Flip
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
# Frame-safe rotation (pure black background [0,0,0])
A.ShiftScaleRotate(
shift_limit=0.03,
scale_limit=0.05,
rotate_limit=15,
border_mode=cv2.BORDER_CONSTANT,
value=0,
p=0.6
),
# Tone adjustment keeping background intact
A.RandomGamma(gamma_limit=(85, 115), p=0.4),
# Very slight blur
A.OneOf([
A.GaussianBlur(blur_limit=(3, 3), p=0.5),
A.MotionBlur(blur_limit=3, p=0.5),
], p=0.1)
])
# 3. Class Analysis
classes = [d for d in os.listdir(TRAIN_DIR) if os.path.isdir(os.path.join(TRAIN_DIR, d)) and not d.startswith('.')]
classes.sort()
class_counts = {cls: len([f for f in os.listdir(os.path.join(TRAIN_DIR, cls)) if f.lower().endswith(VALID_EXTS)]) for cls in classes}
MAX_GLOBAL_CAP = 800
MAX_AUGMENT_MULTIPLIER = 4
# 4. Execution
summary = {}
with tqdm(total=len(classes), desc="Balancing & Augmenting", ncols=100, unit="class") as pbar:
for cls in classes:
src_class_path = os.path.join(TRAIN_DIR, cls)
dst_class_path = os.path.join(AUGMENTED_TRAIN_DIR, cls)
os.makedirs(dst_class_path, exist_ok=True)
original_images = [f for f in os.listdir(src_class_path) if f.lower().endswith(VALID_EXTS)]
orig_count = len(original_images)
# Copy original images
for img_name in original_images:
shutil.copy2(os.path.join(src_class_path, img_name), os.path.join(dst_class_path, img_name))
target_for_this_class = min(MAX_GLOBAL_CAP, orig_count * MAX_AUGMENT_MULTIPLIER)
needed = target_for_this_class - orig_count
# Generate augmentations
if needed > 0 and orig_count > 0:
aug_idx = 0
while aug_idx < needed:
chosen_img_name = random.choice(original_images)
img_path = os.path.join(src_class_path, chosen_img_name)
image = cv2.imread(img_path)
if image is None:
continue
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
augmented = safe_augmentor(image=image)
aug_image_bgr = cv2.cvtColor(augmented['image'], cv2.COLOR_RGB2BGR)
base_name, ext = os.path.splitext(chosen_img_name)
save_name = f"aug_{aug_idx}_{base_name}{ext}"
save_path = os.path.join(dst_class_path, save_name)
if ext.lower() in ['.jpg', '.jpeg']:
cv2.imwrite(save_path, aug_image_bgr, [cv2.IMWRITE_JPEG_QUALITY, 95])
else:
cv2.imwrite(save_path, aug_image_bgr)
aug_idx += 1
# Fixed bug reporting
summary[cls] = max(orig_count, target_for_this_class)
pbar.set_postfix({"Class": cls[:10], "Count": summary[cls]})
pbar.update(1)
print("\n" + "=" * 50)
print(f"Total Training Images: {sum(summary.values())}")
print("=" * 50)
/usr/local/lib/python3.13/dist-packages/albumentations/core/validation.py:114: UserWarning: ShiftScaleRotate is a special case of Affine transform. Please use Affine transform instead. original_init(self, **validated_kwargs) /tmp/ipykernel_2058/2375866332.py:34: UserWarning: Argument(s) 'value' are not valid for transform ShiftScaleRotate A.ShiftScaleRotate(
================================================== Total Training Images: 7308 ==================================================
Rescaling (224×224 px)
In [ ]:
import os
import cv2
import numpy as np
from tqdm.notebook import tqdm
# Apnar current split folder path
SRC_ROOT = "/content/Dataset_Split_70_15_15"
DST_ROOT = "/content/dataset_224_preprocessed"
TARGET_SIZE = 224
def letterbox_save(src_img_path, dst_img_path, target_size=256):
image = cv2.imread(src_img_path)
if image is None:
return
h, w = image.shape[:2]
scale = min(target_size / h, target_size / w)
nh, nw = max(1, int(round(h * scale))), max(1, int(round(w * scale)))
interp = cv2.INTER_AREA if scale < 1.0 else cv2.INTER_LINEAR
resized = cv2.resize(image, (nw, nh), interpolation=interp)
canvas = np.zeros((target_size, target_size, 3), dtype=np.uint8)
top = (target_size - nh) // 2
left = (target_size - nw) // 2
canvas[top:top + nh, left:left + nw] = resized
os.makedirs(os.path.dirname(dst_img_path), exist_ok=True)
dst_img_path = os.path.splitext(dst_img_path)[0] + ".jpg"
cv2.imwrite(dst_img_path, canvas, [cv2.IMWRITE_JPEG_QUALITY, 95])
# Apnar folder e train er jaygay 'augmented_train' ache
for split in ['augmented_train', 'val', 'test']:
split_src = os.path.join(SRC_ROOT, split)
if not os.path.exists(split_src):
continue
all_files = []
for root, _, files in os.walk(split_src):
for f in files:
if f.lower().endswith(('.jpg', '.jpeg', '.png', '.bmp', '.webp')):
all_files.append(os.path.join(root, f))
print(f"\nProcessing {split} set ({len(all_files)} images)...")
for file_path in tqdm(all_files, desc=f"Resizing {split}", ncols=90):
rel_path = os.path.relpath(file_path, split_src)
save_path = os.path.join(DST_ROOT, split, rel_path)
letterbox_save(file_path, save_path, TARGET_SIZE)
print("\n✓ Shob image 224x224 e successfully resize hoyeche!")
Processing augmented_train set (7308 images)...
Processing val set (382 images)...
Processing test set (421 images)...
✓ Shob image 224x224 e successfully resize hoyeche!
Rescaling (384×384 px)
In [ ]:
import os
import cv2
import numpy as np
from tqdm.notebook import tqdm
# Apnar current split folder path
SRC_ROOT = "/content/Dataset_Split_70_15_15"
DST_ROOT = "/content/dataset_384_preprocessed"
TARGET_SIZE = 384
def letterbox_save(src_img_path, dst_img_path, target_size=256):
image = cv2.imread(src_img_path)
if image is None:
return
h, w = image.shape[:2]
scale = min(target_size / h, target_size / w)
nh, nw = max(1, int(round(h * scale))), max(1, int(round(w * scale)))
interp = cv2.INTER_AREA if scale < 1.0 else cv2.INTER_LINEAR
resized = cv2.resize(image, (nw, nh), interpolation=interp)
canvas = np.zeros((target_size, target_size, 3), dtype=np.uint8)
top = (target_size - nh) // 2
left = (target_size - nw) // 2
canvas[top:top + nh, left:left + nw] = resized
os.makedirs(os.path.dirname(dst_img_path), exist_ok=True)
dst_img_path = os.path.splitext(dst_img_path)[0] + ".jpg"
cv2.imwrite(dst_img_path, canvas, [cv2.IMWRITE_JPEG_QUALITY, 95])
# Apnar folder e train er jaygay 'augmented_train' ache
for split in ['augmented_train', 'val', 'test']:
split_src = os.path.join(SRC_ROOT, split)
if not os.path.exists(split_src):
continue
all_files = []
for root, _, files in os.walk(split_src):
for f in files:
if f.lower().endswith(('.jpg', '.jpeg', '.png', '.bmp', '.webp')):
all_files.append(os.path.join(root, f))
print(f"\nProcessing {split} set ({len(all_files)} images)...")
for file_path in tqdm(all_files, desc=f"Resizing {split}", ncols=90):
rel_path = os.path.relpath(file_path, split_src)
save_path = os.path.join(DST_ROOT, split, rel_path)
letterbox_save(file_path, save_path, TARGET_SIZE)
print("\n✓ Shob image 384x384 e successfully resize hoyeche!")
Processing augmented_train set (7308 images)...
Processing val set (382 images)...
Processing test set (421 images)...
✓ Shob image 384x384 e successfully resize hoyeche!
Rescaling (518×518 px)
In [ ]:
import os
import cv2
import numpy as np
from tqdm.notebook import tqdm
# Apnar current split folder path
SRC_ROOT = "/content/Dataset_Split_70_15_15"
DST_ROOT = "/content/dataset_518_preprocessed"
TARGET_SIZE = 518
def letterbox_save(src_img_path, dst_img_path, target_size=256):
image = cv2.imread(src_img_path)
if image is None:
return
h, w = image.shape[:2]
scale = min(target_size / h, target_size / w)
nh, nw = max(1, int(round(h * scale))), max(1, int(round(w * scale)))
interp = cv2.INTER_AREA if scale < 1.0 else cv2.INTER_LINEAR
resized = cv2.resize(image, (nw, nh), interpolation=interp)
canvas = np.zeros((target_size, target_size, 3), dtype=np.uint8)
top = (target_size - nh) // 2
left = (target_size - nw) // 2
canvas[top:top + nh, left:left + nw] = resized
os.makedirs(os.path.dirname(dst_img_path), exist_ok=True)
dst_img_path = os.path.splitext(dst_img_path)[0] + ".jpg"
cv2.imwrite(dst_img_path, canvas, [cv2.IMWRITE_JPEG_QUALITY, 95])
# Apnar folder e train er jaygay 'augmented_train' ache
for split in ['augmented_train', 'val', 'test']:
split_src = os.path.join(SRC_ROOT, split)
if not os.path.exists(split_src):
continue
all_files = []
for root, _, files in os.walk(split_src):
for f in files:
if f.lower().endswith(('.jpg', '.jpeg', '.png', '.bmp', '.webp')):
all_files.append(os.path.join(root, f))
print(f"\nProcessing {split} set ({len(all_files)} images)...")
for file_path in tqdm(all_files, desc=f"Resizing {split}", ncols=90):
rel_path = os.path.relpath(file_path, split_src)
save_path = os.path.join(DST_ROOT, split, rel_path)
letterbox_save(file_path, save_path, TARGET_SIZE)
print("\n✓ Shob image 518x518 e successfully resize hoyeche!")
Processing augmented_train set (7308 images)...
Processing val set (382 images)...
Processing test set (421 images)...
✓ Shob image 518x518 e successfully resize hoyeche!
Exporting Resized and Augmented Splits to Drive
In [ ]:
import os
# Destination folder path in Google Drive (rename if needed)
DRIVE_SAVE_PATH = "/content/drive/MyDrive/Test_Data_Leakage_Free"
os.makedirs(DRIVE_SAVE_PATH, exist_ok=True)
# The 3 folders to zip
folders = ['dataset_split_224', 'dataset_split_384', 'dataset_split_518']
for folder in folders:
print(f"📦 Zipping {folder}...")
zip_output_path = os.path.join(DRIVE_SAVE_PATH, f"{folder}.zip")
# Change directory before zipping to avoid redundant path prefixes
!cd /content && zip -r -q "{zip_output_path}" "{folder}"
print(f"✅ Saved to: {zip_output_path}\n")
print("🎉 All datasets have been successfully saved as separate zip files to Drive!")
📦 Zipping dataset_split_224... ✅ Saved to: /content/drive/MyDrive/Test_Data_Leakage_Free/dataset_split_224.zip 📦 Zipping dataset_split_384... ✅ Saved to: /content/drive/MyDrive/Test_Data_Leakage_Free/dataset_split_384.zip 📦 Zipping dataset_split_518... ✅ Saved to: /content/drive/MyDrive/Test_Data_Leakage_Free/dataset_split_518.zip 🎉 All datasets have been successfully saved as separate zip files to Drive!