Skip to content

feat: Reorganize Colab notebook and update core scripts #48

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RexZ-AI wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: Reorganize Colab notebook and update core scripts #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
055a808
feat: Reorganize Colab notebook and update core scripts
google-labs-jules[bot] Jul 21, 2025
6b50c3c
feat: Add Colab notebook for referral classification
google-labs-jules[bot] Jul 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
295 changes: 295 additions & 0 deletions Diabetic_Retinopathy_Classification.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,295 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Diabetic Retinopathy Classification\n",
"\n",
"This notebook fine-tunes a model for 'Referral' / 'No Referral' diabetic retinopathy classification using the APTOS 2019 and MESSIDOR-2 datasets."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. Environment Setup"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"# --- Install Dependencies ---\n",
"print(\"\\n⏳ Installing dependencies...\")\n",
"!pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cu121 -q\n",
"!pip install timm==0.9.16 pandas==2.2.2 scikit-learn -q\n",
"!pip install gdown -q\n",
"print(\"✅ Dependencies installed.\")\n",
"\n",
"# --- Set up device ---\n",
"import torch\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"print(f\"Using device: {device}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. Download and Preprocess Datasets"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import gdown\n",
"import os\n",
"\n",
"# --- Download APTOS 2019 ---\n",
"aptos_url = 'https://drive.google.com/uc?id=162YPf4OhMVxj9TrQH0GnJv0n7z7gJWpj'\n",
"aptos_output = 'APTOS2019.zip'\n",
"if not os.path.exists(aptos_output):\n",
" print('Downloading APTOS 2019 dataset...')\n",
" gdown.download(aptos_url, aptos_output, quiet=False)\n",
" !unzip -q {aptos_output}\n",
" print('APTOS 2019 dataset downloaded and unzipped.')\n",
"else:\n",
" print('APTOS 2019 dataset already downloaded.')\n",
"\n",
"# --- Download MESSIDOR-2 ---\n",
"messidor_url = 'https://drive.google.com/uc?id=1vOLBUK9xdzNV8eVkRjVdNrRwhPfaOmda'\n",
"messidor_output = 'MESSIDOR2.zip'\n",
"if not os.path.exists(messidor_output):\n",
" print('Downloading MESSIDOR-2 dataset...')\n",
" gdown.download(messidor_url, messidor_output, quiet=False)\n",
" !unzip -q {messidor_output}\n",
" print('MESSIDOR-2 dataset downloaded and unzipped.')\n",
"else:\n",
" print('MESSIDOR-2 dataset already downloaded.')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from sklearn.model_selection import train_test_split\n",
"\n",
"# --- Preprocess APTOS 2019 ---\n",
"aptos_df = pd.read_csv('train.csv')\n",
"aptos_df['image_path'] = aptos_df['id_code'].apply(lambda x: os.path.join('train_images', x + '.png'))\n",
"\n",
"# --- Preprocess MESSIDOR-2 ---\n",
"messidor_df = pd.read_csv('messidor_data.csv')\n",
"messidor_df['image_path'] = messidor_df['image_id'].apply(lambda x: os.path.join('messidor-2', 'images', x + '.jpg'))\n",
"\n",
"# --- Combine datasets ---\n",
"combined_df = pd.concat([\n",
" aptos_df[['image_path', 'diagnosis']],\n",
" messidor_df[['image_path', 'adjudicated_dr_grade']]\n",
"], ignore_index=True)\n",
"combined_df.rename(columns={'diagnosis': 'grade', 'adjudicated_dr_grade': 'grade'}, inplace=True)\n",
"\n",
"# --- Create binary labels ---\n",
"# Referral: grade >= 2\n",
"# No Referral: grade < 2\n",
"combined_df['label'] = combined_df['grade'].apply(lambda x: 1 if x >= 2 else 0)\n",
"\n",
"# --- Split data ---\n",
"train_df, val_df = train_test_split(combined_df, test_size=0.2, stratify=combined_df['label'], random_state=42)\n",
"\n",
"print(f'Training samples: {len(train_df)}')\n",
"print(f'Validation samples: {len(val_df)}')\n",
"print(train_df['label'].value_counts())\n",
"print(val_df['label'].value_counts())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3. Implement Data Loading"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from torch.utils.data import Dataset, DataLoader\n",
"from torchvision import transforms\n",
"from PIL import Image\n",
"\n",
"class DRDataset(Dataset):\n",
" def __init__(self, df, transform=None):\n",
" self.df = df\n",
" self.transform = transform\n",
"\n",
" def __len__(self):\n",
" return len(self.df)\n",
"\n",
" def __getitem__(self, idx):\n",
" image_path = self.df.iloc[idx]['image_path']\n",
" image = Image.open(image_path).convert('RGB')\n",
" label = self.df.iloc[idx]['label']\n",
"\n",
" if self.transform:\n",
" image = self.transform(image)\n",
"\n",
" return image, label\n",
"\n",
"data_transforms = {\n",
" 'train': transforms.Compose([\n",
" transforms.Resize((256, 256)),\n",
" transforms.RandomHorizontalFlip(),\n",
" transforms.RandomRotation(10),\n",
" transforms.ToTensor(),\n",
" transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
" ]),\n",
" 'val': transforms.Compose([\n",
" transforms.Resize((256, 256)),\n",
" transforms.ToTensor(),\n",
" transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
" ]),\n",
"}\n",
"\n",
"train_dataset = DRDataset(train_df, transform=data_transforms['train'])\n",
"val_dataset = DRDataset(val_df, transform=data_transforms['val'])\n",
"\n",
"train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=2)\n",
"val_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=2)\n",
"\n",
"print(f'Train loader: {len(train_loader)} batches')\n",
"print(f'Validation loader: {len(val_loader)} batches')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 4. Enhance Fine-Tuning Process"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"from torchvision import models\n",
"\n",
"# --- Define the model ---\n",
"model = models.resnet50(pretrained=True)\n",
"num_ftrs = model.fc.in_features\n",
"model.fc = nn.Linear(num_ftrs, 2) # Binary classification\n",
"model = model.to(device)\n",
"\n",
"# --- Define loss function and optimizer ---\n",
"criterion = nn.CrossEntropyLoss()\n",
"optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
"\n",
"print(\"Model, loss function, and optimizer are ready.\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 5. Train and Evaluate the Model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
"import numpy as np\n",
"\n",
"def train_one_epoch(model, loader, criterion, optimizer, device):\n",
" model.train()\n",
" running_loss = 0.0\n",
" for images, labels in loader:\n",
" images, labels = images.to(device), labels.to(device)\n",
"\n",
" optimizer.zero_grad()\n",
" outputs = model(images)\n",
" loss = criterion(outputs, labels)\n",
" loss.backward()\n",
" optimizer.step()\n",
"\n",
" running_loss += loss.item() * images.size(0)\n",
"\n",
" epoch_loss = running_loss / len(loader.dataset)\n",
" return epoch_loss\n",
"\n",
"def evaluate_model(model, loader, criterion, device):\n",
" model.eval()\n",
" running_loss = 0.0\n",
" all_preds, all_labels = [], []\n",
"\n",
" with torch.no_grad():\n",
" for images, labels in loader:\n",
" images, labels = images.to(device), labels.to(device)\n",
"\n",
" outputs = model(images)\n",
" loss = criterion(outputs, labels)\n",
" running_loss += loss.item() * images.size(0)\n",
"\n",
" _, preds = torch.max(outputs, 1)\n",
" all_preds.extend(preds.cpu().numpy())\n",
" all_labels.extend(labels.cpu().numpy())\n",
"\n",
" epoch_loss = running_loss / len(loader.dataset)\n",
" accuracy = accuracy_score(all_labels, all_preds)\n",
" precision = precision_score(all_labels, all_preds)\n",
" recall = recall_score(all_labels, all_preds)\n",
" f1 = f1_score(all_labels, all_preds)\n",
"\n",
" return epoch_loss, accuracy, precision, recall, f1\n",
"\n",
"num_epochs = 10\n",
"best_accuracy = 0.0\n",
"\n",
"for epoch in range(num_epochs):\n",
" train_loss = train_one_epoch(model, train_loader, criterion, optimizer, device)\n",
" val_loss, val_acc, val_prec, val_rec, val_f1 = evaluate_model(model, val_loader, criterion, device)\n",
"\n",
" print(f'Epoch {epoch+1}/{num_epochs} | Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.4f} | Val Precision: {val_prec:.4f} | Val Recall: {val_rec:.4f} | Val F1: {val_f1:.4f}')\n",
"\n",
" if val_acc > best_accuracy:\n",
" best_accuracy = val_acc\n",
" torch.save(model.state_dict(), 'best_model.pth')\n",
" print('Best model saved.')"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Loading