testing for low vran GPUs

Author: willGuimont

.gitignore

@@ -1,6 +1,9 @@
 # IDE
 .idea/
 
+# Data
+/data/
+
 # Checkpoints
 /logs/
 /Solutionnaires/logs/

Laboratoire 8.ipynb

@@ -1125,7 +1125,7 @@
    "id": "fd5670ec",
    "metadata": {},
    "source": [
-    "block_size = 256\n",
+    "block_size = 128\n",
     "max_iters = 10_000\n",
     "eval_interval = 1000\n",
     "learning_rate = 3e-4\n",
@@ -1139,7 +1139,7 @@
     "# Model\n",
     "model = TransformerNLP(n_layer, n_embd, n_head, vocab_size, dropout, vocab_size, block_size)\n",
     "# Load a pretrained model\n",
-    "model.load_state_dict(torch.load('checkpoints/transformer_nlp.pth'))\n",
+    "model.load_state_dict(torch.load('checkpoints/transformer_nlp.pth', weights_only=True))\n",
     "model = model.to(device)\n",
     "\n",
     "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)"
@@ -1171,14 +1171,14 @@
     "            print(f\"step {i}: train loss {out['train'].item():.4f}, val loss {out['val'].item():.4f}\")\n",
     "            iter_loss.append((out['train'].item(), out['val'].item()))\n",
     "            model.train()\n",
-    "        x, y = get_batch('train', block_size, device)\n",
+    "    x, y = get_batch('train', block_size, device)\n",
     "        \n",
-    "        pred = model(x)\n",
-    "        loss = model.loss(pred, y)\n",
+    "    pred = model(x)\n",
+    "    loss = model.loss(pred, y)\n",
     "        \n",
-    "        optimizer.zero_grad(set_to_none=True)\n",
-    "        loss.backward()\n",
-    "        optimizer.step()"
+    "    optimizer.zero_grad(set_to_none=True)\n",
+    "    loss.backward()\n",
+    "    optimizer.step()"
    ],
    "outputs": [],
    "execution_count": null
@@ -1245,6 +1245,33 @@
    "outputs": [],
    "execution_count": null
   },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": [
+    "# Supprimer le dernier modele\n",
+    "def find_cuda_tensors():\n",
+    "    \"\"\"Finds all Python variables that are tensors on CUDA.\"\"\"\n",
+    "    cuda_tensors = {}\n",
+    "    for var_name, var_value in globals().items():\n",
+    "        if isinstance(var_value, torch.Tensor) and var_value.is_cuda:\n",
+    "            cuda_tensors[var_name] = var_value\n",
+    "    return cuda_tensors\n",
+    "\n",
+    "for tensor_name, tensor in find_cuda_tensors().items():\n",
+    "    del globals()[tensor_name]\n",
+    "\n",
+    "if 'model' in globals():\n",
+    "    del model\n",
+    "if 'optimizer' in globals():\n",
+    "    del optimizer\n",
+    "torch.cuda.empty_cache()\n",
+    "torch.cuda.reset_peak_memory_stats()\n"
+   ],
+   "id": "45c6409098300c1"
+  },
   {
    "cell_type": "code",
    "id": "b056f4a0",
@@ -1328,7 +1355,7 @@
    "id": "d9573bdc",
    "metadata": {},
    "source": [
-    "model.load_weights('logs/best_epoch_5.ckpt')\n",
+    "model.load_weights('logs/best_epoch_5.ckpt', weights_only=True)\n",
     "test_loss, test_acc = model.evaluate_generator(test_loader)\n",
     "print('Epoch 5: test_loss: {:.4f} test_acc: {:.2f}'.format(test_loss, test_acc))"
    ],

Solutionnaires/Laboratoire 8.ipynb

@@ -1,14 +1,14 @@
 import warnings
+
 import numpy as np
+import poutyne as pt
 import torch
 import torch.nn as nn
 from torch.utils.data import DataLoader
 from torchvision.transforms import ToTensor
 
-import poutyne as pt
-
-from deeplib.history import History
 from deeplib.datasets import train_valid_loaders
+from deeplib.history import History
 
 
 def get_model(network, optimizer=None, criterion=None, use_gpu=True, acc=True):
@@ -140,7 +140,7 @@ def train(network, optimizer, dataset, n_epoch, batch_size, *, use_gpu=True, cri
     history_callback = HistoryCallback()
     callbacks = [history_callback] if callbacks is None else [history_callback] + callbacks
 
-    if dataset.transform is None:
+    if not hasattr(dataset, 'transform') or dataset.transform is None:
         dataset.transform = ToTensor()
 
     train_loader, valid_loader = train_valid_loaders(dataset, batch_size=batch_size)

deeplib/training.py

@@ -12,3 +12,4 @@ scipy
 Pillow
 poutyne
 einops
+transformers[torch]

requirements.txt

@@ -4,6 +4,6 @@
     name='deeplib',
     version='0.1',
     packages=['deeplib'],
-    install_requires=['torch', 'torchvision', 'pandas', 'ipykernel', 'matplotlib',
-                      'scikit-learn', 'graphviz', 'ipython', 'gensim', 'numpy', 'scipy', 'Pillow', 'poutyne', 'einops'],
+    install_requires=['torch', 'torchvision', 'pandas', 'ipykernel', 'matplotlib', 'scikit-learn', 'graphviz',
+                      'ipython', 'gensim', 'numpy', 'scipy', 'Pillow', 'poutyne', 'einops', 'transformers[torch]'],
 )

setup.py

@@ -0,0 +1,68 @@
+import torch
+import torchvision
+import torchvision.transforms as transforms
+from torch.nn import CrossEntropyLoss
+from torch.optim import AdamW
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+from transformers import ViTForImageClassification
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+transform = transforms.Compose([
+    transforms.Resize((224, 224)),
+    transforms.ToTensor(),
+    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+])
+
+train_dataset = torchvision.datasets.CIFAR100(
+    root="./data", train=True, download=True, transform=transform
+)
+test_dataset = torchvision.datasets.CIFAR100(
+    root="./data", train=False, download=True, transform=transform
+)
+
+train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)
+
+# Load a pretrained ViT model from Hugging Face
+model_name = "google/vit-base-patch16-224"
+model = ViTForImageClassification.from_pretrained(
+    model_name, num_labels=100, ignore_mismatched_sizes=True,
+)
+model.to(device)
+
+optimizer = AdamW(model.parameters(), lr=5e-5)
+criterion = CrossEntropyLoss()
+
+# Fine-tuning
+epochs = 3
+for epoch in range(epochs):
+    model.train()
+    total_loss = 0
+    for images, labels in tqdm(train_loader, desc=f"Epoch {epoch + 1}/{epochs}"):
+        images, labels = images.to(device), labels.to(device)
+
+        optimizer.zero_grad()
+        outputs = model(pixel_values=images).logits
+        loss = criterion(outputs, labels)
+        loss.backward()
+        optimizer.step()
+
+        total_loss += loss.item()
+    print(f"Epoch {epoch + 1} Loss: {total_loss / len(train_loader):.4f}")
+
+# Evaluation
+model.eval()
+correct = 0
+total = 0
+with torch.no_grad():
+    for images, labels in test_loader:
+        images, labels = images.to(device), labels.to(device)
+        outputs = model(pixel_values=images).logits
+        _, predicted = torch.max(outputs, 1)
+        total += labels.size(0)
+        correct += (predicted == labels).sum().item()
+
+accuracy = correct / total
+print(f"Test Accuracy: {accuracy * 100:.2f}%")