mp_embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UIcmHNArFv-F"
   },
   "source": [
    "# Version 1 - Populate Pinecone with movie poster embeddings and metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "828je9yXOUag"
   },
   "source": [
    "V1 movie poster dataset sourced from Hugging Face:\n",
    "https://huggingface.co/datasets/pinecone/movie-posters\n",
    "\n",
    "V2 sources movies using TMDB API's popular endpoint (called in August 2024), and is a list of more popular/modern movies (and movie posters of better image quality)\n",
    "https://developer.themoviedb.org/reference/movie-popular-list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YhlqLV5VLMRw",
    "outputId": "b8f1e009-833f-438a-86e3-2cdb7959a1ac"
   },
   "outputs": [],
   "source": [
    "#!pip install datasets\n",
    "!pip install pinecone-client\n",
    "#!pip install tmdbv3api"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dsw5d4bOpL3W"
   },
   "source": [
    "CLIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 369,
     "referenced_widgets": [
      "0c6a2f0d65a04ed5b60f4b5af763cc16",
      "b30553a9823e4b8d9c44b78f97f1fdff",
      "8104a191863b4678832ee5d52116a3b4",
      "a4cf443150464fb3996ff425bf7c1a2f",
      "7bfd7c34832e4fe18c44e6a6f61f6615",
      "bfc884f6923a42a0988d17cd2aa5260e",
      "be3cb32dc5734ce1adfbc7fe6811b8ed",
      "0f361f0f888e4b14bf2f78be14cba43f",
      "a51f05fb1ab5499db9bb3d232253e3f1",
      "4cfe221e52e04775bf5e7a9b7a7446f7",
      "3b14a8f8131b4163a9f6e2d5e35d01c3",
      "b1c0d0ea39584c208ac1800be544e48b",
      "dc2446086a314fb6b2fe543dd82eaf1c",
      "b65e183426cc48b09121a54188eef466",
      "01a9776343454f899ee15d27b2140040",
      "74a9c7bf2f25493a8ce228d01a5a1f5f",
      "150f3ee386b44f9880614ec3ac6d03c5",
      "2fc9bd0e286d4140819ea3aaa7b22b2e",
      "5c770a9d934a4b7bb46d40c0be871e6c",
      "41bfbee4bd5e423f9603a8c1279b1c12",
      "4cf97034d0354044b0ee0d9fed14a7cd",
      "9f5152e0e32e403db1527e5a8ebc2edd",
      "f0e6b889f1674235831c5d011935110a",
      "3f239103a42f44b98143baaeeb466206",
      "3c56d664371740f680ac8071b5c63429",
      "67d1528b96c6481aa9e2b291e4d6fc79",
      "210320508b6845dc95624e3a6b2e7510",
      "ee00f83651a2436cb56997d3af5e2688",
      "209a705801754a3a88ffde3dd230d221",
      "6eebaf3ae6b14050b4703b7bd388a8f3",
      "16f7e1edf6ac417abb05212bfd34795d",
      "58e220c6a4d2482badea26a9e986543a",
      "673ede9190ff48f1aa36c42c268d0d98",
      "65ff2882f5ae4608a99c50d3ef4f34fd",
      "a9b59268d8ee47098a1c370a92c7d563",
      "a92267f8c11044d0b439d705d9e2d085",
      "9850422b9cbe48f4b95e05da70c04835",
      "1e6831d26b99414fb2f6fe1530e4de33",
      "42f1cfdc7188462492bba2486f16cfd7",
      "df0c2567d02c4fd99e41ef9081392774",
      "e6468c3792cf42ac863db149a1581cc3",
      "b67341f008914c858cb61f4474f86489",
      "59986b85476c445ab981178a4125a4a7",
      "912d5dc0667d4c49bba98dee0ee6ebc2",
      "a9e2343086b04528adbb77c719111e6e",
      "073ed1915f874eb3b2b3e43aac5c64a6",
      "3e99a1e14fd34bb5bb28e25ac55137f6",
      "6ec02fcf8ce445ffa42538e5e64f60f8",
      "a8ff2f79e392471fb27832e643c7d4a5",
      "0fb2e75b92064d0894a791aeb0593300",
      "0b6c357a19ce40b4995bcfcc07771e1f",
      "7aec5827c98e4739ac5a3a5d96e4cd63",
      "c7d5cc5c0aec4a088dea338fadc675c8",
      "a744c712c7d740a6b57b572e9a397b56",
      "4517c019c60c4af186a37165af828719",
      "6210fc43b1644762b2e8531eb02030eb",
      "89a41f2bccd6416ba7ef4d5d3f77e22b",
      "c6d819546dea457d8e842cdcc6aea07e",
      "4f6b138146c14bda89aaa60f58b29fc1",
      "46f3e9a8b1e240caab7b4389ad164c6e",
      "8a3f96709f4f47dc925a3de8c03204ce",
      "1729768148e04f0daf1d035469c9f149",
      "c02705fd44e4433296e1b13e15541796",
      "7bf38fa700074b8088b26676776fa629",
      "5123794d7de74fccb7f03ad3d57028ca",
      "98d949ec8d4c43f6aa19e87047a34d8c",
      "cbcb53128c124ad5a5ffd09ec7647749",
      "46df5e6e7a5d4f10806841cc80d2bc8f",
      "c02a699c483044b2836548cd99a56819",
      "f5954322a46940109c5e8fd4a7f2196b",
      "545e006e1e464094acea22e4352d817d",
      "cf86208cc3ed4491b715aedf22ef3f5d",
      "d982ed4c1e454da5b0c6c42ab4a75901",
      "ad7a932a8d214875a15670e236bd7c06",
      "d6a7f87d37c04b3296fd646f6d7378d2",
      "4664b946c89c4f77a8e55bce46558178",
      "9e79b179ce0e4e83971057205c21b7ee",
      "085c58d895484022a25a6d2a13ae8668",
      "0616e141b1d043c0af1a46a389d96c99",
      "7ae99c8f44e54baf8e2c462082d33135",
      "5986bbc09696498f9f99eb733417b673",
      "106116adeb8e4d7d9262f6b04463d582",
      "e5cf6d45fad448839a1f63b9ad81fe08",
      "17b7d2414d9c45aca3f16298447c32b5",
      "7d1fe9bc81e54c73915f3a2a50be6bbd",
      "1784daa459b740a0aeebe6d562e53030",
      "50caa663de9c433a827a4c7d17fd28d3",
      "34241bbbe299485e89095eb07eedcac3",
      "dd57dba95f224358b3a84b5d1070eab6",
      "06eb3af1b5e24fe498975822d375e790",
      "3908867de57440b58c53ec336b58f4b1",
      "18e82cf6a751468d8fb66d23ea3ef6be",
      "7949451930ce41c2ae4ad784fa1e9deb",
      "d73460ae031948e1a6dec6183498c9d0",
      "b18c55b81b0d480da1dcda62b772a302",
      "3391af55149d4a27913358b7af3e7714",
      "409e9fe61ebf469291f3384488efb3c5",
      "13c71037bc3d442e93cba3040aa1fe72",
      "cc99c99803c142629280ed7f5e22d468",
      "9f4c0c01988948439ed962bafba783fb",
      "4133ed2629b4439cb356d664d0080e30",
      "1dada43b539a4a4db5be6095497d807a",
      "0a8b82b536d34c699b8e2831e7bc0159",
      "4eefc98b339a44f3ab25417c5f2fcb71",
      "6cc72ec9cb7a421fa99d75efef16581f",
      "e53108500c8a4baba59d515ea42c1d28",
      "f8624c6b01fe495c99403173ae7e4bf6",
      "c0fd4828aabd49608337f7141040539b",
      "585712f34f5f42efa23ee05d577832e6",
      "e2fc9ffd44774ed39b5a04d7e7de0e15",
      "117393361d364995ad3e77752f189193",
      "21e2272b75d4450db10fe5370cd53f95",
      "0868fae5c2f047a582b0514e86a47fab",
      "ee668bcc8759498d8d570f5b1f85a4c5",
      "c1ea010ef42e4784a491afe1d60b692e",
      "cc616aed30d44a228de8f98ca17672ff",
      "fa3a757e9e44412caebc25271938ca40",
      "384ac9b59b6d4e538cdd55da3b1fea7c",
      "758587c3db55455db62599f4f284c75e",
      "9b0094e3e06d40e78b1ebed834d99a31",
      "abf8066e8b574d6bb786c17c3efdbb30"
     ]
    },
    "id": "YSJ7P6eMKePP",
    "outputId": "bf5b1fa4-9aec-4aa7-ec5f-a7afdf0af901"
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import torch\n",
    "from transformers import CLIPProcessor, CLIPModel\n",
    "from PIL import Image\n",
    "import requests\n",
    "from io import BytesIO\n",
    "import pinecone\n",
    "from tmdbv3api import TMDb, Find\n",
    "from google.colab import userdata\n",
    "import time\n",
    "from pinecone import Pinecone\n",
    "from pinecone import ServerlessSpec\n",
    "from google.colab import userdata\n",
    "\n",
    "# Configuration Variables\n",
    "# MODEL_NAME = \"openai/clip-vit-base-patch32\"\n",
    "# PROCESSOR_NAME = \"openai/clip-vit-base-patch32\"\n",
    "# NAMESPACE = \"CLIP\"\n",
    "\n",
    "MODEL_NAME = \"openai/clip-vit-base-patch32\"\n",
    "PROCESSOR_NAME = \"openai/clip-vit-base-patch32\"\n",
    "NAMESPACE = \"CLIP\"\n",
    "\n",
    "# Initialize Pinecone\n",
    "pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "index = pc.Index(\"movie-posters\")\n",
    "\n",
    "# Initialize TMDB API client\n",
    "tmdb = TMDb()\n",
    "tmdb.api_key = userdata.get(\"TMDB_API_KEY\")\n",
    "\n",
    "# Initialize the Find class\n",
    "find = Find()\n",
    "\n",
    "# Load the Pinecone movie poster dataset\n",
    "ds = load_dataset(\"pinecone/movie-posters\")\n",
    "\n",
    "# Initialize CLIP model and processor\n",
    "model = CLIPModel.from_pretrained(MODEL_NAME)\n",
    "processor = CLIPProcessor.from_pretrained(PROCESSOR_NAME)\n",
    "\n",
    "# Function to fetch an image from a URL and get its embedding\n",
    "def get_image_embedding(poster_url):\n",
    "    response = requests.get(poster_url)\n",
    "    image = Image.open(BytesIO(response.content))\n",
    "    inputs = processor(images=image, return_tensors=\"pt\")\n",
    "    with torch.no_grad():\n",
    "        embedding = model.get_image_features(**inputs)\n",
    "    return embedding.squeeze().numpy()\n",
    "\n",
    "# Function to fetch movie metadata from TMDB API using IMDb ID\n",
    "def get_movie_metadata(imdb_id):\n",
    "    time.sleep(0.02)  # 50 requests per second = 1 request every 0.02 seconds so we don't overload TMDB API\n",
    "    try:\n",
    "        # Use the Find class to search for the movie by IMDb ID\n",
    "        results = find.find(imdb_id, 'imdb_id')\n",
    "        if results.movie_results:\n",
    "            movie = results.movie_results[0]  # Take the first result\n",
    "            return {\n",
    "                'title': movie.title,\n",
    "                'overview': movie.overview,\n",
    "                'release_date': movie.release_date\n",
    "            }\n",
    "        else:\n",
    "            print(f\"No metadata found for IMDb ID {imdb_id}\")\n",
    "            return None\n",
    "    except Exception as e:\n",
    "        print(f\"Error fetching metadata for IMDb ID {imdb_id}: {e}\")\n",
    "        return None\n",
    "\n",
    "# Process the dataset and upload embeddings with metadata to Pinecone\n",
    "def process_and_upload():\n",
    "    for item in ds['train']:\n",
    "        imdb_id = item['imdbId']\n",
    "        poster_url = item['poster']\n",
    "\n",
    "        # Fetch movie metadata\n",
    "        metadata = get_movie_metadata(imdb_id)\n",
    "        if metadata is None:\n",
    "            continue\n",
    "\n",
    "        # Add poster URL from dataset to metadata\n",
    "        metadata['poster_url'] = poster_url\n",
    "\n",
    "        # Generate image embedding\n",
    "        try:\n",
    "            embedding = get_image_embedding(poster_url)\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "            continue\n",
    "\n",
    "        # Upload to Pinecone with namespace and metadata as a dictionary\n",
    "        index.upsert(\n",
    "            vectors=[(imdb_id, embedding, metadata)],\n",
    "            namespace=NAMESPACE\n",
    "        )\n",
    "\n",
    "#process_and_upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-JMYkanAIJf_",
    "outputId": "0f3ab5db-68db-4d3a-f20d-e71732bac41c"
   },
   "outputs": [],
   "source": [
    "get_movie_metadata(\"tt6087562\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SYyM41yJOdvz"
   },
   "outputs": [],
   "source": [
    "# Pinecone and create an index\n",
    "from pinecone import Pinecone\n",
    "from pinecone import ServerlessSpec\n",
    "from google.colab import userdata\n",
    "\n",
    "pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "\n",
    "# pc.create_index(\n",
    "#     name=\"movie-posters\",\n",
    "#     dimension=512, # dimensions from CLIP\n",
    "#     metric=\"cosine\",\n",
    "#     spec=ServerlessSpec(\n",
    "#         cloud=\"aws\",\n",
    "#         region=\"us-east-1\"\n",
    "#     )\n",
    "# )\n",
    "\n",
    "# Access the existing index\n",
    "index = pc.Index(\"movie-posters\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gk7EXrBGD7RB",
    "outputId": "1740b398-3139-4ca4-8bf6-f3849fa8d589"
   },
   "outputs": [],
   "source": [
    "#index.delete(delete_all=True, namespace='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zeFNOCm3LkEa",
    "outputId": "86e15628-1270-4890-b9a1-7b759bd5af8f"
   },
   "outputs": [],
   "source": [
    "print(index.describe_index_stats())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "t4Ls8hbgpQkm"
   },
   "source": [
    "ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 384
    },
    "id": "D0vYUIbZF8lH",
    "outputId": "02d81730-aafe-48b0-d0be-8c6e96a3c105"
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import torch\n",
    "from torchvision import models, transforms\n",
    "from PIL import Image\n",
    "import requests\n",
    "from io import BytesIO\n",
    "import pinecone\n",
    "from tmdbv3api import TMDb, Find\n",
    "from google.colab import userdata\n",
    "import time\n",
    "\n",
    "# Configuration Variables\n",
    "MODEL_NAME = \"resnet50\"  # Using ResNet50 model\n",
    "NAMESPACE = \"ResNet-50\"\n",
    "VECTOR_DIM = 512  # Limiting to 512 to match CLIP dimensions (Pinecone free plan has only one index)\n",
    "\n",
    "# Initialize Pinecone\n",
    "pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "index = pc.Index(\"movie-posters\")\n",
    "\n",
    "# Initialize TMDB API client\n",
    "tmdb = TMDb()\n",
    "tmdb.api_key = userdata.get(\"TMDB_API_KEY\")\n",
    "\n",
    "# Initialize the Find class\n",
    "find = Find()\n",
    "\n",
    "# Load the Pinecone movie poster dataset\n",
    "ds = load_dataset(\"pinecone/movie-posters\")\n",
    "\n",
    "# Initialize ResNet model and modify it to produce 512-dimensional embeddings\n",
    "class ResNet50WithEmbedding(torch.nn.Module):\n",
    "    def __init__(self, embedding_dim):\n",
    "        super(ResNet50WithEmbedding, self).__init__()\n",
    "        self.base_model = models.resnet50(pretrained=True)\n",
    "        # Remove the final fully connected layer\n",
    "        self.base_model = torch.nn.Sequential(*list(self.base_model.children())[:-1])\n",
    "        # Add a new fully connected layer\n",
    "        self.fc = torch.nn.Linear(in_features=2048, out_features=embedding_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.base_model(x)\n",
    "        x = x.view(x.size(0), -1)  # Flatten the tensor\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "model = ResNet50WithEmbedding(VECTOR_DIM)\n",
    "model.eval()\n",
    "\n",
    "# Define image preprocessing\n",
    "preprocess = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "])\n",
    "\n",
    "# Function to fetch an image from a URL and get its embedding\n",
    "def get_image_embedding(poster_url):\n",
    "    response = requests.get(poster_url)\n",
    "    image = Image.open(BytesIO(response.content)).convert('RGB')\n",
    "    inputs = preprocess(image).unsqueeze(0)  # Add batch dimension\n",
    "    with torch.no_grad():\n",
    "        embedding = model(inputs).squeeze().numpy()\n",
    "    return embedding\n",
    "\n",
    "# Function to fetch movie metadata from TMDB API using IMDb ID\n",
    "def get_movie_metadata(imdb_id):\n",
    "    time.sleep(0.02)  # 50 requests per second = 1 request every 0.02 seconds so we don't overload TMDB API\n",
    "    try:\n",
    "        results = find.find(imdb_id, 'imdb_id')\n",
    "        if results.movie_results:\n",
    "            movie = results.movie_results[0]  # Take the first result\n",
    "            return {\n",
    "                'title': movie.title,\n",
    "                'overview': movie.overview,\n",
    "                'release_date': movie.release_date\n",
    "            }\n",
    "        else:\n",
    "            print(f\"No metadata found for IMDb ID {imdb_id}\")\n",
    "            return None\n",
    "    except Exception as e:\n",
    "        print(f\"Error fetching metadata for IMDb ID {imdb_id}: {e}\")\n",
    "        return None\n",
    "\n",
    "# Process the dataset and upload embeddings with metadata to Pinecone\n",
    "def process_and_upload():\n",
    "    for item in ds['train']:\n",
    "        imdb_id = item['imdbId']\n",
    "        poster_url = item['poster']\n",
    "\n",
    "        # Fetch movie metadata\n",
    "        metadata = get_movie_metadata(imdb_id)\n",
    "        if metadata is None:\n",
    "            continue\n",
    "\n",
    "        # Add poster URL from dataset to metadata\n",
    "        metadata['poster_url'] = poster_url\n",
    "\n",
    "        # Generate image embedding\n",
    "        try:\n",
    "            embedding = get_image_embedding(poster_url)\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "            continue\n",
    "\n",
    "        # Upload to Pinecone with the specified namespace and metadata as a dictionary\n",
    "        index.upsert(\n",
    "            vectors=[(imdb_id, embedding.tolist(), metadata)],\n",
    "            namespace=NAMESPACE\n",
    "        )\n",
    "\n",
    "#process_and_upload()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "45Yiq3H9w2Rm"
   },
   "source": [
    "VGG16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "Z85VUPb4F8cA",
    "outputId": "c3180f4a-fc57-4c2e-ab86-080586c8f658"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torchvision import models, transforms\n",
    "from PIL import Image\n",
    "import requests\n",
    "from io import BytesIO\n",
    "import pinecone\n",
    "from tmdbv3api import TMDb, Find\n",
    "from google.colab import userdata\n",
    "import time\n",
    "\n",
    "# Configuration Variables\n",
    "VECTOR_DIM = 512  # Dimension for Pinecone index, limited to 512 by CLIP, and only one index allowed on free tier Pinecone\n",
    "NAMESPACE = \"VGG16\"\n",
    "\n",
    "# Initialize Pinecone\n",
    "pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "index = pc.Index(\"movie-posters\")\n",
    "\n",
    "# Initialize TMDB API client\n",
    "tmdb = TMDb()\n",
    "tmdb.api_key = userdata.get(\"TMDB_API_KEY\")\n",
    "\n",
    "# Initialize the Find class\n",
    "find = Find()\n",
    "\n",
    "# Load the Pinecone movie poster dataset\n",
    "ds = load_dataset(\"pinecone/movie-posters\")\n",
    "\n",
    "# Initialize VGG16 model and modify it to produce 512-dimensional embeddings\n",
    "class VGG16WithEmbedding(torch.nn.Module):\n",
    "    def __init__(self, embedding_dim):\n",
    "        super(VGG16WithEmbedding, self).__init__()\n",
    "        self.base_model = models.vgg16(pretrained=True)\n",
    "        # Remove the final fully connected layers\n",
    "        self.base_model.classifier = torch.nn.Sequential(*list(self.base_model.classifier.children())[:-3])\n",
    "        # Add a new fully connected layer\n",
    "        self.fc = torch.nn.Linear(in_features=4096, out_features=embedding_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.base_model(x)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "model = VGG16WithEmbedding(VECTOR_DIM)\n",
    "model.eval()\n",
    "\n",
    "# Define image preprocessing\n",
    "preprocess = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "])\n",
    "\n",
    "# Function to fetch an image from a URL and get its embedding\n",
    "def get_image_embedding(poster_url):\n",
    "    response = requests.get(poster_url)\n",
    "    image = Image.open(BytesIO(response.content)).convert('RGB')\n",
    "    inputs = preprocess(image).unsqueeze(0)  # Add batch dimension\n",
    "    with torch.no_grad():\n",
    "        embedding = model(inputs).squeeze().numpy()\n",
    "    return embedding\n",
    "\n",
    "# Function to fetch movie metadata from TMDB API using IMDb ID\n",
    "def get_movie_metadata(imdb_id):\n",
    "    time.sleep(0.02)  # 50 requests per second = 1 request every 0.02 seconds so we don't overload TMDB API\n",
    "    try:\n",
    "        results = find.find(imdb_id, 'imdb_id')\n",
    "        if results.movie_results:\n",
    "            movie = results.movie_results[0]  # Take the first result\n",
    "            return {\n",
    "                'title': movie.title,\n",
    "                'overview': movie.overview,\n",
    "                'release_date': movie.release_date\n",
    "            }\n",
    "        else:\n",
    "            print(f\"No metadata found for IMDb ID {imdb_id}\")\n",
    "            return None\n",
    "    except Exception as e:\n",
    "        print(f\"Error fetching metadata for IMDb ID {imdb_id}: {e}\")\n",
    "        return None\n",
    "\n",
    "# Process the dataset and upload embeddings with metadata to Pinecone\n",
    "def process_and_upload():\n",
    "    for item in ds['train']:\n",
    "        imdb_id = item['imdbId']\n",
    "        poster_url = item['poster']\n",
    "\n",
    "        # Fetch movie metadata\n",
    "        metadata = get_movie_metadata(imdb_id)\n",
    "        if metadata is None:\n",
    "            continue\n",
    "\n",
    "        # Add poster URL from dataset to metadata\n",
    "        metadata['poster_url'] = poster_url\n",
    "\n",
    "        # Generate image embedding\n",
    "        try:\n",
    "            embedding = get_image_embedding(poster_url)\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "            continue\n",
    "\n",
    "        # Upload to Pinecone with the specified namespace and metadata as a dictionary\n",
    "        index.upsert(\n",
    "            vectors=[(imdb_id, embedding.tolist(), metadata)],\n",
    "            namespace=NAMESPACE\n",
    "        )\n",
    "\n",
    "#process_and_upload()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ab6iZLHpFjF0"
   },
   "source": [
    "# Mapping from imdb id to movie name\n",
    "##(for the user to be able to query on only movie names which exist in the dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "v3QGkvYX-S8O",
    "outputId": "d0dd289a-420d-4887-81ab-c131591e7c22"
   },
   "outputs": [],
   "source": [
    "# pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "# index = pc.Index(\"movie-posters\")\n",
    "# namespaces =['ResNet-50', 'VGG16', 'CLIP']\n",
    "\n",
    "# mapping = {}\n",
    "# for ns in namespaces:\n",
    "#   for ids in index.list(namespace=ns):\n",
    "#     #print(ids)\n",
    "#     res = index.fetch(ids=ids, namespace=ns)\n",
    "\n",
    "#     vectors = res['vectors']\n",
    "#     for k in vectors:\n",
    "#       title = vectors[k]['metadata']['title']\n",
    "#       if title not in mapping:\n",
    "#         mapping[title] = k\n",
    "# print(mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8l3HrXx7KRRy",
    "outputId": "5f43bdea-a546-4e43-a59b-8e46278ab8e2"
   },
   "outputs": [],
   "source": [
    "# print(len(mapping))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8tl7Bnu4EUSN"
   },
   "source": [
    "Testing embedding results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1-AVgi77Jq_C",
    "outputId": "b7e6aee8-db4b-44c3-fb14-91394e58fa9f"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "# Create a dictionary for quick lookup of poster URLs by IMDb ID\n",
    "id_to_poster = {item['imdbId']: item['poster'] for item in ds['train']}\n",
    "\n",
    "# Choose a random movie from the dataset\n",
    "random_movie = random.choice(ds['train'])\n",
    "\n",
    "# Extract the IMDb ID and poster URL of the random movie\n",
    "specific_imdb_id = random_movie['imdbId']\n",
    "\n",
    "id_to_poster = {item['imdbId']: item['poster'] for item in ds['train']}\n",
    "\n",
    "# Look up the movie in the dataset using the specific IMDb ID\n",
    "if specific_imdb_id in id_to_poster:\n",
    "    poster_url = id_to_poster[specific_imdb_id]\n",
    "\n",
    "    # Get the embedding of the specified movie (use the function get_image_embedding)\n",
    "    embedding = get_image_embedding(poster_url)  # Assuming you have already defined this function\n",
    "\n",
    "    # Query Pinecone to get the top 10 closest movie posters\n",
    "    result = index.query(vector=embedding.tolist(), top_k=10, include_metadata=True, namespace=\"CLIP\")\n",
    "\n",
    "    # Print the specified movie and the top 10 similar movies with metadata\n",
    "    print(f\"Specified Movie IMDb ID: {specific_imdb_id}\")\n",
    "    print(f\"Poster URL: {poster_url}\")\n",
    "    print(\"Top 10 Similar Movies:\")\n",
    "    for match in result['matches']:\n",
    "        similar_imdb_id = match['id']\n",
    "        similar_poster_url = id_to_poster.get(similar_imdb_id, \"URL not found\")\n",
    "        # Access the metadata\n",
    "        metadata = match.get('metadata', {})\n",
    "        title = metadata.get('title', \"Title not found\")\n",
    "        overview = metadata.get('overview', \"Overview not found\")\n",
    "        release_date = metadata.get('release_date', \"Release date not found\")\n",
    "        poster_url = metadata.get('poster_url', \"Poster URL not found\")\n",
    "\n",
    "        print(f\"IMDb ID: {similar_imdb_id}, Score: {match['score']}\")\n",
    "        print(f\"Title: {title}\")\n",
    "        print(f\"Overview: {overview}\")\n",
    "        print(f\"Release Date: {release_date}\")\n",
    "        print(f\"Poster URL: {poster_url}\")\n",
    "else:\n",
    "    print(f\"IMDb ID {specific_imdb_id} not found in the dataset.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fG21JPF0BQiS",
    "outputId": "0f1ad24c-dc0e-48bd-97a7-4f36687e517f"
   },
   "outputs": [],
   "source": [
    "# Specify the IMDb ID of the movie you want to use\n",
    "specific_imdb_id = 'tt3638686'\n",
    "\n",
    "id_to_poster = {item['imdbId']: item['poster'] for item in ds['train']}\n",
    "\n",
    "# Look up the movie in the dataset using the specific IMDb ID\n",
    "if specific_imdb_id in id_to_poster:\n",
    "    poster_url = id_to_poster[specific_imdb_id]\n",
    "\n",
    "    # Get the embedding of the specified movie (use the function get_image_embedding)\n",
    "    embedding = get_image_embedding(poster_url)  # Assuming you have already defined this function\n",
    "\n",
    "    # Query Pinecone to get the top 10 closest movie posters\n",
    "    result = index.query(vector=embedding.tolist(), top_k=10, include_metadata=True, namespace=\"CLIP\")\n",
    "\n",
    "    # Print the specified movie and the top 10 similar movies with metadata\n",
    "    print(f\"Specified Movie IMDb ID: {specific_imdb_id}\")\n",
    "    print(f\"Poster URL: {poster_url}\")\n",
    "    print(\"Top 10 Similar Movies:\")\n",
    "    for match in result['matches']:\n",
    "        similar_imdb_id = match['id']\n",
    "        similar_poster_url = id_to_poster.get(similar_imdb_id, \"URL not found\")\n",
    "        # Access the metadata\n",
    "        metadata = match.get('metadata', {})\n",
    "        title = metadata.get('title', \"Title not found\")\n",
    "        overview = metadata.get('overview', \"Overview not found\")\n",
    "        release_date = metadata.get('release_date', \"Release date not found\")\n",
    "        poster_url = metadata.get('poster_url', \"Poster URL not found\")\n",
    "\n",
    "        print(f\"IMDb ID: {similar_imdb_id}, Score: {match['score']}\")\n",
    "        print(f\"Title: {title}\")\n",
    "        print(f\"Overview: {overview}\")\n",
    "        print(f\"Release Date: {release_date}\")\n",
    "        print(f\"Poster URL: {poster_url}\")\n",
    "else:\n",
    "    print(f\"IMDb ID {specific_imdb_id} not found in the dataset.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0IJ4iFgEmHs1"
   },
   "source": [
    "Test TMDB API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "soN78GujJQgS",
    "outputId": "3d544aaf-a914-4923-876c-d7550b5d9c50"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "tmdb_api_key = userdata.get('TMDB_API_KEY')\n",
    "\n",
    "# Function to get movie details from TMDB using IMDb ID\n",
    "def get_movie_details(imdb_id):\n",
    "    url = f\"https://api.themoviedb.org/3/find/{imdb_id}\"\n",
    "    params = {\n",
    "        'api_key': tmdb_api_key,\n",
    "        'external_source': 'imdb_id'\n",
    "    }\n",
    "    response = requests.get(url, params=params)\n",
    "    data = response.json()\n",
    "\n",
    "    # print(\"respones data\")\n",
    "    # print(data)\n",
    "\n",
    "    if response.status_code == 200 and 'movie_results' in data:\n",
    "        if len(data['movie_results']) > 0:\n",
    "            return data['movie_results'][0]  # Return the first movie result\n",
    "        else:\n",
    "            print(\"No movie found for this IMDb ID.\")\n",
    "            return None\n",
    "    else:\n",
    "        print(f\"Error: {response.status_code}\")\n",
    "        return None\n",
    "\n",
    "# Example usage:\n",
    "imdb_id = 'tt2514894'  # Example IMDb ID\n",
    "movie_details = get_movie_details(imdb_id)\n",
    "\n",
    "print(movie_details)\n",
    "\n",
    "if movie_details:\n",
    "    print(\"Title:\", movie_details.get('title'))\n",
    "    print(\"Overview:\", movie_details.get('overview'))\n",
    "    print(\"Release Date:\", movie_details.get('release_date'))\n",
    "    print(\"Poster Path:\", id_to_poster.get(imdb_id, \"URL not found\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "q6On28mfqPJQ"
   },
   "source": [
    "# Version 2 - Populate separate indices using \"popular\" TMDB endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IrL4YNMO7wKa"
   },
   "outputs": [],
   "source": [
    "# Pinecone and create an index\n",
    "from pinecone import Pinecone\n",
    "from pinecone import ServerlessSpec\n",
    "from google.colab import userdata\n",
    "\n",
    "pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "\n",
    "pc.create_index(\n",
    "    name=\"movie-posters-v2-clip\",\n",
    "    dimension=768, # dimensions from CLIP\n",
    "    metric=\"cosine\",\n",
    "    spec=ServerlessSpec(\n",
    "        cloud=\"aws\",\n",
    "        region=\"us-east-1\"\n",
    "    )\n",
    ")\n",
    "\n",
    "# pc.create_index(\n",
    "#     name=\"movie-posters-v2-resnet-50\",\n",
    "#     dimension=2048,\n",
    "#     metric=\"euclidean\",\n",
    "#     spec=ServerlessSpec(\n",
    "#         cloud=\"aws\",\n",
    "#         region=\"us-east-1\"\n",
    "#     )\n",
    "# )\n",
    "\n",
    "# pc.create_index(\n",
    "#     name=\"movie-posters-v2-vgg16\",\n",
    "#     dimension=4096,\n",
    "#     metric=\"euclidean\",\n",
    "#     spec=ServerlessSpec(\n",
    "#         cloud=\"aws\",\n",
    "#         region=\"us-east-1\"\n",
    "#     )\n",
    "# )\n",
    "\n",
    "# Access the existing index\n",
    "#index = pc.Index(\"movie-posters\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YgcHNJQXSJQ4"
   },
   "source": [
    "Getting top 10,000 popular movies (limited by page 500 max for tmdb, 20 results per page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zon-CdhZtZw3"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from google.colab import userdata\n",
    "import json\n",
    "import pprint\n",
    "import time\n",
    "\n",
    "def get_popular_page(page, TMDB_AUTH):\n",
    "  '''\n",
    "  Returns results page as python dict\n",
    "  20 movie results per page\n",
    "  '''\n",
    "  time.sleep(0.02)  # 50 requests per second = 1 request every 0.02 seconds so we don't overload TMDB API\n",
    "  url = f\"https://api.themoviedb.org/3/movie/popular?language=en-US&page={page}\"\n",
    "  headers = {\n",
    "    \"accept\": \"application/json\",\n",
    "    \"Authorization\": TMDB_AUTH\n",
    "  }\n",
    "  try:\n",
    "    response = requests.get(url, headers=headers)\n",
    "  except Exception as e:\n",
    "    print(f\"Problem getting response for page {page} {e}\")\n",
    "    return None\n",
    "  response_dict = json.loads(response.text)\n",
    "  return response_dict['results']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "G98xYFEctn5n",
    "outputId": "cbb00f01-eedf-4faa-ddb7-bfef53fd0992"
   },
   "outputs": [],
   "source": [
    "TMDB_AUTH = userdata.get(\"TMDB_AUTHORIZATION\")\n",
    "\n",
    "movies = []\n",
    "page_max = 2501 # not inclusive\n",
    "for page in range(1, page_max):\n",
    "  results = None\n",
    "  try:\n",
    "    results = get_popular_page(page, TMDB_AUTH)\n",
    "  except Exception as e:\n",
    "    print(f\"Broke at {page} with error {e}\")\n",
    "    page_max = page\n",
    "    break\n",
    "  if results:\n",
    "    movies.extend(results)\n",
    "\n",
    "json_path = f'/content/drive/My Drive/Colab_Notebooks/popular_tmdb_endpoint_until_page_{page_max}.json'\n",
    "\n",
    "with open(json_path, 'w') as f:\n",
    "    json.dump(movies, f)\n",
    "\n",
    "\n",
    "# # Load the movies list from the JSON file\n",
    "# with open(json_path, 'r') as f:\n",
    "#     movies = json.load(f)\n",
    "\n",
    "# pp = pprint.PrettyPrinter(indent=4)\n",
    "# #print(len(movies))\n",
    "# #pp.pprint(movies)\n",
    "# print(len(movies))\n",
    "# print(json.dumps(movies))\n",
    "\n",
    "# https://image.tmdb.org/t/p/original/[poster_path]\n",
    "\n",
    "# https://image.tmdb.org/t/p/original\n",
    "# https://image.tmdb.org/t/p/original/vKVUsumbCzK5Kn3aDpKM4EizKCA.jpg\n",
    "# https://image.tmdb.org/t/p/original/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tTzHcNrlN9L2",
    "outputId": "321bd720-80dc-4b0a-86eb-c7527dad98c4"
   },
   "outputs": [],
   "source": [
    "json_path = f'/content/drive/My Drive/Colab_Notebooks/popular_tmdb_endpoint_until_page_501.json'\n",
    "\n",
    "# Load the movies list from the JSON file\n",
    "with open(json_path, 'r') as f:\n",
    "    movies = json.load(f)\n",
    "\n",
    "for i, m in enumerate(movies):\n",
    "  print(m)\n",
    "  if i == 5:\n",
    "    break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_Q23O7dGQaAo"
   },
   "source": [
    "CLIP Processing on each popular movie"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-HYHkkCmMdoC"
   },
   "source": [
    "# Version 2 - 3 models (CLIP, VGG16, ResNet-50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "44ffcc9259ca44409f4effe9d92a0def",
      "cb0c3c677c2540c5b19d62840950677f",
      "ba8b8e77fb754eee92f77690a1f7cb3c",
      "b853b5b5ea154faa8954d3ad8a7cea18",
      "1e7464ab06d8436783e2990f0e68ff74",
      "3794070b005d4773996258180489ded2",
      "25b275e75879470ca09bb7916761767b",
      "a7ce86c7d7584101a8fa608dfa0a9e1a",
      "21501814946443c49ee78332c8b58462",
      "0cfcc07391694241a9b89eafa67c61c3",
      "c6e257f2336440ccb8e1d4619b831e60",
      "0329180a92be4c7db9b6aee13149d9ff",
      "f8189c458fdb4e519e272411b47a3b03",
      "4edde7f74fbd4e8599de9f352b96ef31",
      "087298befcd745649a306991df68c18a",
      "374fd58afbf14c91a68356933fd3850f",
      "507f80f58c5249c280ae8c2cf9e4f4b3",
      "4a090ff49a374f49b7d69280455116a1",
      "cf8dfb41077d457289797b30c1ad7fc1",
      "b0fc47ad2a0a413cbff5f3f03167beb9",
      "cad2e6c9b4ba458b9ecf86ad930c1bac",
      "feff97faae624211a5d97dccab5473b7",
      "514e672834a14701bd1368babdaaaef1",
      "6784c347f4cd4ef69d73db22a44fc26e",
      "ded7aa4d9a354e15bdc1ee9d70b9b991",
      "1b9ce94ca4254d32a5cedb2bad6fee6b",
      "e63a47230ede4cb69272634f88983aa6",
      "de506a1fb2db4c5a9ce6aedc4c4f2552",
      "0c245793118c4f0f987c00cdfd4e6a54",
      "45d57d93ac0d416d9ba59cdd123c0c86",
      "911805666901492a81c0d480a3622b4b",
      "fdb00882fdc24db785b9f91297474ef0",
      "8448fc16249c4b3c953be3845a5ad46f",
      "b0d1c479ce024c78844e9ee245f2104e",
      "27b99798eb24445ea2c391f62fa4961f",
      "4b87e04a10c74c3a8ce8ed7feccd51c9",
      "2f55552f28254ee2934dc649f12397e7",
      "5ba30dc4f5374489b26b2051c9c3ac4e",
      "81c66588182c4dcfbcb069a41fc4dc08",
      "b184845cecc44f3693ae1f5e3ca8019b",
      "83b6133241b04203bfb14f0a3d50cf7e",
      "beb090984b3f4ca2a12d1fc39dd713f4",
      "0689d93c424f4f9faab4ba8672d9bb05",
      "127a4acd1d2145f183a9014e610c4560",
      "0d831a5fde1b4909aaa57629e21dbe71",
      "5e1d90cc40f04b7ea2c6f9fd8053793d",
      "609df8cf08904e82bb2dcd68108450f1",
      "036677554a6b45f595707763c0a23363",
      "792e3c32f05647778dfcddb4f8c1ce3f",
      "1c82bdb929c94728bff7eceda3e1b0ae",
      "75f07bbbf05e4f6fb788339e9de63072",
      "db2c74b68ebd48d289d54fa521429020",
      "66bb8241741249ae8c7b0736db1fed0d",
      "1429025083c844d0a463378b1d6a80e7",
      "db8df895491546f4bad570a42c3bf38b",
      "a4d8c4bc34ed4864a428ee26c57f3036",
      "5cc3d158695d4cd596e5b3e9115c73ea",
      "30170b8484c741069c40d50043f82129",
      "f9db892871514a5783be54640b9ad4bd",
      "05c493110ed14d3eb2fa0d59827d8904",
      "f4e9e3e7cc4f489e8d718681c189d502",
      "beaf18e3ee314bdcbec14d80f03ca197",
      "728f639455a34ac2a52093f92b9b5e40",
      "1b0980d86d5243418d349cc81aa4f309",
      "84ae0e8407d348fa97b808acf5a6bc5a",
      "56ca5a7580c44e6f955723df1890fb4c",
      "d9c74604747e4a079d95b6735c98bb72",
      "64f4fd3115094f8d87e98b23a7fea3cc",
      "55cdbc0133724691bbbd5121e4e55420",
      "2db2dc69be0c4c58b98acbc725fde74c",
      "446dc7a4b8864d4a9e41b9bdc9ba9ccb",
      "820e36d0e58f449891f298d94d4ba757",
      "92114d553d5243bfa129476d15838fb4",
      "eba2567bebe94196b0274cdfac285a2f",
      "ad9708087bcc43f399bd3e24ce3dd4b5",
      "92c99124e0354621a518ade0da93289f",
      "98153c92150543a2b70f69c928745298",
      "ffc1f1da2c88472fbfae264b432eb915",
      "d64efe75eb254ce5879b5dd683821a90",
      "ca3ccceb2fee487a83fe9cf9ef7e5442",
      "92287b129e8d4d639592fcfe5c1c492b",
      "bd65012e25ed49f7bea94e412dddd6c4",
      "80374f40d9304f1684c9fe360679f1e0",
      "5cf36aa1f6a94481a54042aebaac128a",
      "63b870a32a7b4f3484d520a724a78003",
      "5b001bd36a894f53b77d6c9cf75cb43f",
      "3adfd6f04cdf48598ef7fc96fe140805",
      "feaf13ee65cd433b8e808470be9e83d9"
     ]
    },
    "id": "uQg1lQhY7RZS",
    "outputId": "5597aa7d-43f4-4c5c-9f9b-c00bfe43391e"
   },
   "outputs": [],
   "source": [
    "from transformers import CLIPProcessor, CLIPModel\n",
    "from PIL import Image\n",
    "import requests\n",
    "import json\n",
    "from io import BytesIO\n",
    "import torch\n",
    "import pinecone\n",
    "from pinecone import Pinecone\n",
    "from google.colab import userdata\n",
    "import time\n",
    "\n",
    "# Load the model and processor\n",
    "model = CLIPModel.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
    "processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
    "NAMESPACE = \"clip-vit-large-patch14\"\n",
    "\n",
    "pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "index = pc.Index(\"movie-posters-v2-clip\")\n",
    "\n",
    "json_path = f'/content/drive/My Drive/Colab_Notebooks/popular_tmdb_endpoint_until_page_501.json'\n",
    "\n",
    "# Load the movies list from the JSON file\n",
    "with open(json_path, 'r') as f:\n",
    "    movies = json.load(f)\n",
    "\n",
    "def get_image_embedding(poster_url):\n",
    "    # Fetch the image from the URL\n",
    "    try:\n",
    "      poster_url = \"https://image.tmdb.org/t/p/original\" + poster_url\n",
    "      response = requests.get(poster_url)\n",
    "      image = Image.open(BytesIO(response.content)).convert('RGB')\n",
    "\n",
    "      # Preprocess the image\n",
    "      inputs = processor(images=image, return_tensors=\"pt\")\n",
    "\n",
    "      # Generate image embedding\n",
    "      with torch.no_grad():\n",
    "          outputs = model.get_image_features(**inputs)\n",
    "          embedding = outputs.squeeze().numpy()\n",
    "\n",
    "      return embedding\n",
    "    except Exception as e:\n",
    "      print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "      return None\n",
    "\n",
    "def prepare_metadata(movie):\n",
    "    return {\n",
    "        \"adult\": movie.get(\"adult\", False),\n",
    "        \"backdrop_path\": movie.get(\"backdrop_path\", \"empty\") if movie.get(\"backdrop_path\") is not None else \"empty\",\n",
    "        \"genre_ids\": ','.join(map(str, movie.get(\"genre_ids\", []))),\n",
    "        \"id\": str(movie.get(\"id\", \"\")),\n",
    "        \"original_language\": movie.get(\"original_language\", \"\"),\n",
    "        \"original_title\": movie.get(\"original_title\", \"\"),\n",
    "        \"overview\": movie.get(\"overview\", \"\"),\n",
    "        \"popularity\": movie.get(\"popularity\", 0.0),\n",
    "        \"poster_path\": movie.get(\"poster_path\", \"\"),\n",
    "        \"release_date\": movie.get(\"release_date\", \"\"),\n",
    "        \"title\": movie.get(\"title\", \"\"),\n",
    "        \"video\": movie.get(\"video\", False),\n",
    "        \"vote_average\": movie.get(\"vote_average\", 0.0),\n",
    "        \"vote_count\": movie.get(\"vote_count\", 0)\n",
    "    }\n",
    "\n",
    "def process_and_upload():\n",
    "    for i in range(0, 10001):\n",
    "    #for i, movie in enumerate(movies):\n",
    "        movie = movies[i]\n",
    "        imdb_id = str(movie.get('id', '')) # Ensure ID is a string\n",
    "        poster_url = movie.get('poster_path', '')\n",
    "        print(f\"Processing {imdb_id} - {movie.get('title')}\")\n",
    "\n",
    "        # Generate image embedding\n",
    "        try:\n",
    "            embedding = get_image_embedding(poster_url)\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "            continue\n",
    "\n",
    "        metadata = prepare_metadata(movie)\n",
    "        #print(metadata)\n",
    "\n",
    "        try:\n",
    "        # Upload to Pinecone with the specified namespace and metadata as a dictionary\n",
    "          index.upsert(\n",
    "              vectors=[(imdb_id, embedding.tolist(), metadata)],\n",
    "              namespace=NAMESPACE\n",
    "          )\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to upload to Pinecone: {e}\")\n",
    "\n",
    "process_and_upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nlnQO0-GRmBv"
   },
   "outputs": [],
   "source": [
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mZOFsQSzzMbt",
    "outputId": "bddf447c-69fc-4ae3-98c0-bf26167963f1"
   },
   "outputs": [],
   "source": [
    "!pip install timm torch torchvision pinecone-client requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 315
    },
    "id": "fCE3jNP6zkm-",
    "outputId": "7cb3591a-986f-40c8-ff75-48457d52acff"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision.models import vgg16\n",
    "from PIL import Image\n",
    "from urllib.request import urlopen\n",
    "import numpy as np\n",
    "import pinecone\n",
    "from pinecone import Pinecone\n",
    "import json\n",
    "from google.colab import userdata\n",
    "\n",
    "# Load the VGG16 model with its classifier\n",
    "model = vgg16(pretrained=True)\n",
    "model.eval()\n",
    "\n",
    "# Remove the final classification layer\n",
    "model.classifier = torch.nn.Sequential(*list(model.classifier.children())[:-1])\n",
    "\n",
    "# Define the transform for input images\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "])\n",
    "\n",
    "NAMESPACE = \"vgg16-tv-in1k\"\n",
    "pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "index = pc.Index(\"movie-posters-v2-vgg16\")\n",
    "\n",
    "json_path = '/content/drive/My Drive/Colab_Notebooks/popular_tmdb_endpoint_until_page_501.json'\n",
    "\n",
    "# Load the movies list from the JSON file\n",
    "with open(json_path, 'r') as f:\n",
    "    movies = json.load(f)\n",
    "\n",
    "def get_image_embedding(poster_url):\n",
    "    try:\n",
    "        poster_url = \"https://image.tmdb.org/t/p/original\" + poster_url\n",
    "        img = Image.open(urlopen(poster_url)).convert('RGB')  # Ensure image is in RGB format\n",
    "        input_tensor = transform(img).unsqueeze(0)  # Add batch dimension\n",
    "\n",
    "        with torch.no_grad():\n",
    "            # Forward pass through the model\n",
    "            features = model(input_tensor)\n",
    "            embedding = features.squeeze().numpy()  # Remove batch dimension and convert to NumPy array\n",
    "\n",
    "        #print(f\"Embedding shape: {embedding.shape}\")  # Print shape to confirm size\n",
    "\n",
    "        if embedding.shape[0] != 4096:\n",
    "            raise ValueError(f\"Embedding dimension mismatch: Expected 4096, but got {embedding.shape[0]}\")\n",
    "\n",
    "        # Convert to list of floats\n",
    "        embedding_list = embedding.tolist()  # Convert NumPy array to list of floats\n",
    "\n",
    "        return embedding_list  # Return as a list of floats\n",
    "    except Exception as e:\n",
    "        print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "        return None\n",
    "\n",
    "def prepare_metadata(movie):\n",
    "    return {\n",
    "        \"adult\": movie.get(\"adult\", False),\n",
    "        \"backdrop_path\": movie.get(\"backdrop_path\", \"empty\") if movie.get(\"backdrop_path\") is not None else \"empty\",\n",
    "        \"genre_ids\": ','.join(map(str, movie.get(\"genre_ids\", []))),\n",
    "        \"id\": str(movie.get(\"id\", \"\")),\n",
    "        \"original_language\": movie.get(\"original_language\", \"\"),\n",
    "        \"original_title\": movie.get(\"original_title\", \"\"),\n",
    "        \"overview\": movie.get(\"overview\", \"\"),\n",
    "        \"popularity\": movie.get(\"popularity\", 0.0),\n",
    "        \"poster_path\": movie.get(\"poster_path\", \"\"),\n",
    "        \"release_date\": movie.get(\"release_date\", \"\"),\n",
    "        \"title\": movie.get(\"title\", \"\"),\n",
    "        \"video\": movie.get(\"video\", False),\n",
    "        \"vote_average\": movie.get(\"vote_average\", 0.0),\n",
    "        \"vote_count\": movie.get(\"vote_count\", 0)\n",
    "    }\n",
    "\n",
    "def process_and_upload():\n",
    "    for i in range(0, 10001):\n",
    "        movie = movies[i]\n",
    "        imdb_id = str(movie.get('id', ''))\n",
    "        poster_url = movie.get('poster_path', '')\n",
    "        print(f\"Processing {i} - {imdb_id} - {movie.get('title')}\")\n",
    "\n",
    "        try:\n",
    "            embedding = get_image_embedding(poster_url)\n",
    "            if embedding is None:\n",
    "                continue\n",
    "\n",
    "            metadata = prepare_metadata(movie)\n",
    "\n",
    "            try:\n",
    "                # Upload to Pinecone with the specified namespace and metadata as a dictionary\n",
    "                index.upsert(\n",
    "                    vectors=[(imdb_id, embedding, metadata)],  # Ensure embedding is a list of floats\n",
    "                    namespace=NAMESPACE\n",
    "                )\n",
    "            except Exception as e:\n",
    "                print(f\"Failed to upload to Pinecone: {e}\")\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "\n",
    "process_and_upload()\n",
    "\n",
    "# reprocessed first movie"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5CT15kC0oO39"
   },
   "source": [
    "resnet50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 411,
     "referenced_widgets": [
      "7ecfa96986234e3faa874a90e83a9619",
      "84b546dfe048437a86894dfa2b4744a3",
      "34fd04cae73d4b36bf50759eb709d1a5",
      "ec203f4444d3458d94eba39ccf744d5e",
      "bf014f7ec85046388f9d722740f93bed",
      "191bb8209edf4191a5676e75325f1144",
      "29bc7871b82043ed80fcf27a2daacc6c",
      "9760d08e1dba43b4b5d8aef1ac965c5c",
      "fb5e9f94fc79443fb3254398beadd4b1",
      "10006351d1414d1f9aafc4469706fbfe",
      "e5b1dce866854a6682acab4190a9010b",
      "aad30c2827504a46bc643cb39296d200",
      "640a152d0bac4c078f50fbccd91c3741",
      "21d05463e1684e939948f4db687f89e7",
      "972327d944b24ac08e7b2ccdded3414e",
      "b3d204589a35480aa6e44b8eb666dd4f",
      "b58fd05d69654970afb84d6fcf86ff5b",
      "3b0e7f0c0dd349dd9db3cb2f19fc0584",
      "2ca4d116738a4b72880e681af9233c52",
      "30e200147ebc4d5485c2d457128b3285",
      "914f9fd3c8e54606a030afdb8f51e4d6",
      "d6f12dd589d249768b2f40fcd0948f38",
      "879a19e54a09400f8af579755c46d7b6",
      "26d5a680e7cb4c93a9e9bd06aee68017",
      "a15d6d1ac71341c89b3940592739d7a7",
      "1d36da9459e44343bd71e5b87d78f54d",
      "48e81e1678744e35b89bd59b09658888",
      "d930458c5f5348cd9b577bdeb3357699",
      "187a93a470864e8a99273636098dfed3",
      "236278ff90d14292991f8046ab874f5c",
      "7be2cee3b9ee497693cdcc5f118396f0",
      "ed9d0810943c4005a89fbe21859e7647",
      "d66fe79fefb6402da8ade599b0d36063"
     ]
    },
    "id": "Zg33SKSzka9D",
    "outputId": "4b244d48-cf10-4204-d5d5-e64fa688b908"
   },
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "import torch\n",
    "from transformers import AutoModel, AutoImageProcessor\n",
    "from PIL import Image\n",
    "import json\n",
    "import pinecone\n",
    "from google.colab import userdata\n",
    "import numpy as np\n",
    "\n",
    "# Load the ResNet-50 model and feature extractor\n",
    "model_name = \"microsoft/resnet-50\"\n",
    "model = AutoModel.from_pretrained(model_name)\n",
    "image_processor = AutoImageProcessor.from_pretrained(model_name)\n",
    "\n",
    "# Set model to evaluation mode\n",
    "model.eval()\n",
    "\n",
    "NAMESPACE = \"resnet50\"\n",
    "pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "index = pc.Index(\"movie-posters-v2-resnet-50\")\n",
    "\n",
    "json_path = '/content/drive/My Drive/Colab_Notebooks/popular_tmdb_endpoint_until_page_501.json'\n",
    "\n",
    "# Load the movies list from the JSON file\n",
    "with open(json_path, 'r') as f:\n",
    "    movies = json.load(f)\n",
    "\n",
    "def get_image_embedding(poster_url):\n",
    "    try:\n",
    "        poster_url = \"https://image.tmdb.org/t/p/original\" + poster_url\n",
    "        img = Image.open(urlopen(poster_url))\n",
    "\n",
    "        # Preprocess the image\n",
    "        inputs = image_processor(images=img, return_tensors=\"pt\")\n",
    "\n",
    "        # Generate embeddings\n",
    "        with torch.no_grad():\n",
    "            outputs = model(**inputs)\n",
    "            pooled_output = outputs.pooler_output  # Get the pooled output\n",
    "            embedding = pooled_output.numpy()\n",
    "\n",
    "        #print(f\"Embedding shape: {embedding.shape}\")  # Print shape to confirm size\n",
    "\n",
    "        # Convert to list of floats\n",
    "        if isinstance(embedding, np.ndarray):\n",
    "            embedding_list = embedding.flatten().tolist()\n",
    "        else:\n",
    "            embedding_list = list(map(float, embedding))\n",
    "\n",
    "        return embedding_list\n",
    "    except Exception as e:\n",
    "        print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "        return None\n",
    "\n",
    "def prepare_metadata(movie):\n",
    "    return {\n",
    "        \"adult\": movie.get(\"adult\", False),\n",
    "        \"backdrop_path\": movie.get(\"backdrop_path\", \"empty\") if movie.get(\"backdrop_path\") is not None else \"empty\",\n",
    "        \"genre_ids\": ','.join(map(str, movie.get(\"genre_ids\", []))),\n",
    "        \"id\": str(movie.get(\"id\", \"\")),\n",
    "        \"original_language\": movie.get(\"original_language\", \"\"),\n",
    "        \"original_title\": movie.get(\"original_title\", \"\"),\n",
    "        \"overview\": movie.get(\"overview\", \"\"),\n",
    "        \"popularity\": movie.get(\"popularity\", 0.0),\n",
    "        \"poster_path\": movie.get(\"poster_path\", \"\"),\n",
    "        \"release_date\": movie.get(\"release_date\", \"\"),\n",
    "        \"title\": movie.get(\"title\", \"\"),\n",
    "        \"video\": movie.get(\"video\", False),\n",
    "        \"vote_average\": movie.get(\"vote_average\", 0.0),\n",
    "        \"vote_count\": movie.get(\"vote_count\", 0)\n",
    "    }\n",
    "\n",
    "def process_and_upload():\n",
    "    for i in range(0, 10001):\n",
    "        movie = movies[i]\n",
    "        imdb_id = str(movie.get('id', ''))\n",
    "        poster_url = movie.get('poster_path', '')\n",
    "        print(f\"Processing {imdb_id} - {movie.get('title')}\")\n",
    "\n",
    "        try:\n",
    "            embedding = get_image_embedding(poster_url)\n",
    "            if embedding is None:\n",
    "                continue\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "            continue\n",
    "\n",
    "        metadata = prepare_metadata(movie)\n",
    "\n",
    "        try:\n",
    "            index.upsert(\n",
    "                vectors=[(imdb_id, embedding, metadata)],\n",
    "                namespace=NAMESPACE\n",
    "            )\n",
    "        except Exception as e:\n",
    "            print(f\"Failed to upload to Pinecone: {e}\")\n",
    "\n",
    "process_and_upload()\n",
    "\n",
    "# reprocessed first movie"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ebQ_yVgLDi6P"
   },
   "source": [
    "Tried batches below instead of single instance - found that batching does not significantly change processing time for CLIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "fMMZ2c4JDihQ",
    "outputId": "d90db29f-8cc3-4d57-e20d-7da921f9f5b0"
   },
   "outputs": [],
   "source": [
    "# from transformers import CLIPProcessor, CLIPModel\n",
    "# from PIL import Image\n",
    "# import requests\n",
    "# import json\n",
    "# from io import BytesIO\n",
    "# import torch\n",
    "# import pinecone\n",
    "# from pinecone import Pinecone\n",
    "# from google.colab import userdata\n",
    "# import time\n",
    "\n",
    "# # Load the model and processor\n",
    "# model = CLIPModel.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
    "# processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
    "# NAMESPACE = \"clip-vit-large-patch14\"\n",
    "\n",
    "# pc = Pinecone(api_key=userdata.get('PINECONE_KEY'))\n",
    "# index = pc.Index(\"movie-posters-v2-clip\")\n",
    "\n",
    "# json_path = f'/content/drive/My Drive/Colab_Notebooks/popular_tmdb_endpoint_until_page_501.json'\n",
    "\n",
    "# # Load the movies list from the JSON file\n",
    "# with open(json_path, 'r') as f:\n",
    "#     movies = json.load(f)\n",
    "\n",
    "# def get_image_embedding(poster_url):\n",
    "#     # Fetch the image from the URL\n",
    "#     try:\n",
    "#       poster_url = \"https://image.tmdb.org/t/p/original\" + poster_url\n",
    "#       response = requests.get(poster_url)\n",
    "#       image = Image.open(BytesIO(response.content)).convert('RGB')\n",
    "\n",
    "#       # Preprocess the image\n",
    "#       inputs = processor(images=image, return_tensors=\"pt\")\n",
    "\n",
    "#       # Generate image embedding\n",
    "#       with torch.no_grad():\n",
    "#           outputs = model.get_image_features(**inputs)\n",
    "#           embedding = outputs.squeeze().numpy()\n",
    "\n",
    "#       return embedding\n",
    "#     except Exception as e:\n",
    "#       print(f\"Failed to get embedding for {poster_url}: {e}\")\n",
    "#       return None\n",
    "\n",
    "# def get_image_embedding_batch(poster_urls):\n",
    "#     try:\n",
    "#         images = []\n",
    "#         for poster_url in poster_urls:\n",
    "#             poster_url = \"https://image.tmdb.org/t/p/original\" + poster_url\n",
    "#             response = requests.get(poster_url)\n",
    "#             image = Image.open(BytesIO(response.content)).convert('RGB')\n",
    "#             images.append(image)\n",
    "\n",
    "#         # Preprocess the images as a batch\n",
    "#         inputs = processor(images=images, return_tensors=\"pt\")\n",
    "\n",
    "#         # Generate image embeddings for the batch\n",
    "#         with torch.no_grad():\n",
    "#             outputs = model.get_image_features(**inputs)\n",
    "#             embeddings = outputs.squeeze().numpy()\n",
    "\n",
    "#         return embeddings\n",
    "#     except Exception as e:\n",
    "#         print(f\"Failed to get embedding for batch: {e}\")\n",
    "#         return None\n",
    "\n",
    "# def prepare_metadata(movie):\n",
    "#     return {\n",
    "#         \"adult\": movie.get(\"adult\", False),\n",
    "#         \"backdrop_path\": movie.get(\"backdrop_path\", \"empty\") if movie.get(\"backdrop_path\") is not None else \"empty\",\n",
    "#         \"genre_ids\": ','.join(map(str, movie.get(\"genre_ids\", []))),\n",
    "#         \"id\": str(movie.get(\"id\", \"\")),\n",
    "#         \"original_language\": movie.get(\"original_language\", \"\"),\n",
    "#         \"original_title\": movie.get(\"original_title\", \"\"),\n",
    "#         \"overview\": movie.get(\"overview\", \"\"),\n",
    "#         \"popularity\": movie.get(\"popularity\", 0.0),\n",
    "#         \"poster_path\": movie.get(\"poster_path\", \"\"),\n",
    "#         \"release_date\": movie.get(\"release_date\", \"\"),\n",
    "#         \"title\": movie.get(\"title\", \"\"),\n",
    "#         \"video\": movie.get(\"video\", False),\n",
    "#         \"vote_average\": movie.get(\"vote_average\", 0.0),\n",
    "#         \"vote_count\": movie.get(\"vote_count\", 0)\n",
    "#     }\n",
    "\n",
    "# def process_and_upload():\n",
    "#   batch_size = 16\n",
    "#   for i in range(3904, len(movies), batch_size):\n",
    "#       batch_movies = movies[i:i+batch_size]\n",
    "#       poster_urls = [movie.get('poster_path', '') for movie in batch_movies]\n",
    "#       embeddings = get_image_embedding_batch(poster_urls)\n",
    "\n",
    "#       if embeddings is not None:\n",
    "#           for j, movie in enumerate(batch_movies):\n",
    "#               imdb_id = str(movie.get('id', ''))\n",
    "#               print(f\"Processing {imdb_id} - {movie.get('title')}\")\n",
    "#               metadata = prepare_metadata(movie)\n",
    "#               index.upsert(\n",
    "#                   vectors=[(imdb_id, embeddings[j].tolist(), metadata)],\n",
    "#                   namespace=NAMESPACE\n",
    "#               )\n",
    "# process_and_upload()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}