run_in_google_colab.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Akshat6133/DeepMoon/blob/master/run_in_google_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# make a dir.\n",
        "!mkdir -p /content/drive/MyDrive/comp_vis"
      ],
      "metadata": {
        "id": "jriXoDYrz7ZE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "#clone repo in that dir\n",
        "!git clone https://github.com/Akshat6133/DeepMoon.git /content/drive/MyDrive/comp_vis\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iJ5Lv47Czxc0",
        "outputId": "b9d01ce0-6567-4ccc-df82-32ef4714609e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into '/content/drive/MyDrive/comp_vis'...\n",
            "remote: Enumerating objects: 397, done.\u001b[K\n",
            "remote: Counting objects: 100% (38/38), done.\u001b[K\n",
            "remote: Compressing objects: 100% (14/14), done.\u001b[K\n",
            "remote: Total 397 (delta 26), reused 30 (delta 24), pack-reused 359\u001b[K\n",
            "Receiving objects: 100% (397/397), 17.32 MiB | 14.79 MiB/s, done.\n",
            "Resolving deltas: 100% (233/233), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# sequential downloading\n",
        "\n",
        "# pull all the data from the ...\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/dev_craters.hdf5?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/dev_images.hdf5?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/LunarLROLrocKaguya_118mperpix.png?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/model_keras1.2.2.h5?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/model_keras2.h5?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/post-processed_sample_images.zip?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/post-processed_test_craters.npy?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/test_craters.hdf5?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/test_images.hdf5?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/train_craters.hdf5?download=1\n",
        "!wget  -P /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/ https://zenodo.org/records/1133969/files/train_images.hdf5?download=1\n"
      ],
      "metadata": {
        "id": "baOkyFAL2Rpd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "  # Define download URLs\n",
        "download_urls = [\n",
        "    \"https://zenodo.org/records/1133969/files/dev_craters.hdf5?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/dev_images.hdf5?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/LunarLROLrocKaguya_118mperpix.png?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/model_keras1.2.2.h5?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/model_keras2.h5?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/post-processed_sample_images.zip?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/post-processed_test_craters.npy?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/test_craters.hdf5?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/test_images.hdf5?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/train_craters.hdf5?download=1\",\n",
        "    \"https://zenodo.org/records/1133969/files/train_images.hdf5?download=1\"\n",
        "]\n",
        "\n",
        "# Define destination directory\n",
        "destination_dir = \"/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/\"\n",
        "\n",
        "# Define function to download a single URL\n",
        "def download_url(url):\n",
        "    !wget -P \"$destination_dir\" \"$url\"\n",
        "\n",
        "# Import ThreadPoolExecutor to run tasks in parallel\n",
        "from concurrent.futures import ThreadPoolExecutor\n",
        "\n",
        "# Download files in parallel using ThreadPoolExecutor\n",
        "with ThreadPoolExecutor(max_workers=10) as executor:\n",
        "    executor.map(download_url, download_urls)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Io_-KmZR15mk",
        "outputId": "b7dad1a8-789a-448a-d847-dbd26b1a53d7"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/dev_images.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... --2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/LunarLROLrocKaguya_118mperpix.png?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... --2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/model_keras1.2.2.h5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... --2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/dev_craters.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... --2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/post-processed_sample_images.zip?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... --2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/post-processed_test_craters.npy?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... --2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/test_craters.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... connected.\n",
            "connected.\n",
            "--2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/test_images.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... --2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/train_craters.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... connected.\n",
            "--2024-05-12 17:28:55--  https://zenodo.org/records/1133969/files/model_keras2.h5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.103.159|:443... connected.\n",
            "connected.\n",
            "connected.\n",
            "HTTP request sent, awaiting response... connected.\n",
            "connected.\n",
            "connected.\n",
            "connected.\n",
            "HTTP request sent, awaiting response... HTTP request sent, awaiting response... HTTP request sent, awaiting response... HTTP request sent, awaiting response... HTTP request sent, awaiting response... HTTP request sent, awaiting response... HTTP request sent, awaiting response... HTTP request sent, awaiting response... HTTP request sent, awaiting response... 200 OK\n",
            "Length: 33669382 (32M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_sample_images.zip?download=1.1’\n",
            "\n",
            "          post-proc   0%[                    ]       0  --.-KB/s               200 OK\n",
            "Length: 9878694864 (9.2G) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_images.hdf5?download=1.1’\n",
            "\n",
            "          dev_image   0%[                    ]       0  --.-KB/s               200 OK\n",
            "Length: 274371452 (262M) [image/png]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/LunarLROLrocKaguya_118mperpix.png?download=1.1’\n",
            "\n",
            "          LunarLROL   0%[                    ]       0  --.-KB/s               200 OK\n",
            "Length: 9878694864 (9.2G) [application/octet-stream]\n",
            "200 OK\n",
            "Length: 181405180 (173M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_craters.hdf5?download=1.1’\n",
            "\n",
            "          dev_crate   0%[                    ]       0  --.-KB/s               200 OK\n",
            "Length: 123444040 (118M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras2.h5?download=1.1’\n",
            "\n",
            "          model_ker   0%[                    ]       0  --.-KB/s               200 OK\n",
            "Length: 375872 (367K) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_test_craters.npy?download=1.1’\n",
            "\n",
            "          post-proc   0%[                    ]       0  --.-KB/s               200 OK\n",
            "Length: 264761344 (252M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_craters.hdf5?download=1.1’\n",
            "\n",
            "          test_crat   0%[                    ]       0  --.-KB/s               Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_images.hdf5?download=1.1’\n",
            "\n",
            "          test_imag   0%[                    ]       0  --.-KB/s               200 OK\n",
            "Length: 261653680 (250M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/train_craters.hdf5?download=1’\n",
            "\n",
            "        dev_images.   0%[                    ] 257.55K   574KB/s               200 OK\n",
            "Length: 123423448 (118M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras1.2.2.h5?download=1.1’\n",
            "\n",
            "post-processed_test 100%[===================>] 367.06K   837KB/s    in 0.4s    \n",
            "\n",
            "2024-05-12 17:28:56 (837 KB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_test_craters.npy?download=1.1’ saved [375872/375872]\n",
            "\n",
            "       post-process   2%[                    ] 725.90K  1.12MB/s               --2024-05-12 17:28:56--  https://zenodo.org/records/1133969/files/train_images.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.103.159, 188.185.79.172, 188.184.98.238, ...\n",
            "       model_keras2   0%[                    ] 903.37K  1.34MB/s               connected.\n",
            "    post-processed_  10%[=>                  ]   3.29M  2.66MB/s               200 OK\n",
            "Length: 9878694864 (9.2G) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/train_images.hdf5?download=1’\n",
            "\n",
            "post-processed_samp 100%[===================>]  32.11M  7.74MB/s    in 5.3s    \n",
            "\n",
            "2024-05-12 17:29:01 (6.05 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_sample_images.zip?download=1.1’ saved [33669382/33669382]\n",
            "\n",
            "model_keras1.2.2.h5 100%[===================>] 117.71M  6.68MB/s    in 18s     \n",
            "\n",
            "2024-05-12 17:29:14 (6.44 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras1.2.2.h5?download=1.1’ saved [123423448/123423448]\n",
            "\n",
            "model_keras2.h5?dow 100%[===================>] 117.72M  6.34MB/s    in 19s     \n",
            "\n",
            "2024-05-12 17:29:14 (6.27 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras2.h5?download=1.1’ saved [123444040/123444040]\n",
            "\n",
            "dev_craters.hdf5?do 100%[===================>] 173.00M  10.0MB/s    in 24s     \n",
            "\n",
            "2024-05-12 17:29:20 (7.18 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_craters.hdf5?download=1.1’ saved [181405180/181405180]\n",
            "\n",
            "test_craters.hdf5?d 100%[===================>] 252.50M  6.11MB/s    in 35s     \n",
            "\n",
            "2024-05-12 17:29:30 (7.30 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_craters.hdf5?download=1.1’ saved [264761344/264761344]\n",
            "\n",
            "train_craters.hdf5? 100%[===================>] 249.53M  6.85MB/s    in 35s     \n",
            "\n",
            "2024-05-12 17:29:30 (7.19 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/train_craters.hdf5?download=1’ saved [261653680/261653680]\n",
            "\n",
            "LunarLROLrocKaguya_ 100%[===================>] 261.66M  9.41MB/s    in 36s     \n",
            "\n",
            "2024-05-12 17:29:31 (7.36 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/LunarLROLrocKaguya_118mperpix.png?download=1.1’ saved [274371452/274371452]\n",
            "\n",
            "ad=1.1               51%[=========>          ]   4.78G  7.62MB/s    eta 6m 45s ^C\n",
            "^C\n",
            "^C\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-a514556bfd45>\u001b[0m in \u001b[0;36m<cell line: 27>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;31m# Download files in parallel using ThreadPoolExecutor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mThreadPoolExecutor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexecutor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m   \u001b[0mexecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdownload_url\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdownload_urls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, exc_type, exc_val, exc_tb)\u001b[0m\n\u001b[1;32m    647\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_tb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 649\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    650\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    651\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/thread.py\u001b[0m in \u001b[0;36mshutdown\u001b[0;34m(self, wait, cancel_futures)\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_threads\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m     \u001b[0mshutdown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_base\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1094\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1095\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1096\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1097\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1098\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1115\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1116\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1117\u001b[0m                 \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1118\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# other method for || download:\n",
        "# Define download URLs\n",
        "%%bash\n",
        "download_urls=(\n",
        "    \"https://zenodo.org/records/1133969/files/dev_craters.hdf5?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/dev_images.hdf5?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/LunarLROLrocKaguya_118mperpix.png?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/model_keras1.2.2.h5?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/model_keras2.h5?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/post-processed_sample_images.zip?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/post-processed_test_craters.npy?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/test_craters.hdf5?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/test_images.hdf5?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/train_craters.hdf5?download=1\"\n",
        "    \"https://zenodo.org/records/1133969/files/train_images.hdf5?download=1\"\n",
        ")\n",
        "\n",
        "# Define destination directory\n",
        "destination_dir=\"/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/\"\n",
        "\n",
        "sudo apt-get install parallel\n",
        "# Download files in parallel\n",
        "parallel -j 10 wget -P \"$destination_dir\" ::: \"${download_urls[@]}\"\n"
      ],
      "metadata": {
        "id": "VFMSYZor3gZU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oni2xuG49VIx",
        "outputId": "24a832e2-9137-4651-b3b7-31e8af6b1f0d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-05-12 17:19:51--  https://zenodo.org/records/1133969/files/dev_craters.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.185.79.172, 188.184.103.159, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 181405180 (173M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_craters.hdf5?download=1’\n",
            "\n",
            "dev_craters.hdf5?do 100%[===================>] 173.00M  23.9MB/s    in 8.4s    \n",
            "\n",
            "2024-05-12 17:20:00 (20.5 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_craters.hdf5?download=1’ saved [181405180/181405180]\n",
            "\n",
            "--2024-05-12 17:20:01--  https://zenodo.org/records/1133969/files/dev_images.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.185.79.172, 188.184.103.159, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 9878694864 (9.2G) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_images.hdf5?download=1’\n",
            "\n",
            "dev_images.hdf5?dow 100%[===================>]   9.20G  25.3MB/s    in 6m 25s  \n",
            "\n",
            "2024-05-12 17:26:26 (24.5 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_images.hdf5?download=1’ saved [9878694864/9878694864]\n",
            "\n",
            "--2024-05-12 17:26:26--  https://zenodo.org/records/1133969/files/LunarLROLrocKaguya_118mperpix.png?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.184.103.159, 188.185.79.172, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 274371452 (262M) [image/png]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/LunarLROLrocKaguya_118mperpix.png?download=1’\n",
            "\n",
            "  LunarLROLrocKaguy   8%[>                   ]  21.81M  12.6MB/s               ^C\n",
            "--2024-05-12 17:26:29--  https://zenodo.org/records/1133969/files/model_keras1.2.2.h5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.184.103.159, 188.185.79.172, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 123423448 (118M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras1.2.2.h5?download=1’\n",
            "\n",
            "model_keras1.2.2.h5 100%[===================>] 117.71M  18.8MB/s    in 6.6s    \n",
            "\n",
            "2024-05-12 17:26:36 (17.7 MB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras1.2.2.h5?download=1’ saved [123423448/123423448]\n",
            "\n",
            "--2024-05-12 17:26:36--  https://zenodo.org/records/1133969/files/model_keras2.h5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.185.79.172, 188.184.103.159, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 123444040 (118M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras2.h5?download=1’\n",
            "\n",
            "keras2.h5?download=   4%[                    ]   5.76M  1.61MB/s    eta 81s    ^C\n",
            "--2024-05-12 17:26:41--  https://zenodo.org/records/1133969/files/post-processed_sample_images.zip?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.185.79.172, 188.184.103.159, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 33669382 (32M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_sample_images.zip?download=1’\n",
            "\n",
            "         post-proce   0%[                    ]  46.23K   201KB/s               ^C\n",
            "--2024-05-12 17:26:42--  https://zenodo.org/records/1133969/files/post-processed_test_craters.npy?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.185.79.172, 188.184.103.159, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 375872 (367K) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_test_craters.npy?download=1’\n",
            "\n",
            "post-processed_test 100%[===================>] 367.06K   677KB/s    in 0.5s    \n",
            "\n",
            "2024-05-12 17:26:43 (677 KB/s) - ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_test_craters.npy?download=1’ saved [375872/375872]\n",
            "\n",
            "--2024-05-12 17:26:43--  https://zenodo.org/records/1133969/files/test_craters.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.185.79.172, 188.184.103.159, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 264761344 (252M) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_craters.hdf5?download=1’\n",
            "\n",
            "_craters.hdf5?downl  14%[=>                  ]  36.79M  11.4MB/s    eta 21s    ^C\n",
            "--2024-05-12 17:26:47--  https://zenodo.org/records/1133969/files/test_images.hdf5?download=1\n",
            "Resolving zenodo.org (zenodo.org)... 188.184.98.238, 188.185.79.172, 188.184.103.159, ...\n",
            "Connecting to zenodo.org (zenodo.org)|188.184.98.238|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 9878694864 (9.2G) [application/octet-stream]\n",
            "Saving to: ‘/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_images.hdf5?download=1’\n",
            "\n",
            "    test_images.hdf   0%[                    ]   1.05M   686KB/s               "
          ]
        }
      ],
      "source": [
        "\n",
        "#rename all the data files\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_craters.hdf5?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_craters.hdf5\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_images.hdf5?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/dev_images.hdf5\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/LunarLROLrocKaguya_118mperpix.png?download=1  /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/LunarLROLrocKaguya_118mperpix.png\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras1.2.2.h5?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras1.2.2.h5\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras2.h5?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/model_keras2.h5\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_sample_images.zip?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_sample_images.zip\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_test_craters.npy?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/post-processed_test_craters.npy\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_craters.hdf5?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_craters.hdf5\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_images.hdf5?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/test_images.hdf5\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/train_craters.hdf5?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/train_craters.hdf5\n",
        "!mv /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/train_images.hdf5?download=1 /content/drive/MyDrive/comp_vis/DeepMoon/catalogues/train_images.hdf5\n",
        "\n",
        "# wastage code because it is on colab :(\n",
        "    # Make a directory\n",
        "  # !mkdir -p /content/drive/MyDrive/comp_vis\n",
        "\n",
        "  # # Clone repo in that directory\n",
        "  # !git clone https://github.com/Akshat6133/DeepMoon /content/drive/MyDrive/comp_vis\n",
        "\n",
        "  # # Define file URLs and names\n",
        "  # file_urls = [\n",
        "  #     \"https://zenodo.org/records/1133969/files/dev_craters.hdf5?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/dev_images.hdf5?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/LunarLROLrocKaguya_118mperpix.png?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/model_keras1.2.2.h5?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/model_keras2.h5?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/post-processed_sample_images.zip?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/post-processed_test_craters.npy?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/test_craters.hdf5?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/test_images.hdf5?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/train_craters.hdf5?download=1\",\n",
        "  #     \"https://zenodo.org/records/1133969/files/train_images.hdf5?download=1\"\n",
        "  # ]\n",
        "  # file_names = [\n",
        "  #     \"dev_craters.hdf5\",\n",
        "  #     \"dev_images.hdf5\",\n",
        "  #     \"LunarLROLrocKaguya_118mperpix.png\",\n",
        "  #     \"model_keras1.2.2.h5\",\n",
        "  #     \"model_keras2.h5\",\n",
        "  #     \"post-processed_sample_images.zip\",\n",
        "  #     \"post-processed_test_craters.npy\",\n",
        "  #     \"test_craters.hdf5\",\n",
        "  #     \"test_images.hdf5\",\n",
        "  #     \"train_craters.hdf5\",\n",
        "  #     \"train_images.hdf5\"\n",
        "  # ]\n",
        "\n",
        "  # # Download and rename files\n",
        "  # for url, name in zip(file_urls, file_names):\n",
        "  #     !wget -O \"/content/drive/MyDrive/comp_vis/DeepMoon/catalogues/{name}\" \"{url}\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GlTnDg3S4cdr"
      },
      "outputs": [],
      "source": [
        "# remove default python path\n",
        "%env PYTHONPATH=\n",
        "\n",
        "# install miniconda\n",
        "!mkdir -p ~/miniconda3\n",
        "!wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh\n",
        "!bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3\n",
        "!rm -rf ~/miniconda3/miniconda.sh\n",
        "#------------------------------------\n",
        "# followed instructions from the miniconda docs\n",
        "!~/miniconda3/bin/conda init bash"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kVTk_GBt4lT6"
      },
      "outputs": [],
      "source": [
        "# Specify the path to the Conda executable\n",
        "conda_path = '/root/miniconda3/bin/conda'\n",
        "\n",
        "# Use the specified Conda path to create a new environment with Python 3.5\n",
        "!{conda_path} create -y -n myenv python=3.5\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# update the repo before moving forward\n",
        "!git -C /content/drive/MyDrive/comp_vis/DeepMoon pull"
      ],
      "metadata": {
        "id": "cjjdXC22V-W9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1GkKoAsd5IMY"
      },
      "outputs": [],
      "source": [
        "import subprocess\n",
        "\n",
        "# Specify the path to the Python executable in the Conda environment\n",
        "python_executable = '/root/miniconda3/envs/myenv/bin/python'\n",
        "\n",
        "# path to the requirements.txt file\n",
        "requirements_file = '/content/drive/MyDrive/comp_vis/DeepMoon/requirements.txt'\n",
        "# Check Python version\n",
        "version_check = subprocess.run([python_executable, \"--version\"], capture_output=True, text=True)\n",
        "print(version_check.stdout)\n",
        "\n",
        "# Check system whether Conda environment or not\n",
        "sys_check = subprocess.run([python_executable, \"-c\", \"import sys; print(sys.version)\"], capture_output=True, text=True)\n",
        "print(sys_check.stdout)\n",
        "\n",
        "# Install dependencies using pip with the Python executable from the Conda environment\n",
        "result = subprocess.run([python_executable, '-m', 'pip', 'install', '-r', requirements_file], capture_output=True, text=True)\n",
        "print(result.stdout)\n",
        "\n",
        "# Check installed package versions\n",
        "installed_packages = subprocess.run([python_executable, '-m', 'pip', 'list'], capture_output=True, text=True)\n",
        "print(installed_packages.stdout)\n",
        "\n",
        "\n",
        "# result = subprocess.run([python_executable, \"model_train.py\"], capture_output=True, text=True)\n",
        "# result = subprocess.run([python_executable, '-m', 'pip', 'install', '-r', requirements_file], capture_output=True, text=True)\n",
        "# result = subprocess.run([python_executable, '-m', 'pip', 'install', '-r', requirements_file], capture_output=True, text=True)\n",
        "# result = subprocess.run([python_executable, '-m', 'pip', 'install', '-r', requirements_file], capture_output=True, text=True)\n",
        "# result = subprocess.run([python_executable, '-m', 'pip', 'install', '-r', requirements_file], capture_output=True, text=True)\n",
        "\n",
        "# Print the output of the installation process\n",
        "# print(result.stdout)\n",
        "# print(result.stdout)\n",
        "# print(result.stdout)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q_lBZq4r_nPk"
      },
      "outputs": [],
      "source": [
        "# trying to set the default path\n",
        "%env PYTHONPATH= \"/root/miniconda3/envs/myenv/bin/python\"\n",
        "!python --version\n",
        "#Failure"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Farjiwada\n",
        "\n",
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "8HTMZfC6RJWd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F0_oGDd77Ln7"
      },
      "outputs": [],
      "source": [
        "# from os import getcwd\n",
        "# path = getcwd() + \"/lunar_data/\"\n",
        "!cd /root/miniconda3/envs/myenv/bin\n",
        "!ls /root/miniconda3/envs/myenv/bin"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "09VOmViG0Wqh"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1k3BnMNQTwFl8h_mEK79b8NRtoT5xGsI2",
      "authorship_tag": "ABX9TyOREHqy0zFoPxIWXoiSYSkH",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}