guides/extract_data.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "AB2A-pf9kzzL"
      },
      "outputs": [],
      "source": [
        "# RUN THIS CELL EVERY RESET RUNTIME\n",
        "WORKDIR='/content/AIC2022-VER'\n",
        "DATADIR='/content/AIC2022-VER/data/' # replace this path\n",
        "GITHUB_TOKEN=''"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oUEeZwdFgNHr"
      },
      "source": [
        "## Clone repo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "2ci-7ESN_Y5-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "50a0e46f-ceae-4272-c330-1c1e595dfc64"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: gdown in /usr/local/lib/python3.7/dist-packages (4.4.0)\n",
            "Requirement already satisfied: google-cloud-storage in /usr/local/lib/python3.7/dist-packages (2.3.0)\n",
            "Collecting python-Levenshtein\n",
            "  Downloading python-Levenshtein-0.12.2.tar.gz (50 kB)\n",
            "\u001b[K     |████████████████████████████████| 50 kB 30.0 MB/s \n",
            "\u001b[?25hCollecting webcolors\n",
            "  Downloading webcolors-1.11.1-py3-none-any.whl (9.9 kB)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from gdown) (4.6.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from gdown) (1.15.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from gdown) (4.64.0)\n",
            "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.7/dist-packages (from gdown) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from gdown) (3.6.0)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage) (3.17.3)\n",
            "Requirement already satisfied: google-resumable-media>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage) (2.3.2)\n",
            "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage) (1.31.5)\n",
            "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage) (2.3.0)\n",
            "Requirement already satisfied: google-auth<3.0dev,>=1.25.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage) (1.35.0)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage) (1.56.0)\n",
            "Requirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage) (59.5.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage) (2018.9)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.7/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage) (21.3)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage) (4.2.4)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage) (4.8)\n",
            "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.7/dist-packages (from google-resumable-media>=2.3.2->google-cloud-storage) (1.3.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=14.3->google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage) (3.0.8)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-cloud-storage) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (1.25.11)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (2.10)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (1.7.1)\n",
            "Building wheels for collected packages: python-Levenshtein\n",
            "  Building wheel for python-Levenshtein (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-Levenshtein: filename=python_Levenshtein-0.12.2-cp37-cp37m-linux_x86_64.whl size=149859 sha256=b546b22b022fe062a4dbd2adc833e73f90ec09ec41ea3e95cbe80e29ce1db6f3\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-1a66asns/wheels/05/5f/ca/7c4367734892581bb5ff896f15027a932c551080b2abd3e00d\n",
            "Successfully built python-Levenshtein\n",
            "Installing collected packages: webcolors, python-Levenshtein\n",
            "Successfully installed python-Levenshtein-0.12.2 webcolors-1.11.1\n"
          ]
        }
      ],
      "source": [
        "!pip install --upgrade --no-cache-dir gdown google-cloud-storage python-Levenshtein webcolors"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6noGGK9fiyBW",
        "outputId": "60ee8526-a3c9-4c9e-9467-0c891f2012ec"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'AIC2022-VER'...\n",
            "remote: Enumerating objects: 2253, done.\u001b[K\n",
            "remote: Counting objects: 100% (2253/2253), done.\u001b[K\n",
            "remote: Compressing objects: 100% (1176/1176), done.\u001b[K\n",
            "remote: Total 2253 (delta 1129), reused 2038 (delta 965), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (2253/2253), 12.82 MiB | 34.45 MiB/s, done.\n",
            "Resolving deltas: 100% (1129/1129), done.\n"
          ]
        }
      ],
      "source": [
        "REPO_URL=f'https://{GITHUB_TOKEN}@github.com/nhtlongcs/AIC2022-VER/'\n",
        "!git clone $REPO_URL"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "UfN3fZBVpOJ_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7e79fd59-b0f2-45ce-d53b-2676f70f37d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/AIC2022-VER\n",
            "Already on 'main'\n",
            "Your branch is up to date with 'origin/main'.\n",
            "HEAD is now at fa8a6a1 Update workflows.yml\n",
            "Already up to date.\n"
          ]
        }
      ],
      "source": [
        "%cd $WORKDIR\n",
        "!git checkout main\n",
        "!git reset --hard HEAD\n",
        "!git pull"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "TLDc3bwUk5PZ"
      },
      "outputs": [],
      "source": [
        "!chmod +x $WORKDIR/tools/*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "pxPgv5J0lb_N"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "%cd $WORKDIR\n",
        "!pip install -r requirements.txt\n",
        "!pip install -e . "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "79AEtMK2odjV"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!python -m spacy download en_core_web_sm\n",
        "!python -m nltk.downloader punkt wordnet omw-1.4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "PpSaOVemhdCh"
      },
      "outputs": [],
      "source": [
        "import gdown \n",
        "def download(token, output: str):\n",
        "    url = f'https://drive.google.com/uc?id={token}'\n",
        "    gdown.download(url, output, quiet=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FAvtoTtYgPM9"
      },
      "source": [
        "## Download data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd $DATADIR\n",
        "# download raw data\n",
        "download('1NvtjIe8-wUYiWu9VD52dWPejgA5DvGcO','aic2022.zip') # replicate link"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nd95jhe1gktY",
        "outputId": "d6ef67eb-7079-4831-9fe8-2465e0d241a6"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/AIC2022-VER/data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1NvtjIe8-wUYiWu9VD52dWPejgA5DvGcO\n",
            "To: /content/AIC2022-VER/data/aic2022.zip\n",
            "100%|██████████| 16.2G/16.2G [03:02<00:00, 88.9MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!unzip aic2022.zip -d AIC22_Track2_NL_Retrieval/"
      ],
      "metadata": {
        "id": "mxMGNiISjzoO"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Extract to frames"
      ],
      "metadata": {
        "id": "lbBeVL_6hRKI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir $DATADIR/meta\n",
        "%cp $DATADIR/AIC22_Track2_NL_Retrieval/*.json $DATADIR/meta/"
      ],
      "metadata": {
        "id": "4l2uSqE_kaUr"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "%cd $WORKDIR\n",
        "!./tools/extract_vdo2frms_AIC.sh \\\n",
        "      $DATADIR/AIC22_Track2_NL_Retrieval/ \\\n",
        "      $DATADIR/meta/extracted_frames/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "COlaJtxxg-ae",
        "outputId": "b13ea2e5-2d21-4aa5-c08c-86027a832489"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/AIC2022-VER\n",
            "Processing [TRAIN][S01][c001]\n",
            "Processing [TRAIN][S01][c002]\n",
            "Processing [TRAIN][S01][c003]\n",
            "Processing [TRAIN][S01][c004]\n",
            "Processing [TRAIN][S01][c005]\n",
            "Processing [TRAIN][S03][c010]\n",
            "Processing [TRAIN][S03][c011]\n",
            "Processing [TRAIN][S03][c012]\n",
            "Processing [TRAIN][S03][c013]\n",
            "Processing [TRAIN][S03][c014]\n",
            "Processing [TRAIN][S03][c015]\n",
            "Processing [TRAIN][S04][c016]\n",
            "Processing [TRAIN][S04][c017]\n",
            "Processing [TRAIN][S04][c018]\n",
            "Processing [TRAIN][S04][c019]\n",
            "Processing [TRAIN][S04][c020]\n",
            "Processing [TRAIN][S04][c021]\n",
            "Processing [TRAIN][S04][c022]\n",
            "Processing [TRAIN][S04][c023]\n",
            "Processing [TRAIN][S04][c024]\n",
            "Processing [TRAIN][S04][c025]\n",
            "Processing [TRAIN][S04][c026]\n",
            "Processing [TRAIN][S04][c027]\n",
            "Processing [TRAIN][S04][c028]\n",
            "Processing [TRAIN][S04][c029]\n",
            "Processing [TRAIN][S04][c030]\n",
            "Processing [TRAIN][S04][c031]\n",
            "Processing [TRAIN][S04][c032]\n",
            "Processing [TRAIN][S04][c033]\n",
            "Processing [TRAIN][S04][c034]\n",
            "Processing [TRAIN][S04][c035]\n",
            "Processing [TRAIN][S04][c036]\n",
            "Processing [TRAIN][S04][c037]\n",
            "Processing [TRAIN][S04][c038]\n",
            "Processing [TRAIN][S04][c039]\n",
            "Processing [TRAIN][S04][c040]\n",
            "Processing [VALID][S02][c006]\n",
            "Processing [VALID][S02][c007]\n",
            "Processing [VALID][S02][c008]\n",
            "Processing [VALID][S02][c009]\n",
            "Processing [VALID][S05][c010]\n",
            "Processing [VALID][S05][c016]\n",
            "Processing [VALID][S05][c017]\n",
            "Processing [VALID][S05][c018]\n",
            "Processing [VALID][S05][c019]\n",
            "Processing [VALID][S05][c020]\n",
            "Processing [VALID][S05][c021]\n",
            "Processing [VALID][S05][c022]\n",
            "Processing [VALID][S05][c023]\n",
            "Processing [VALID][S05][c024]\n",
            "Processing [VALID][S05][c025]\n",
            "Processing [VALID][S05][c026]\n",
            "Processing [VALID][S05][c027]\n",
            "Processing [VALID][S05][c028]\n",
            "Processing [VALID][S05][c029]\n",
            "Processing [VALID][S05][c033]\n",
            "Processing [VALID][S05][c034]\n",
            "Processing [VALID][S05][c035]\n",
            "Processing [VALID][S05][c036]\n",
            "CPU times: user 28.6 s, sys: 4.34 s, total: 33 s\n",
            "Wall time: 1h 2min 14s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extract meta data"
      ],
      "metadata": {
        "id": "5w-SpFV6hWUr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VTNehmexk2ql",
        "outputId": "8b412541-9607-45cc-f362-07b8351ac9a9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/AIC2022-VER\n",
            "Extracting motion map ...\n",
            "59it [00:35,  1.67it/s]\n",
            "2339it [1:45:02,  2.69s/it]\n",
            "DONE\n",
            "CPU times: user 54.1 s, sys: 6.73 s, total: 1min\n",
            "Wall time: 1h 45min 40s\n"
          ]
        }
      ],
      "source": [
        "%%time \n",
        "%cd $WORKDIR\n",
        "!./tools/preproc_motion.sh $DATADIR/meta/"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "%cd $WORKDIR\n",
        "!./tools/preproc_srl.sh $DATADIR/meta"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nbYcOeWjiHYU",
        "outputId": "61cbe93f-bdee-4d8d-e24e-05f9b530516e"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/AIC2022-VER\n",
            "Augmenting data ...\n",
            "100% 2155/2155 [03:26<00:00, 10.46it/s]\n",
            "DONE\n",
            "100% 184/184 [00:10<00:00, 17.72it/s]\n",
            "DONE\n",
            "Extracting SRL data ...\n",
            "Load data_info successfully\n",
            "downloading: 100%|##########| 387M/387M [00:10<00:00, 38.6MiB/s]\n",
            "Downloading: 100% 28.0/28.0 [00:00<00:00, 21.5kB/s]\n",
            "Downloading: 100% 570/570 [00:00<00:00, 403kB/s]\n",
            "Downloading: 100% 226k/226k [00:00<00:00, 15.7MB/s]\n",
            "Downloading: 100% 455k/455k [00:00<00:00, 18.4MB/s]\n",
            "Downloading: 100% 420M/420M [00:10<00:00, 42.7MB/s]\n",
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Load other info successfully\n",
            "  0% 6/2155 [00:04<19:41,  1.82it/s](1) 7_1: a green go to the straight .\n",
            "  2% 47/2155 [00:24<17:29,  2.01it/s](2) 48_1: a white suv among other vehicles .\n",
            "  4% 85/2155 [00:44<15:58,  2.16it/s](3) 86_2: a black suv in traffic on the highway .\n",
            "  4% 86/2155 [00:44<17:07,  2.01it/s](4) 87_2: a black suv in traffic on the highway .\n",
            "  4% 87/2155 [00:45<17:53,  1.93it/s](5) 88_3: a black suv in traffic on the highway .\n",
            "  6% 140/2155 [01:11<15:54,  2.11it/s](6) 141_1: a cargo-truck with a flatbed drove to straight .\n",
            " 11% 236/2155 [01:58<13:35,  2.35it/s](7) 237_3: a white mini-cooper drove to straight through the intersection .\n",
            " 11% 237/2155 [01:59<12:26,  2.57it/s](8) 238_3: white mini-cooper .\n",
            " 11% 247/2155 [02:03<14:09,  2.25it/s](9) 248_3: a red suv .\n",
            " 13% 288/2155 [02:23<13:56,  2.23it/s](10) 289_1: sedan medium and agile .\n",
            " 14% 301/2155 [02:33<17:04,  1.81it/s](11) 302_1: gray suv with a red suv up ahead .\n",
            " 15% 334/2155 [02:49<14:13,  2.13it/s](12) 335_1: a door white pickup .\n",
            " 16% 349/2155 [02:56<15:43,  1.91it/s](13) 350_2: door sedan luxury car and its very simple with beautiful .\n",
            " 18% 390/2155 [03:17<15:07,  1.94it/s](14) 391_2: a maroon car .\n",
            " 19% 418/2155 [03:32<17:41,  1.64it/s](15) 419_3: blue suv bs to the left .\n",
            " 21% 459/2155 [03:54<19:35,  1.44it/s](16) 460_1: gray door sedan with no other traffic on the road .\n",
            " 22% 468/2155 [03:59<17:33,  1.60it/s](17) 469_1: a big suv .\n",
            " 22% 471/2155 [04:00<13:22,  2.10it/s](18) 472_3: a big suv .\n",
            " 22% 472/2155 [04:00<12:07,  2.31it/s](19) 473_1: a big suv .\n",
            " 22% 482/2155 [04:05<13:15,  2.10it/s](20) 483_3: a midsize crossover .\n",
            " 24% 523/2155 [04:26<14:31,  1.87it/s](21) 524_1: a gray sedan .\n",
            " 24% 524/2155 [04:27<14:06,  1.93it/s](22) 525_1: a gray sedan .\n",
            " 26% 556/2155 [04:43<13:55,  1.91it/s](23) 557_1: a large midsize black sedan .\n",
            " 27% 572/2155 [04:51<14:10,  1.86it/s](24) 573_1: a red suv .\n",
            " 27% 574/2155 [04:52<12:08,  2.17it/s](25) 575_1: a red suv turnedn illegal to the left turn .\n",
            " 27% 585/2155 [04:58<13:29,  1.94it/s](26) 586_2: white suv with a gray suv ahead .\n",
            " 27% 590/2155 [05:01<14:53,  1.75it/s](27) 591_1: white suv with a gray suv ahead .\n",
            " 32% 691/2155 [05:55<11:30,  2.12it/s](28) 692_2: a gray suv downs in the street .\n",
            " 32% 693/2155 [05:56<10:47,  2.26it/s](29) 694_1: a gray suv downs in the street .\n",
            " 33% 714/2155 [06:06<11:25,  2.10it/s](30) 715_3: a typical jeep .\n",
            " 33% 719/2155 [06:09<12:22,  1.93it/s](31) 720_1: a typical jeep .\n",
            " 38% 816/2155 [07:03<12:38,  1.77it/s](32) 817_1: gray suv with no car .\n",
            " 38% 817/2155 [07:03<10:53,  2.05it/s](33) 818_3: gray suv with no car .\n",
            " 39% 837/2155 [07:14<11:20,  1.94it/s](34) 838_3: a small sedan among other cars .\n",
            " 39% 843/2155 [07:16<10:21,  2.11it/s](35) 844_3: a small sedan among other cars .\n",
            " 39% 844/2155 [07:17<10:35,  2.06it/s](36) 845_3: a large burgundy van .\n",
            " 39% 848/2155 [07:19<10:34,  2.06it/s](37) 849_1: a large burgundy van .\n",
            " 40% 866/2155 [07:28<11:38,  1.85it/s](38) 867_2: small gray suv on a busy road .\n",
            " 40% 867/2155 [07:28<10:48,  1.99it/s](39) 868_3: small gray suv on a busy road .\n",
            " 40% 868/2155 [07:29<10:45,  1.99it/s](40) 869_1: small gray suv on a busy road .\n",
            " 42% 897/2155 [07:45<12:54,  1.62it/s](41) 898_3: light short to the right .\n",
            " 43% 919/2155 [07:57<11:47,  1.75it/s](42) 920_3: a silver suv rides down the street .\n",
            " 43% 920/2155 [07:57<10:12,  2.02it/s](43) 921_3: a silver suv rides down the street .\n",
            " 43% 934/2155 [08:05<11:15,  1.81it/s](44) 935_2: a green pickup with items in the black .\n",
            " 44% 940/2155 [08:08<10:11,  1.99it/s](45) 941_3: a green pickup with items in the black .\n",
            " 44% 941/2155 [08:08<09:54,  2.04it/s](46) 942_2: a gray pickup .\n",
            " 44% 944/2155 [08:10<10:00,  2.02it/s](47) 945_1: a green pickup with items in the black .\n",
            " 44% 945/2155 [08:10<09:27,  2.13it/s](48) 946_1: a green pickup with items in the black .\n",
            " 55% 1195/2155 [10:25<08:21,  1.92it/s](49) 1196_3: a white jeep crues down straight down the street .\n",
            " 56% 1197/2155 [10:26<07:17,  2.19it/s](50) 1198_2: a white jeep crues down straight down the street .\n",
            " 63% 1366/2155 [11:59<08:07,  1.62it/s](51) 1367_3: a large burgundy hatchback .\n",
            " 64% 1371/2155 [12:01<07:07,  1.83it/s](52) 1372_2: a large burgundy hatchback .\n",
            " 66% 1419/2155 [12:27<06:43,  1.82it/s](53) 1420_3: sedan door .\n",
            " 67% 1454/2155 [12:49<06:02,  1.93it/s](54) 1455_1: a black suv .\n",
            " 68% 1467/2155 [12:55<04:59,  2.30it/s](55) 1468_2: a gray sedan .\n",
            " 76% 1638/2155 [14:26<04:13,  2.04it/s](56) 1639_1: a red suv .\n",
            " 76% 1640/2155 [14:27<03:48,  2.25it/s](57) 1641_3: a red suv .\n",
            " 76% 1642/2155 [14:28<03:52,  2.20it/s](58) 1643_2: a white pickup .\n",
            " 76% 1644/2155 [14:29<03:51,  2.21it/s](59) 1645_2: a white pickup .\n",
            " 76% 1645/2155 [14:29<03:37,  2.34it/s](60) 1646_2: a white pickup .\n",
            " 81% 1742/2155 [15:23<03:22,  2.04it/s](61) 1743_1: white pickup .\n",
            " 81% 1743/2155 [15:24<03:06,  2.21it/s](62) 1744_1: white pickup .\n",
            " 83% 1796/2155 [15:51<03:24,  1.76it/s](63) 1797_2: a white jeep .\n",
            " 86% 1846/2155 [16:19<02:53,  1.78it/s](64) 1847_1: it a white car with black windows .\n",
            " 97% 2096/2155 [18:32<00:28,  2.06it/s](65) 2097_2: black sedan wuth a gray suv ahead .\n",
            " 97% 2097/2155 [18:33<00:27,  2.08it/s](66) 2098_3: black sedan wuth a gray suv ahead .\n",
            " 97% 2098/2155 [18:33<00:27,  2.10it/s](67) 2099_1: black sedan wuth a gray suv ahead .\n",
            " 97% 2101/2155 [18:35<00:27,  2.00it/s](68) 2102_3: a white suv .\n",
            "100% 2155/2155 [19:06<00:00,  1.88it/s]\n",
            "100% 50/50 [00:00<00:00, 138517.31it/s]\n",
            "Load data_info successfully\n",
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Load other info successfully\n",
            " 47% 86/184 [00:42<00:48,  2.00it/s](1) 87_3: a silver sedan .\n",
            " 51% 93/184 [00:45<00:42,  2.14it/s](2) 94_2: a white suv .\n",
            " 60% 110/184 [00:54<00:36,  2.01it/s](3) 111_3: a black midsize crossover suv .\n",
            " 66% 121/184 [01:00<00:33,  1.90it/s](4) 122_3: it a white sedan .\n",
            " 68% 126/184 [01:03<00:32,  1.77it/s](5) 127_1: red sedan straight .\n",
            " 69% 127/184 [01:03<00:28,  2.02it/s](6) 128_2: a red sedan when down the road .\n",
            " 79% 146/184 [01:12<00:18,  2.09it/s](7) 147_2: a black sedan .\n",
            " 89% 164/184 [01:22<00:10,  1.94it/s](8) 165_3: black pickup .\n",
            "100% 184/184 [01:32<00:00,  2.00it/s]\n",
            "100% 6/6 [00:00<00:00, 78398.21it/s]\n",
            "DONE\n",
            "mkdir: cannot create directory ‘/content/AIC2022-VER/data//meta/srl’: File exists\n",
            "mkdir: cannot create directory ‘/content/AIC2022-VER/data//meta/srl/action’: File exists\n",
            "mkdir: cannot create directory ‘/content/AIC2022-VER/data//meta/srl/color’: File exists\n",
            "mkdir: cannot create directory ‘/content/AIC2022-VER/data//meta/srl/veh’: File exists\n",
            "Extracting SRL data [action] ...\n",
            "{'turn': 0, 'curve': 0, 'bare': 0, 'stop': 1, 'park': 1, 'wait': 1, 'pause': 1, 'run': 2, 'go': 2, 'drive': 2, 'keep': 2, 'cross': 2, 'move': 2, 'continue': 2, 'travel': 2, 'proceed': 2, 'merge': 2, 'switch': 2, 'change': 2}\n",
            "  0% 0/2155 [00:00<?, ?it/s]['']\n",
            "['']\n",
            " 11% 228/2155 [00:00<00:01, 1146.03it/s]['']\n",
            " 33% 710/2155 [00:00<00:01, 1055.21it/s]['']\n",
            "['']\n",
            " 38% 819/2155 [00:00<00:01, 980.00it/s] ['']\n",
            "['']\n",
            " 43% 920/2155 [00:00<00:01, 913.08it/s]['']\n",
            " 54% 1156/2155 [00:01<00:01, 975.26it/s] ['']\n",
            " 60% 1284/2155 [00:01<00:00, 1058.66it/s]['']\n",
            "['']\n",
            "['']\n",
            " 68% 1470/2155 [00:01<00:00, 1248.63it/s]['']\n",
            " 76% 1645/2155 [00:01<00:00, 1362.87it/s]['']\n",
            " 85% 1839/2155 [00:01<00:00, 1525.45it/s]['']\n",
            "['']\n",
            "['']\n",
            "100% 2155/2155 [00:01<00:00, 1139.23it/s]\n",
            "save result to /content/AIC2022-VER/data//meta/srl/action directory\n",
            "2 fail queries: ['f1888776-42e4-4270-aabb-184ac16aa411: []', '2774bbbf-3201-4c30-9769-7e3001915611: []']\n",
            "  0% 0/2155 [00:00<?, ?it/s]['']\n",
            "['']\n",
            " 11% 238/2155 [00:00<00:01, 1200.95it/s]['']\n",
            " 33% 720/2155 [00:00<00:01, 1086.12it/s]['']\n",
            "['']\n",
            " 39% 831/2155 [00:00<00:01, 1064.02it/s]['']\n",
            "['']\n",
            " 44% 939/2155 [00:00<00:01, 948.96it/s] ['']\n",
            " 55% 1184/2155 [00:01<00:00, 992.32it/s] ['']\n",
            "['']\n",
            "['']\n",
            " 64% 1372/2155 [00:01<00:00, 1232.69it/s]['']\n",
            " 74% 1585/2155 [00:01<00:00, 1482.83it/s]['']\n",
            "['']\n",
            " 88% 1893/2155 [00:01<00:00, 1485.69it/s]['']\n",
            "['']\n",
            "['']\n",
            "100% 2155/2155 [00:01<00:00, 1195.56it/s]\n",
            "save result to /content/AIC2022-VER/data//meta/srl/action directory\n",
            "2 fail queries: ['f1888776-42e4-4270-aabb-184ac16aa411: []', '2774bbbf-3201-4c30-9769-7e3001915611: []']\n",
            "100% 184/184 [00:00<00:00, 1268.57it/s]\n",
            "save result to /content/AIC2022-VER/data//meta/srl/action directory\n",
            "0 fail queries: []\n",
            "100% 184/184 [00:00<00:00, 1177.70it/s]\n",
            "save result to /content/AIC2022-VER/data//meta/srl/action directory\n",
            "0 fail queries: []\n",
            "DONE\n",
            "Extracting SRL data [color] ...\n",
            "{'gray': 0, 'silver': 0, 'gold': 1, 'tan': 1, 'beige': 1, 'brown': 1, 'yellow': 1, 'orange': 1, 'maroon': 2, 'red': 2, 'blue': 3, 'black': 4, 'purple': 5, 'green': 6, 'white': 7}\n",
            "RUN TRAIN\n",
            "  1% 29/2155 [00:03<04:18,  8.24it/s]['']\n",
            "  2% 47/2155 [00:05<03:42,  9.47it/s]['']\n",
            " 14% 294/2155 [00:33<03:23,  9.13it/s]['']\n",
            " 37% 799/2155 [01:59<03:40,  6.15it/s]['']\n",
            " 37% 806/2155 [02:00<05:08,  4.37it/s]['']\n",
            " 39% 847/2155 [02:08<03:57,  5.50it/s]['']\n",
            " 40% 855/2155 [02:09<03:11,  6.77it/s]['']\n",
            " 46% 987/2155 [02:33<03:42,  5.26it/s]['']\n",
            " 57% 1219/2155 [03:16<03:28,  4.49it/s]['']\n",
            " 63% 1352/2155 [03:38<02:27,  5.46it/s]['']\n",
            " 63% 1358/2155 [03:39<01:40,  7.96it/s]['']\n",
            " 67% 1434/2155 [03:48<01:24,  8.49it/s]['']\n",
            " 74% 1589/2155 [04:06<01:07,  8.33it/s]['']\n",
            " 80% 1728/2155 [04:22<00:49,  8.56it/s]['']\n",
            " 88% 1901/2155 [04:42<00:28,  9.00it/s]['']\n",
            " 88% 1903/2155 [04:42<00:28,  8.81it/s]['']\n",
            " 88% 1904/2155 [04:42<00:27,  8.97it/s]['']\n",
            "100% 2155/2155 [05:11<00:00,  6.92it/s]\n",
            "save result to /content/AIC2022-VER/data//meta/srl/color directory\n",
            "Fail queries: []\n",
            "  1% 29/2155 [00:02<03:17, 10.75it/s]['']\n",
            "  2% 47/2155 [00:04<03:15, 10.81it/s]['']\n",
            " 14% 293/2155 [00:27<02:51, 10.85it/s]['']\n",
            " 37% 798/2155 [01:30<03:04,  7.35it/s]['']\n",
            " 37% 806/2155 [01:31<03:42,  6.06it/s]['']\n",
            " 39% 847/2155 [01:37<03:01,  7.22it/s]['']\n",
            " 40% 854/2155 [01:38<02:50,  7.61it/s]['']\n",
            " 46% 986/2155 [01:56<03:20,  5.82it/s]['']\n",
            " 57% 1219/2155 [02:29<02:36,  5.97it/s]['']\n",
            " 63% 1352/2155 [02:47<01:44,  7.65it/s]['']\n",
            " 63% 1357/2155 [02:47<01:31,  8.77it/s]['']\n",
            " 67% 1434/2155 [02:55<01:05, 10.98it/s]['']\n",
            " 74% 1588/2155 [03:09<00:51, 11.05it/s]['']\n",
            " 80% 1728/2155 [03:22<00:39, 10.91it/s]['']\n",
            " 88% 1900/2155 [03:38<00:23, 10.86it/s]['']\n",
            " 88% 1902/2155 [03:38<00:23, 10.90it/s]['']\n",
            " 88% 1904/2155 [03:38<00:23, 10.79it/s]['']\n",
            "100% 2155/2155 [04:01<00:00,  8.92it/s]\n",
            "save result to /content/AIC2022-VER/data//meta/srl/color directory\n",
            "Fail queries: []\n",
            "RUN TEST\n",
            "100% 184/184 [00:00<00:00, 1235.03it/s]\n",
            "save result to /content/AIC2022-VER/data//meta/srl/color directory\n",
            "Fail queries: []\n",
            "100% 184/184 [00:00<00:00, 1365.02it/s]\n",
            "save result to /content/AIC2022-VER/data//meta/srl/color directory\n",
            "Fail queries: []\n",
            "DONE\n",
            "Extracting SRL data [vehicle] ...\n",
            "{'sedan': 0, 'coupe': 0, 'mini-cooper': 0, 'prius': 0, 'suv': 1, 'wagon': 1, 'hatchback': 1, 'mpv': 1, 'minivan': 1, 'crossover': 1, 'van': 2, 'jeep': 3, 'pickup': 4, 'semi-trailer-truck': 5, 'cement-truck': 5, 'cargo-truck': 5, 'bus': 5, 'coach': 5, 'truck': 5, 'bus-truck': 5}\n",
            "RUN TRAIN\n",
            "  1% 29/2155 [00:03<04:06,  8.63it/s]['']\n",
            "  2% 47/2155 [00:05<03:34,  9.84it/s]['']\n",
            " 14% 294/2155 [00:31<03:17,  9.42it/s]['']\n",
            " 37% 799/2155 [01:47<03:36,  6.28it/s]['']\n",
            " 37% 806/2155 [01:49<04:55,  4.57it/s]['']\n",
            " 39% 847/2155 [01:56<03:41,  5.91it/s]['']\n",
            " 40% 855/2155 [01:57<03:05,  7.02it/s]['']\n",
            " 46% 987/2155 [02:23<10:39,  1.83it/s]['']\n",
            " 57% 1219/2155 [03:04<03:21,  4.64it/s]['']\n",
            " 63% 1352/2155 [03:25<02:05,  6.40it/s]['']\n",
            " 63% 1357/2155 [03:26<01:40,  7.98it/s]['']\n",
            " 67% 1434/2155 [03:34<01:16,  9.47it/s]['']\n",
            " 74% 1589/2155 [03:51<01:01,  9.26it/s]['']\n",
            " 80% 1728/2155 [04:06<00:46,  9.22it/s]['']\n",
            " 88% 1901/2155 [04:25<00:26,  9.57it/s]['']\n",
            " 88% 1903/2155 [04:25<00:26,  9.54it/s]['']\n",
            " 88% 1904/2155 [04:25<00:26,  9.62it/s]['']\n",
            "100% 2155/2155 [04:52<00:00,  7.37it/s]\n",
            "Extract dataframe to /content/AIC2022-VER/data//meta/srl/veh/train_fraction.csv\n",
            "Fail queries: []\n",
            "RUN TEST\n",
            "100% 184/184 [00:00<00:00, 1295.38it/s]\n",
            "Extract dataframe to /content/AIC2022-VER/data//meta/srl/veh/veh_test_fraction.csv\n",
            "Fail queries: []\n",
            "100% 184/184 [00:00<00:00, 1275.58it/s]\n",
            "Extract dataframe to /content/AIC2022-VER/data//meta/srl/veh/veh_test.csv\n",
            "Fail queries: []\n",
            "DONE\n",
            "mkdir: cannot create directory ‘/content/AIC2022-VER/data//meta/srl/postproc’: File exists\n",
            "Extracting SRL data [postproc] ...\n",
            "========== Parse result in train ==========\n",
            "  0% 0/2155 [00:00<?, ?it/s]['']\n",
            "['']\n",
            "  9% 198/2155 [00:00<00:01, 989.65it/s]['']\n",
            " 37% 791/2155 [00:00<00:01, 869.03it/s]['']\n",
            "['']\n",
            "['']\n",
            "['']\n",
            " 45% 961/2155 [00:01<00:01, 760.87it/s]['']\n",
            " 56% 1203/2155 [00:01<00:01, 804.60it/s]['']\n",
            "['']\n",
            "['']\n",
            " 63% 1367/2155 [00:01<00:00, 1006.95it/s]['']\n",
            " 69% 1485/2155 [00:01<00:00, 1052.01it/s]['']\n",
            " 76% 1641/2155 [00:01<00:00, 1171.87it/s]['']\n",
            " 88% 1888/2155 [00:01<00:00, 1199.24it/s]['']\n",
            "['']\n",
            "['']\n",
            "100% 2155/2155 [00:02<00:00, 963.10it/s] \n",
            "========== Parse result in test ==========\n",
            "100% 184/184 [00:00<00:00, 1046.32it/s]\n",
            "DONE\n",
            "CPU times: user 33.1 s, sys: 4.3 s, total: 37.4 s\n",
            "Wall time: 40min 14s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gpLncp3No_uD",
        "outputId": "def47384-a84f-40c0-93cd-bc18f1736a14"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/AIC2022-VER\n",
            "mkdir: cannot create directory ‘/content/AIC2022-VER/data///meta/relation’: File exists\n",
            "Extracting auxiliary tracks ...\n",
            "  0% 0/36 [00:00<?, ?it/s]Number of auxiliary tracks: 77\n",
            "  3% 1/36 [00:32<19:02, 32.63s/it]Number of auxiliary tracks: 75\n",
            "  6% 2/36 [03:17<1:02:25, 110.17s/it]Number of auxiliary tracks: 82\n",
            "  8% 3/36 [05:54<1:12:24, 131.64s/it]Number of auxiliary tracks: 81\n",
            " 11% 4/36 [08:48<1:19:14, 148.58s/it]Number of auxiliary tracks: 94\n",
            " 14% 5/36 [15:11<2:00:25, 233.07s/it]Number of auxiliary tracks: 14\n",
            " 17% 6/36 [15:39<1:21:33, 163.13s/it]Number of auxiliary tracks: 17\n",
            " 19% 7/36 [15:48<54:31, 112.80s/it]  Number of auxiliary tracks: 17\n",
            " 22% 8/36 [20:44<1:19:53, 171.18s/it]Number of auxiliary tracks: 17\n",
            " 25% 9/36 [20:49<53:43, 119.37s/it]  Number of auxiliary tracks: 14\n",
            " 28% 10/36 [22:10<46:29, 107.29s/it]Number of auxiliary tracks: 1\n",
            "Number of auxiliary tracks: 7\n",
            " 33% 12/36 [22:17<23:29, 58.75s/it] Number of auxiliary tracks: 6\n",
            " 36% 13/36 [22:19<17:06, 44.61s/it]Number of auxiliary tracks: 7\n",
            " 39% 14/36 [22:21<12:11, 33.24s/it]Number of auxiliary tracks: 8\n",
            " 42% 15/36 [22:31<09:26, 26.96s/it]Number of auxiliary tracks: 7\n",
            " 44% 16/36 [22:34<06:41, 20.08s/it]Number of auxiliary tracks: 6\n",
            " 47% 17/36 [22:37<04:50, 15.28s/it]Number of auxiliary tracks: 7\n",
            " 50% 18/36 [22:42<03:40, 12.26s/it]Number of auxiliary tracks: 9\n",
            " 53% 19/36 [22:46<02:46,  9.79s/it]Number of auxiliary tracks: 9\n",
            "Number of auxiliary tracks: 11\n",
            " 58% 21/36 [22:49<01:32,  6.15s/it]Number of auxiliary tracks: 10\n",
            " 61% 22/36 [23:19<02:46, 11.86s/it]Number of auxiliary tracks: 7\n",
            " 64% 23/36 [23:33<02:40, 12.36s/it]Number of auxiliary tracks: 3\n",
            " 67% 24/36 [23:33<01:50,  9.24s/it]Number of auxiliary tracks: 9\n",
            " 69% 25/36 [23:42<01:40,  9.13s/it]Number of auxiliary tracks: 22\n",
            " 72% 26/36 [23:53<01:37,  9.72s/it]Number of auxiliary tracks: 22\n",
            " 75% 27/36 [23:54<01:02,  6.98s/it]Number of auxiliary tracks: 22\n",
            " 78% 28/36 [24:02<00:59,  7.45s/it]Number of auxiliary tracks: 26\n",
            " 81% 29/36 [24:37<01:48, 15.52s/it]Number of auxiliary tracks: 23\n",
            " 83% 30/36 [25:12<02:07, 21.19s/it]Number of auxiliary tracks: 14\n",
            " 86% 31/36 [25:16<01:20, 16.01s/it]Number of auxiliary tracks: 21\n",
            " 89% 32/36 [25:38<01:11, 17.83s/it]Number of auxiliary tracks: 16\n",
            " 92% 33/36 [25:46<00:45, 15.12s/it]Number of auxiliary tracks: 19\n",
            " 94% 34/36 [25:52<00:24, 12.21s/it]Number of auxiliary tracks: 15\n",
            " 97% 35/36 [25:55<00:09,  9.42s/it]Number of auxiliary tracks: 13\n",
            "100% 36/36 [26:09<00:00, 43.60s/it]\n",
            "  0% 0/23 [00:00<?, ?it/s]Number of auxiliary tracks: 124\n",
            "  4% 1/23 [00:38<14:17, 38.98s/it]Number of auxiliary tracks: 90\n",
            "  9% 2/23 [02:23<27:10, 77.64s/it]Number of auxiliary tracks: 98\n",
            " 13% 3/23 [03:57<28:21, 85.08s/it]"
          ]
        }
      ],
      "source": [
        "%cd $WORKDIR\n",
        "!./tools/preproc_relation.sh $DATADIR/ $DATADIR/meta/test_tracks.json\n",
        "!./tools/preproc_relation.sh $DATADIR/ $DATADIR/meta/train_tracks.json"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Compress and save"
      ],
      "metadata": {
        "id": "00wOuYJevBe5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%cd $DATADIR/meta\n",
        "!zip -r meta_base.zip extracted_frames/ *.json"
      ],
      "metadata": {
        "id": "x3KDvPM8LWXN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r meta_motion_part.zip bk_map/ motion_map/"
      ],
      "metadata": {
        "id": "53tsuBAlRKN7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r meta_relation_part.zip relation/"
      ],
      "metadata": {
        "id": "FV0GUQANRYuM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r meta_srl_part.zip bk_map/ srl/"
      ],
      "metadata": {
        "id": "xIzQ8XxgRROY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls *.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Em_0n0nYZw2-",
        "outputId": "5cc178fa-402a-478c-b67d-c8954e13eea1"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "meta_base.zip  meta_motion_part.zip  meta_relation_part.zip  meta_srl_part.zip\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!ls -lh *.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HxRJQ2vZQ6j-",
        "outputId": "a5b1ec42-ff20-44e9-8b87-31111926ed82"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-rw-r--r-- 1 root root  13G Apr 19 00:36 meta_base.zip\n",
            "-rw-r--r-- 1 root root 1.1G Apr 19 00:43 meta_motion_part.zip\n",
            "-rw-r--r-- 1 root root 2.8M Apr 19 01:16 meta_relation_part.zip\n",
            "-rw-r--r-- 1 root root 1.7G Apr 19 01:19 meta_srl_part.zip\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "oUEeZwdFgNHr"
      ],
      "name": "[AIC-VER] Extract data.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}