chapter2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/UdayLab/Hands-on-Pattern-Mining/blob/main/chapter2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Chapter 2: Handling Big Data - Classification, Storage, and Processing Techniques"
      ],
      "metadata": {
        "id": "1GHrHEuK6lBk"
      },
      "id": "1GHrHEuK6lBk"
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install the PAMI package"
      ],
      "metadata": {
        "id": "fx5Hz6eW7Rke"
      },
      "id": "fx5Hz6eW7Rke"
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install PAMI"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CnQTyyz67VyB",
        "outputId": "0ce13f4d-83e4-46d5-b847-4318cee75dbf"
      },
      "id": "CnQTyyz67VyB",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting PAMI\n",
            "  Downloading pami-2024.11.15.1-py3-none-any.whl.metadata (80 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/80.3 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m80.3/80.3 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from PAMI) (5.9.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from PAMI) (2.2.2)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.10/dist-packages (from PAMI) (5.24.1)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from PAMI) (3.8.0)\n",
            "Collecting resource (from PAMI)\n",
            "  Downloading Resource-0.2.1-py2.py3-none-any.whl.metadata (478 bytes)\n",
            "Collecting validators (from PAMI)\n",
            "  Downloading validators-0.34.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.10/dist-packages (from PAMI) (2.2.3)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from PAMI) (11.0.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from PAMI) (1.26.4)\n",
            "Requirement already satisfied: sphinx in /usr/local/lib/python3.10/dist-packages (from PAMI) (8.1.3)\n",
            "Collecting sphinx-rtd-theme (from PAMI)\n",
            "  Downloading sphinx_rtd_theme-3.0.2-py2.py3-none-any.whl.metadata (4.4 kB)\n",
            "Collecting discord.py (from PAMI)\n",
            "  Downloading discord.py-2.4.0-py3-none-any.whl.metadata (6.9 kB)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from PAMI) (3.4.2)\n",
            "Requirement already satisfied: deprecated in /usr/local/lib/python3.10/dist-packages (from PAMI) (1.2.15)\n",
            "Collecting fastparquet (from PAMI)\n",
            "  Downloading fastparquet-2024.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.2 kB)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated->PAMI) (1.16.0)\n",
            "Requirement already satisfied: aiohttp<4,>=3.7.4 in /usr/local/lib/python3.10/dist-packages (from discord.py->PAMI) (3.11.2)\n",
            "Collecting cramjam>=2.3 (from fastparquet->PAMI)\n",
            "  Downloading cramjam-2.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from fastparquet->PAMI) (2024.10.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from fastparquet->PAMI) (24.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->PAMI) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->PAMI) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->PAMI) (2024.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->PAMI) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->PAMI) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->PAMI) (4.55.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->PAMI) (1.4.7)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->PAMI) (3.2.0)\n",
            "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from plotly->PAMI) (9.0.0)\n",
            "Collecting JsonForm>=0.0.2 (from resource->PAMI)\n",
            "  Downloading JsonForm-0.0.2.tar.gz (2.4 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting JsonSir>=0.0.2 (from resource->PAMI)\n",
            "  Downloading JsonSir-0.0.2.tar.gz (2.2 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-easyconfig>=0.1.0 (from resource->PAMI)\n",
            "  Downloading Python_EasyConfig-0.1.7-py2.py3-none-any.whl.metadata (462 bytes)\n",
            "Requirement already satisfied: sphinxcontrib-applehelp>=1.0.7 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.0.0)\n",
            "Requirement already satisfied: sphinxcontrib-devhelp>=1.0.6 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.0.0)\n",
            "Requirement already satisfied: sphinxcontrib-htmlhelp>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.1.0)\n",
            "Requirement already satisfied: sphinxcontrib-jsmath>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (1.0.1)\n",
            "Requirement already satisfied: sphinxcontrib-qthelp>=1.0.6 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.0.0)\n",
            "Requirement already satisfied: sphinxcontrib-serializinghtml>=1.1.9 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.0.0)\n",
            "Requirement already satisfied: Jinja2>=3.1 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (3.1.4)\n",
            "Requirement already satisfied: Pygments>=2.17 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.18.0)\n",
            "Requirement already satisfied: docutils<0.22,>=0.20 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (0.21.2)\n",
            "Requirement already satisfied: snowballstemmer>=2.2 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.2.0)\n",
            "Requirement already satisfied: babel>=2.13 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.16.0)\n",
            "Requirement already satisfied: alabaster>=0.7.14 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (1.0.0)\n",
            "Requirement already satisfied: imagesize>=1.3 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (1.4.1)\n",
            "Requirement already satisfied: requests>=2.30.0 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.32.3)\n",
            "Requirement already satisfied: tomli>=2 in /usr/local/lib/python3.10/dist-packages (from sphinx->PAMI) (2.1.0)\n",
            "Collecting sphinxcontrib-jquery<5,>=4 (from sphinx-rtd-theme->PAMI)\n",
            "  Downloading sphinxcontrib_jquery-4.1-py2.py3-none-any.whl.metadata (2.6 kB)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4,>=3.7.4->discord.py->PAMI) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4,>=3.7.4->discord.py->PAMI) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4,>=3.7.4->discord.py->PAMI) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4,>=3.7.4->discord.py->PAMI) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4,>=3.7.4->discord.py->PAMI) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4,>=3.7.4->discord.py->PAMI) (0.2.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4,>=3.7.4->discord.py->PAMI) (1.17.2)\n",
            "Requirement already satisfied: async-timeout<6.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4,>=3.7.4->discord.py->PAMI) (4.0.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from Jinja2>=3.1->sphinx->PAMI) (3.0.2)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from JsonForm>=0.0.2->resource->PAMI) (4.23.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->PAMI) (1.16.0)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from python-easyconfig>=0.1.0->resource->PAMI) (6.0.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.30.0->sphinx->PAMI) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.30.0->sphinx->PAMI) (3.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.30.0->sphinx->PAMI) (2024.8.30)\n",
            "Requirement already satisfied: typing-extensions>=4.1.0 in /usr/local/lib/python3.10/dist-packages (from multidict<7.0,>=4.5->aiohttp<4,>=3.7.4->discord.py->PAMI) (4.12.2)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->JsonForm>=0.0.2->resource->PAMI) (2024.10.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->JsonForm>=0.0.2->resource->PAMI) (0.35.1)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->JsonForm>=0.0.2->resource->PAMI) (0.21.0)\n",
            "Downloading pami-2024.11.15.1-py3-none-any.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m20.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading discord.py-2.4.0-py3-none-any.whl (1.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m37.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fastparquet-2024.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m42.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading Resource-0.2.1-py2.py3-none-any.whl (25 kB)\n",
            "Downloading sphinx_rtd_theme-3.0.2-py2.py3-none-any.whl (7.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.7/7.7 MB\u001b[0m \u001b[31m66.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading validators-0.34.0-py3-none-any.whl (43 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.5/43.5 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading cramjam-2.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m62.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading Python_EasyConfig-0.1.7-py2.py3-none-any.whl (5.4 kB)\n",
            "Downloading sphinxcontrib_jquery-4.1-py2.py3-none-any.whl (121 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.1/121.1 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: JsonForm, JsonSir\n",
            "  Building wheel for JsonForm (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for JsonForm: filename=JsonForm-0.0.2-py3-none-any.whl size=3311 sha256=ff63c0a64237cf266c7fada5f2fee981933a3b19df10849489e3049d84989e80\n",
            "  Stored in directory: /root/.cache/pip/wheels/b6/e5/87/11026246d3bd4ad67c0615682d2d6748bbd9a40ac0490882bd\n",
            "  Building wheel for JsonSir (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for JsonSir: filename=JsonSir-0.0.2-py3-none-any.whl size=4753 sha256=91c7dc98913de83d7d2db4ae7da5c77275d4dea9a5f186a7199826c45e856322\n",
            "  Stored in directory: /root/.cache/pip/wheels/1d/4c/d3/4d9757425983b43eb709be1043d82cd03fb863ce5f56f117e6\n",
            "Successfully built JsonForm JsonSir\n",
            "Installing collected packages: JsonSir, validators, python-easyconfig, cramjam, sphinxcontrib-jquery, fastparquet, sphinx-rtd-theme, JsonForm, discord.py, resource, PAMI\n",
            "Successfully installed JsonForm-0.0.2 JsonSir-0.0.2 PAMI-2024.11.15.1 cramjam-2.9.0 discord.py-2.4.0 fastparquet-2024.11.0 python-easyconfig-0.1.7 resource-0.2.1 sphinx-rtd-theme-3.0.2 sphinxcontrib-jquery-4.1 validators-0.34.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Downloading a sample file"
      ],
      "metadata": {
        "id": "Q8htUugz7YcK"
      },
      "id": "Q8htUugz7YcK"
    },
    {
      "cell_type": "code",
      "source": [
        "!wget -nc https://web-ext.u-aizu.ac.jp/~udayrage/datasets/transactionalDatabases/Transactional_T10I4D100K.csv"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tCi3Di8p7gQY",
        "outputId": "d23d99bb-20ef-4212-c5d0-ceb476d85f04"
      },
      "id": "tCi3Di8p7gQY",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-12-03 05:11:36--  https://web-ext.u-aizu.ac.jp/~udayrage/datasets/transactionalDatabases/Transactional_T10I4D100K.csv\n",
            "Resolving web-ext.u-aizu.ac.jp (web-ext.u-aizu.ac.jp)... 163.143.103.34\n",
            "Connecting to web-ext.u-aizu.ac.jp (web-ext.u-aizu.ac.jp)|163.143.103.34|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4019277 (3.8M) [text/csv]\n",
            "Saving to: ‘Transactional_T10I4D100K.csv’\n",
            "\n",
            "Transactional_T10I4 100%[===================>]   3.83M  3.18MB/s    in 1.2s    \n",
            "\n",
            "2024-12-03 05:11:38 (3.18 MB/s) - ‘Transactional_T10I4D100K.csv’ saved [4019277/4019277]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Converting a CSV file into a Paraquet file"
      ],
      "metadata": {
        "id": "1PFXFFlT6ucs"
      },
      "id": "1PFXFFlT6ucs"
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Generic\n",
        "\n",
        "\n",
        "\n",
        "```python\n",
        "from PAMI.extras.convert import CSV2Parquet as alg\n",
        "\n",
        "obj = alg.CSV2Parquet(inputFile,outputFile,sep)\n",
        "obj.convert()\n",
        "\n",
        "print('Runtime: ' + str(obj.getRuntime()))\n",
        "print('Memory (RSS): ' + str(obj.getMemoryRSS()))\n",
        "print('Memory (USS): ' + str(obj.getMemoryUSS()))\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "nOE0pBDh7J-F"
      },
      "id": "nOE0pBDh7J-F"
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Example 1: CSV2Paraquet"
      ],
      "metadata": {
        "id": "2Xp10l1q7nP_"
      },
      "id": "2Xp10l1q7nP_"
    },
    {
      "cell_type": "code",
      "source": [
        "import PAMI.extras.convert.CSV2Parquet as cp\n",
        "\n",
        "obj = cp.CSV2Parquet(inputFile='Transactional_T10I4D100K.csv',\\\n",
        "      outputFile='Transactional_T10I4D100K.parquet',sep='\\t')\n",
        "obj.convert()\n",
        "\n",
        "print('Runtime: ' + str(obj.getRuntime()))\n",
        "print('Memory (RSS): ' + str(obj.getMemoryRSS()))\n",
        "print('Memory (USS): ' + str(obj.getMemoryUSS()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pcYsmPmS7Jb8",
        "outputId": "bfe40ad5-a378-49b2-8648-efbf2e1270f9"
      },
      "id": "pcYsmPmS7Jb8",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runtime: 1.2596509456634521\n",
            "Memory (RSS): 316493824\n",
            "Memory (USS): 293793792\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Converting a Paraquet file into a CSV file"
      ],
      "metadata": {
        "id": "l2rgi_u-7vgb"
      },
      "id": "l2rgi_u-7vgb"
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Generic\n",
        "\n",
        "\n",
        "```python\n",
        "from PAMI.extras.convert import Parquet2CSV as alg\n",
        "\n",
        "obj = alg.Parquet2CSV(inputFile,outputFile,sep)\n",
        "obj.convert()\n",
        "\n",
        "print('Runtime: ' + str(obj.getRuntime()))\n",
        "print('Memory (RSS): ' + str(obj.getMemoryRSS()))\n",
        "print('Memory (USS): ' + str(obj.getMemoryUSS()))\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "42PpT09771IK"
      },
      "id": "42PpT09771IK"
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Example 2: Paraquet2CSV"
      ],
      "metadata": {
        "id": "UbOyc6I77-xQ"
      },
      "id": "UbOyc6I77-xQ"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "initial_id",
      "metadata": {
        "collapsed": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "initial_id",
        "outputId": "be26b81d-4ffe-4b14-c560-7f59b4a857f3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runtime: 3.2891857624053955\n",
            "Memory (RSS): 216768512\n",
            "Memory (USS): 195346432\n"
          ]
        }
      ],
      "source": [
        "import PAMI.extras.convert.Parquet2CSV as cp\n",
        "\n",
        "obj = cp.Parquet2CSV(inputFile='Transactional_T10I4D100K.parquet',\\\n",
        "      outputFile='new_Tran_T10I4D100K.csv',sep='\\t')\n",
        "obj.convert()\n",
        "\n",
        "print('Runtime: ' + str(obj.getRuntime()))\n",
        "print('Memory (RSS): ' + str(obj.getMemoryRSS()))\n",
        "print('Memory (USS): ' + str(obj.getMemoryUSS()))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Converting a Dataframe into a Particular Database Type"
      ],
      "metadata": {
        "id": "Ebjt91yQ8Ork"
      },
      "id": "Ebjt91yQ8Ork"
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Generic\n",
        "\n",
        "```python\n",
        "from PAMI.extras.convert import DF2DB as alg\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "obj = alg.DF2DB(dataFrame)\n",
        "obj.convert2ParticularDatabase(outputFileName, other parameters)\n",
        "\n",
        "print('Runtime: ' + str(obj.getRuntime()))\n",
        "print('Memory (RSS): ' + str(obj.getMemoryRSS()))\n",
        "print('Memory (USS): ' + str(obj.getMemoryUSS()))\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "phvlOkxS8bxY"
      },
      "id": "phvlOkxS8bxY"
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Example 3: Dataframe to transactional database"
      ],
      "metadata": {
        "id": "Ql_OVBuQ8sIm"
      },
      "id": "Ql_OVBuQ8sIm"
    },
    {
      "cell_type": "code",
      "source": [
        "from PAMI.extras.convert import DF2DB as alg\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "data = np.random.randint(1, 100, size=(1000, 4))\n",
        "dataFrame = pd.DataFrame(data,\n",
        "             columns=['Item1', 'Item2', 'Item3', 'Item4']\n",
        "            )\n",
        "\n",
        "obj = alg.DF2DB(dataFrame)\n",
        "obj.convert2TransactionalDatabase(oFile='transactionalDB.csv',\n",
        "       condition='>=', thresholdValue=36\n",
        "     )\n",
        "print('Runtime: ' + str(obj.getRuntime()))\n",
        "print('Memory (RSS): ' + str(obj.getMemoryRSS()))\n",
        "print('Memory (USS): ' + str(obj.getMemoryUSS()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "daMtu0F48epP",
        "outputId": "23406cb1-27af-45cd-f062-f92c17792217"
      },
      "id": "daMtu0F48epP",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runtime: 0.04881429672241211\n",
            "Memory (RSS): 216625152\n",
            "Memory (USS): 195784704\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!head transactionalDB.csv #printing the created transactional database"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nJWWRCEd8V-7",
        "outputId": "25519703-0c3a-4fc9-8e91-b0235cd3fa53"
      },
      "id": "nJWWRCEd8V-7",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Item2\tItem4\n",
            "Item1\tItem2\n",
            "Item1\tItem3\n",
            "Item1\tItem2\n",
            "Item1\tItem2\tItem3\tItem4\n",
            "Item1\tItem2\tItem3\n",
            "Item3\tItem4\n",
            "Item3\tItem4\n",
            "Item1\tItem2\tItem3\tItem4\n",
            "Item2\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 2
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython2",
      "version": "2.7.6"
    },
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}