PHASE1(preparing&preprocessing) (2).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PHASE1(preparing&preprocessing).ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "#LIBRARIES\n",
        "from google.colab import files\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import nltk\n",
        "import re\n",
        "from gensim.utils import simple_preprocess\n",
        "!pip install camel-tools\n",
        "from camel_tools.utils.dediac import dediac_ar\n",
        "from camel_tools.utils.normalize import normalize_alef_maksura_ar\n",
        "from camel_tools.utils.normalize import normalize_alef_ar\n",
        "from camel_tools.utils.normalize import normalize_teh_marbuta_ar"
      ],
      "metadata": {
        "id": "AV9ztqF8G5rp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Jt0cZnI1_fAF"
      },
      "outputs": [],
      "source": [
        "#upload the txt file \n",
        "#files.upload()  #NOTE: comment this line after running the cell\n",
        "\n",
        "#open dataset after uploading it to colab\n",
        "txtds = []\n",
        "with open(\"/content/OSACT2022-sharedTask-dev.txt\") as f:\n",
        "    txtds = f.readlines()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "txtds[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "S5eUS9KPLoiu",
        "outputId": "9688e11c-52b0-4243-8a82-70e45d150aea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'8888\\t@USER افطرت عليك بعقاء واثنين من فروخها الجن 🔪😂\\tNOT_OFF\\tNOT_HS\\tNOT_VLG\\tNOT_VIO\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ud = [\"\\t\",\"\\r\",\"\\n\",\"\\a\"]   #undesired characters\n",
        "files.upload()  #NOTE: comment this line after running the cell\n",
        "stopwords_list = []\n",
        "ads = []\n",
        "\n",
        "#open arabic stopwords list after uploading it to colab\n",
        "with open(\"/content/arstopwords.txt\") as f:\n",
        "    stopwords_list = f.readlines()"
      ],
      "metadata": {
        "id": "SIrgjslQR5nS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#replace all undesired characters with space except EOL \\n replace it with NONE\n",
        "def replaceitem(x):\n",
        "  for i in range(len(ud)):\n",
        "    if ud[i] == \"\\n\":\n",
        "      x = x.replace(ud[i],'')\n",
        "    else:\n",
        "      x = x.replace(ud[i],' ')\n",
        "  return x\n",
        "\n",
        "\n",
        "#arrange all the tweets with their labels, all in one list\n",
        "def allInOne(x):\n",
        "  if x.find(\"NOT_OFF\") != -1:\n",
        "    idx = x.find(\"NOT_OFF\")\n",
        "    z = x[:idx-1].split()\n",
        "    artxt = ' '.join(z[1:])\n",
        "    ads.append(artxt)\n",
        "    labels = x[idx:].split()\n",
        "    for i in range(len(labels)):\n",
        "      ads.append(labels[i])\n",
        "    ads.append(';')\n",
        "  elif x.find(\"OFF\") != -1:\n",
        "    idx = x.find(\"OFF\")\n",
        "    z = x[:idx-1].split()\n",
        "    artxt = ' '.join(z[1:])\n",
        "    ads.append(artxt)\n",
        "    labels = x[idx:].split()\n",
        "    for i in range(len(labels)):\n",
        "      ads.append(labels[i])\n",
        "    ads.append(';')\n",
        "\n",
        "#reducing orthographic ambiguity by removing particular symbols from particular letters\n",
        "def ortho_normalize(text):\n",
        "    text = normalize_alef_maksura_ar(text)\n",
        "    text = normalize_alef_ar(text)\n",
        "    text = normalize_teh_marbuta_ar(text)\n",
        "    return text\n",
        "\n",
        "\n",
        "#clean tweets from any non-arabic letters after that we apply dediacritizating followed with orthographic normalization\n",
        "def cleanTweets(tweet):\n",
        "  tweet = re.findall(r\"[\\u0621-\\u064A]+\", tweet) #return a list of all the arabic words in the tweet\n",
        "  tweet = ' '.join(tweet) #concatenate items in the list to a single string\n",
        "  tweet = dediac_ar(tweet)\n",
        "  tweet = ortho_normalize(tweet)\n",
        "  return tweet\n",
        "\n",
        "\n",
        "#preprocess the tweets and convert the sentence into list of tokens\n",
        "def preprocess(text):\n",
        "    result=[]\n",
        "    for token in simple_preprocess(text) :\n",
        "        if token not in stopwords_list and len(token) > 2:\n",
        "            result.append(token)    \n",
        "    return result\n",
        "\n",
        "\n",
        "#take arranged list as an argument \n",
        "def listToCsv(liSt, ds_name):\n",
        "  #lists of columns in the dataset\n",
        "  labels_list = [\"OFF\", \"NOT_OFF\", \"HS1\", \"HS2\", \"HS3\", \"HS4\", \"HS5\", \"HS6\", \"NOT_HS\", \"VLG\", \"NOT_VLG\", \"VIO\", \"NOT_VIO\"]\n",
        "  fc = pd.DataFrame(index=np.arange(len(txtds)), columns=[\"Tweets\"])\n",
        "  rc = pd.DataFrame(0, index=np.arange(len(txtds)), columns=labels_list)\n",
        "  for i in range(len(txtds)):\n",
        "    fc[\"Tweets\"][i] = cleanTweets(liSt[i*6])\n",
        "    rc[liSt[(i*6)+1]][i] = 1\n",
        "    rc[liSt[(i*6)+2]][i] = 1\n",
        "    rc[liSt[(i*6)+3]][i] = 1\n",
        "    rc[liSt[(i*6)+4]][i] = 1\n",
        "\n",
        "  cls = [\"Tweets\"] + labels_list\n",
        "  csvds = pd.concat([fc, rc], axis=1, names=cls)\n",
        "  csvds.to_csv(ds_name +'.csv', index = False, header=True, encoding=\"utf-8-sig\")\n",
        "  files.download(ds_name +'.csv')"
      ],
      "metadata": {
        "id": "GP-_ovgISKTx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#preparing our dataset to preprocess\n",
        "stopwords_list = list(map(replaceitem, stopwords_list))\n",
        "list(map(allInOne, list(map(replaceitem, txtds))))\n",
        "listToCsv(ads)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "xj1rQZMuZJvG",
        "outputId": "33fec31a-124b-4034-ec82-080f2af89150"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_55ae89af-c636-48a7-8e4d-4d32859bdda5\", \"OSACT2022_dev set.csv\", 171146)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "d1 = pd.read_csv('/content/drive/MyDrive/datasets/1.csv')\n",
        "d2 = pd.read_csv('/content/drive/MyDrive/datasets/2.csv')\n",
        "d3 = pd.read_csv('/content/drive/MyDrive/datasets/3.csv')\n",
        "d4 = pd.read_csv('/content/drive/MyDrive/datasets/4.csv')\n",
        "d5 = pd.read_csv('/content/drive/MyDrive/datasets/5.csv')\n",
        "d6 = pd.read_csv('/content/drive/MyDrive/datasets/6.csv')\n",
        "d7 = pd.read_csv('/content/drive/MyDrive/datasets/7.csv')\n",
        "d8 = pd.read_csv('/content/drive/MyDrive/datasets/8.csv')\n",
        "osc = pd.read_csv('/content/drive/MyDrive/datasets/oscat5.csv')"
      ],
      "metadata": {
        "id": "2sOynsLSQXyf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "OFF = osc.drop(columns=['NOT_OFF', 'NOT_HS', 'HS', 'VLG', 'NOT_VLG', 'VIO', 'NOT_VIO',])\n",
        "HS = osc.drop(columns=['OFF', 'NOT_OFF', 'NOT_HS', 'VLG', 'NOT_VLG', 'VIO', 'NOT_VIO',])"
      ],
      "metadata": {
        "id": "dwSgfHLKMj0t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(len(d1)):\n",
        "  if d1['OFF'][i] == '1':\n",
        "    OFF.loc[len(OFF.index)] = [cleanTweets(d1['tweet'][i]), 1]\n",
        "  if d1['HS_NOT_HS'][i] == 0:\n",
        "    HS.loc[len(HS.index)] = [cleanTweets(d1['tweet'][i]), 1]\n",
        "for i in range(len(d2)):\n",
        "  if d2['Offensive'][i] == 'OFF':\n",
        "    OFF.loc[len(OFF.index)] = [cleanTweets(d2['Tweet'][i]), 1]\n",
        "  if d2['Hate'][i] == 'HS':\n",
        "    HS.loc[len(HS.index)] = [cleanTweets(d2['Tweet'][i]), 1]\n",
        "for i in range(len(d3)):\n",
        "  if d3['Majority_Label'][i] == 'Offensive':\n",
        "    OFF.loc[len(OFF.index)] = [cleanTweets(d3['Comment'][i]), 1]\n",
        "for i in range(len(d4)):\n",
        "  if d4['Class'][i] == 'abusive':\n",
        "    OFF.loc[len(OFF.index)] = [cleanTweets(d4['Tweet'][i]), 1]\n",
        "  if d4['Class'][i] == 'hate':\n",
        "    HS.loc[len(HS.index)] = [cleanTweets(d4['Tweet'][i]), 1]\n",
        "for i in range(len(d5)):\n",
        "  if d5['Offensive'][i] == 'OFF':\n",
        "    OFF.loc[len(OFF.index)] = [cleanTweets(d5['Tweet'][i]), 1]\n",
        "  if d5['Hate'][i] == 'HS':\n",
        "    HS.loc[len(HS.index)] = [cleanTweets(d5['Tweet'][i]), 1]\n",
        "for i in range(len(d6)):\n",
        "  if d6['Offensive'][i] == 'OFF':\n",
        "    OFF.loc[len(OFF.index)] = [cleanTweets(d6['Tweet'][i]), 1]\n",
        "  if d6['Hate'][i] == 'HS':\n",
        "    HS.loc[len(HS.index)] = [cleanTweets(d6['Tweet'][i]), 1]\n",
        "for i in range(len(d7)):\n",
        "  if d7['Class'][i] == 'abusive':\n",
        "    OFF.loc[len(OFF.index)] = [cleanTweets(d7['Tweet'][i]), 1]\n",
        "  if d7['Class'][i] == 'hate':\n",
        "    HS.loc[len(HS.index)] = [cleanTweets(d7['Tweet'][i]), 1]\n",
        "for i in range(len(d8)):\n",
        "  if d8['hate'][i] == 1:\n",
        "    HS.loc[len(HS.index)] = [cleanTweets(d8['tweet'][i]), 1]"
      ],
      "metadata": {
        "id": "0e0eENFVMqoK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vc = OFF['OFF'].value_counts()\n",
        "while(vc[0] < vc[1]):\n",
        "  \n",
        "  for i in range(len(d1)):\n",
        "    vc = OFF['OFF'].value_counts()\n",
        "    OFF.drop_duplicates(subset=['Tweets'], inplace = True)\n",
        "    if vc[0] < vc[1]:\n",
        "      if d1['OFF'][i] == '0':\n",
        "        OFF.loc[len(OFF.index)] = [cleanTweets(d1['tweet'][i]), 0]\n",
        "    else:\n",
        "      break\n",
        "  \n",
        "  for i in range(len(d2)):\n",
        "    vc = OFF['OFF'].value_counts()\n",
        "    OFF.drop_duplicates(subset=['Tweets'], inplace = True)\n",
        "    if vc[0] < vc[1]:\n",
        "      if d2['Offensive'][i] == 'NOT_OFF':\n",
        "        OFF.loc[len(OFF.index)] = [cleanTweets(d2['Tweet'][i]), 0]\n",
        "    else:\n",
        "      break\n",
        "\n",
        "  for i in range(len(d3)):\n",
        "    vc = OFF['OFF'].value_counts()\n",
        "    OFF.drop_duplicates(subset=['Tweets'], inplace = True)\n",
        "    if vc[0] < vc[1]:\n",
        "      if d3['Majority_Label'][i] == 'Non-Offensive':\n",
        "        OFF.loc[len(OFF.index)] = [cleanTweets(d3['Comment'][i]), 0]\n",
        "    else:\n",
        "      break\n",
        "\n",
        "  \n",
        "  for i in range(len(d4)):\n",
        "    vc = OFF['OFF'].value_counts()\n",
        "    OFF.drop_duplicates(subset=['Tweets'], inplace = True)\n",
        "    if vc[0] < vc[1]:\n",
        "      if d4['Class'][i] == 'normal':\n",
        "        OFF.loc[len(OFF.index)] = [cleanTweets(d4['Tweet'][i]), 0]\n",
        "    else:\n",
        "      break\n",
        "\n",
        "  for i in range(len(d5)):\n",
        "    vc = OFF['OFF'].value_counts()\n",
        "    OFF.drop_duplicates(subset=['Tweets'], inplace = True)\n",
        "    if vc[0] < vc[1]:\n",
        "      if d5['Offensive'][i] == 'NOT_OFF':\n",
        "        OFF.loc[len(OFF.index)] = [cleanTweets(d5['Tweet'][i]), 0]\n",
        "    else:\n",
        "      break\n",
        "\n",
        "  \n",
        "  for i in range(len(d6)):\n",
        "    vc = OFF['OFF'].value_counts()\n",
        "    OFF.drop_duplicates(subset=['Tweets'], inplace = True)\n",
        "    if vc[0] < vc[1]:\n",
        "      if d6['Offensive'][i] == 'NOT_OFF':\n",
        "        OFF.loc[len(OFF.index)] = [cleanTweets(d6['Tweet'][i]), 0]\n",
        "    else:\n",
        "      break\n",
        "\n",
        "  for i in range(len(d7)):\n",
        "    vc = OFF['OFF'].value_counts()\n",
        "    OFF.drop_duplicates(subset=['Tweets'], inplace = True)\n",
        "    if vc[0] < vc[1]:\n",
        "      if d7['Class'][i] == 'normal':\n",
        "        OFF.loc[len(OFF.index)] = [cleanTweets(d7['Tweet'][i]), 0]\n",
        "    else:\n",
        "      break"
      ],
      "metadata": {
        "id": "pzf6PEbb9Zba"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ss = HS['HS'].value_counts()\n",
        "while(ss[1] <= ss[0]):\n",
        "  ss = HS['HS'].value_counts()\n",
        "  drop_indices = np.random.choice(HS.index, 50, replace=False)\n",
        "  for i in drop_indices:\n",
        "    if(ss[1] <= ss[0]):\n",
        "      if HS['HS'][i] != 1:\n",
        "        HS.drop(i, inplace = True)\n",
        "    else:\n",
        "      break"
      ],
      "metadata": {
        "id": "AKu5a_OuIAG4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "OFF['OFF'].value_counts().plot(kind='bar')\n",
        "plt\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "M72kLWCAdNj8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "74d89772-84c1-4212-fd89-1ce190ede85c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQPklEQVR4nO3cf6yeZX3H8fdHKm7RxRY5a1h/rCR2GvxDZSeAcVk2G9uCi+UPJZhlnJAmZ39Uo8mSCfunGUiC/8xJMkka6VaMExs2Q+OI7KRqlmUBehCGArIe0a5tgB49BeeIOvC7P85VfazncJ4Dp09Nr/crefJc9/e67vu57uTkc9+5nvs5qSokSX14zdmegCRpdAx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOLBn6Sd6S5JGB1w+TfCzJBUmmkhxu72va+CS5LclMkkeTXDpwrIk2/nCSiTN5YpKkX5XlPKef5DzgOHA5sAuYq6pbk9wArKmqjye5CvgIcFUb9+mqujzJBcA0MA4U8BDw+1V1ckXPSJK0qOUu72wBvlNVR4AdwL5W3wdc3do7gDtr3v3A6iQXAduAqaqaa0E/BWx/1WcgSRraqmWOvxb4QmuvraqnW/sZYG1rrwOODuxzrNUWqy/qwgsvrE2bNi1zipLUt4ceeuj7VTW2UN/QoZ/kfOD9wI2n91VVJVmR/+eQZBKYBNi4cSPT09MrcVhJ6kaSI4v1LWd550rgG1X1bNt+ti3b0N5PtPpxYMPAfutbbbH6L6mqPVU1XlXjY2MLXqgkSa/QckL/Q/xiaQfgAHDqCZwJ4J6B+nXtKZ4rgOfbMtB9wNYka9qTPltbTZI0IkMt7yR5PfBe4M8HyrcC+5PsBI4A17T6vcw/uTMDvABcD1BVc0luBg61cTdV1dyrPgNJ0tCW9cjmqI2Pj5dr+pK0PEkeqqrxhfr8Ra4kdcTQl6SOGPqS1BFDX5I6stxf5GoBm274l7M9hXPK925939mewjnFv8+Vcy78bXqnL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0ZKvSTrE5yd5JvJ3kiybuSXJBkKsnh9r6mjU2S25LMJHk0yaUDx5lo4w8nmThTJyVJWtiwd/qfBr5SVW8F3g48AdwAHKyqzcDBtg1wJbC5vSaB2wGSXADsBi4HLgN2n7pQSJJGY8nQT/JG4A+BOwCq6qdV9RywA9jXhu0Drm7tHcCdNe9+YHWSi4BtwFRVzVXVSWAK2L6iZyNJelnD3OlfDMwCf5/k4SSfTfJ6YG1VPd3GPAOsbe11wNGB/Y+12mJ1SdKIDBP6q4BLgdur6p3A//KLpRwAqqqAWokJJZlMMp1kenZ2diUOKUlqhgn9Y8Cxqnqgbd/N/EXg2bZsQ3s/0fqPAxsG9l/faovVf0lV7amq8aoaHxsbW865SJKWsGToV9UzwNEkb2mlLcDjwAHg1BM4E8A9rX0AuK49xXMF8HxbBroP2JpkTfsCd2urSZJGZNWQ4z4CfD7J+cBTwPXMXzD2J9kJHAGuaWPvBa4CZoAX2liqai7JzcChNu6mqppbkbOQJA1lqNCvqkeA8QW6tiwwtoBdixxnL7B3OROUJK0cf5ErSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6MlToJ/lekm8meSTJdKtdkGQqyeH2vqbVk+S2JDNJHk1y6cBxJtr4w0kmzswpSZIWs5w7/T+uqndU1XjbvgE4WFWbgYNtG+BKYHN7TQK3w/xFAtgNXA5cBuw+daGQJI3Gq1ne2QHsa+19wNUD9Ttr3v3A6iQXAduAqaqaq6qTwBSw/VV8viRpmYYN/QL+NclDSSZbbW1VPd3azwBrW3sdcHRg32OttlhdkjQiq4Yc9wdVdTzJbwNTSb492FlVlaRWYkLtojIJsHHjxpU4pCSpGepOv6qOt/cTwJeYX5N/ti3b0N5PtOHHgQ0Du69vtcXqp3/Wnqoar6rxsbGx5Z2NJOllLRn6SV6f5LdOtYGtwLeAA8CpJ3AmgHta+wBwXXuK5wrg+bYMdB+wNcma9gXu1laTJI3IMMs7a4EvJTk1/h+r6itJDgH7k+wEjgDXtPH3AlcBM8ALwPUAVTWX5GbgUBt3U1XNrdiZSJKWtGToV9VTwNsXqP8A2LJAvYBdixxrL7B3+dOUJK0Ef5ErSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6MnToJzkvycNJvty2L07yQJKZJF9Mcn6rv65tz7T+TQPHuLHVn0yybaVPRpL08pZzp/9R4ImB7U8Cn6qqNwMngZ2tvhM42eqfauNIcglwLfA2YDvwmSTnvbrpS5KWY6jQT7IeeB/w2bYd4D3A3W3IPuDq1t7Rtmn9W9r4HcBdVfWTqvouMANcthInIUkazrB3+n8L/CXws7b9JuC5qnqxbR8D1rX2OuAoQOt/vo3/eX2BfSRJI7Bk6Cf5E+BEVT00gvmQZDLJdJLp2dnZUXykJHVjmDv9dwPvT/I94C7ml3U+DaxOsqqNWQ8cb+3jwAaA1v9G4AeD9QX2+bmq2lNV41U1PjY2tuwTkiQtbsnQr6obq2p9VW1i/ovYr1bVnwJfAz7Qhk0A97T2gbZN6/9qVVWrX9ue7rkY2Aw8uGJnIkla0qqlhyzq48BdST4BPAzc0ep3AJ9LMgPMMX+hoKoeS7IfeBx4EdhVVS+9is+XJC3TskK/qr4OfL21n2KBp2+q6sfABxfZ/xbgluVOUpK0MvxFriR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6siSoZ/kN5I8mOQ/kzyW5K9b/eIkDySZSfLFJOe3+uva9kzr3zRwrBtb/ckk287USUmSFjbMnf5PgPdU1duBdwDbk1wBfBL4VFW9GTgJ7GzjdwInW/1TbRxJLgGuBd4GbAc+k+S8lTwZSdLLWzL0a96P2uZr26uA9wB3t/o+4OrW3tG2af1bkqTV76qqn1TVd4EZ4LIVOQtJ0lCGWtNPcl6SR4ATwBTwHeC5qnqxDTkGrGvtdcBRgNb/PPCmwfoC+0iSRmCo0K+ql6rqHcB65u/O33qmJpRkMsl0kunZ2dkz9TGS1KVlPb1TVc8BXwPeBaxOsqp1rQeOt/ZxYANA638j8IPB+gL7DH7Gnqoar6rxsbGx5UxPkrSEYZ7eGUuyurV/E3gv8ATz4f+BNmwCuKe1D7RtWv9Xq6pa/dr2dM/FwGbgwZU6EUnS0lYtPYSLgH3tSZvXAPur6stJHgfuSvIJ4GHgjjb+DuBzSWaAOeaf2KGqHkuyH3gceBHYVVUvrezpSJJezpKhX1WPAu9coP4UCzx9U1U/Bj64yLFuAW5Z/jQlSSvBX+RKUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOLBn6STYk+VqSx5M8luSjrX5Bkqkkh9v7mlZPktuSzCR5NMmlA8eaaOMPJ5k4c6clSVrIMHf6LwJ/UVWXAFcAu5JcAtwAHKyqzcDBtg1wJbC5vSaB22H+IgHsBi4HLgN2n7pQSJJGY8nQr6qnq+obrf0/wBPAOmAHsK8N2wdc3do7gDtr3v3A6iQXAduAqaqaq6qTwBSwfUXPRpL0spa1pp9kE/BO4AFgbVU93bqeAda29jrg6MBux1ptsfrpnzGZZDrJ9Ozs7HKmJ0lawtChn+QNwD8BH6uqHw72VVUBtRITqqo9VTVeVeNjY2MrcUhJUjNU6Cd5LfOB//mq+udWfrYt29DeT7T6cWDDwO7rW22xuiRpRIZ5eifAHcATVfU3A10HgFNP4EwA9wzUr2tP8VwBPN+Wge4DtiZZ077A3dpqkqQRWTXEmHcDfwZ8M8kjrfZXwK3A/iQ7gSPANa3vXuAqYAZ4AbgeoKrmktwMHGrjbqqquRU5C0nSUJYM/ar6dyCLdG9ZYHwBuxY51l5g73ImKElaOf4iV5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdWTJ0E+yN8mJJN8aqF2QZCrJ4fa+ptWT5LYkM0keTXLpwD4TbfzhJBNn5nQkSS9nmDv9fwC2n1a7AThYVZuBg20b4Epgc3tNArfD/EUC2A1cDlwG7D51oZAkjc6SoV9V/wbMnVbeAexr7X3A1QP1O2ve/cDqJBcB24CpqpqrqpPAFL96IZEknWGvdE1/bVU93drPAGtbex1wdGDcsVZbrP4rkkwmmU4yPTs7+wqnJ0layKv+IreqCqgVmMup4+2pqvGqGh8bG1upw0qSeOWh/2xbtqG9n2j148CGgXHrW22xuiRphF5p6B8ATj2BMwHcM1C/rj3FcwXwfFsGug/YmmRN+wJ3a6tJkkZo1VIDknwB+CPgwiTHmH8K51Zgf5KdwBHgmjb8XuAqYAZ4AbgeoKrmktwMHGrjbqqq078cliSdYUuGflV9aJGuLQuMLWDXIsfZC+xd1uwkSSvKX+RKUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOjDz0k2xP8mSSmSQ3jPrzJalnIw39JOcBfwdcCVwCfCjJJaOcgyT1bNR3+pcBM1X1VFX9FLgL2DHiOUhSt1aN+PPWAUcHto8Blw8OSDIJTLbNHyV5ckRz68GFwPfP9iSWkk+e7RnoLPBvc2X97mIdow79JVXVHmDP2Z7HuSjJdFWNn+15SKfzb3N0Rr28cxzYMLC9vtUkSSMw6tA/BGxOcnGS84FrgQMjnoMkdWukyztV9WKSDwP3AecBe6vqsVHOoXMum+nXlX+bI5KqOttzkCSNiL/IlaSOGPqS1BFDX5I68mv3nL6kc1+StzL/a/x1rXQcOFBVT5y9WfXBO/0OJbn+bM9B/Uryceb/BUuAB9srwBf8J4xnnk/vdCjJf1fVxrM9D/UpyX8Bb6uq/zutfj7wWFVtPjsz64PLO+eoJI8u1gWsHeVcpNP8DPgd4Mhp9Ytan84gQ//ctRbYBpw8rR7gP0Y/HennPgYcTHKYX/wDxo3Am4EPn7VZdcLQP3d9GXhDVT1yekeSr49+OtK8qvpKkt9j/l+tD36Re6iqXjp7M+uDa/qS1BGf3pGkjhj6ktQRQ1+SOmLoS1JHDH1J6sj/AyRdZnEzz8nAAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "OFF.isnull().values.any()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TdF1eXgWOLAv",
        "outputId": "92c82f43-f417-442f-aef9-bb0fd0fe67ad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "OFF.dropna(inplace=True)"
      ],
      "metadata": {
        "id": "waxt9TNZh_2-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "HS = HS.sample(frac=1, random_state=1).reset_index(drop=True)\n",
        "OFF = OFF.sample(frac=1, random_state=1).reset_index(drop=True)"
      ],
      "metadata": {
        "id": "szQ327acilAo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "HS.to_csv('HS-NOT_HS.csv', index = False, header=True, encoding=\"utf-8-sig\")\n",
        "files.download('HS-NOT_HS.csv')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "SQE6aV7Hih9-",
        "outputId": "91b7b163-3775-4c17-8a2b-7f7d1b332560"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_feaeb4f4-a557-486c-81fa-4fe970633919\", \"HS-NOT_HS.csv\", 926629)"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}