PCA_on_Coh_Metrix.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyN2iWMJ56lUSuQCMgcMDv++",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/IvyWang845/PCA-on-Coh-Metrix/blob/main/PCA_on_Coh_Metrix.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YLfMgWReAdhK"
      },
      "outputs": [],
      "source": [
        "# install pydrive to load data\n",
        "!pip install -U -q PyDrive\n",
        "\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# the same way we get id from last class\n",
        "#https://drive.google.com/file/d/1VVpKto8PDxN93o_IZVyaCpknxk5FGLeY/view?usp=sharing\n",
        "id = \"1VVpKto8PDxN93o_IZVyaCpknxk5FGLeY\"\n",
        "file = drive.CreateFile({'id':id})\n",
        "file.GetContentFile('Cause_Effect_EDA_data.csv')"
      ],
      "metadata": {
        "id": "LXYP4KO2AiPj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv('Cause_Effect_EDA_data.csv')\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 698
        },
        "id": "oe8_aYJbAkOf",
        "outputId": "623d74fa-5776-4bd5-867c-b94c3d354d00"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Semester  Research ID  Student ID  NGCI  \\\n",
              "0  Spring22            1        6594     2   \n",
              "1  Spring22            2        6463     2   \n",
              "2  Spring22            3        2489     2   \n",
              "3  Spring22            4        1718     2   \n",
              "4  Spring22            5        9064     2   \n",
              "\n",
              "                                             Reponse  DESWC_3  DESPL  DESPLd  \\\n",
              "0  The runoff of the wastewater used to irrigate ...    152.0    3.0     0.0   \n",
              "1  I think that since corn is being grown near by...    200.0    2.0     0.0   \n",
              "2  the corn that has been growing with the treate...    117.0    2.0     0.0   \n",
              "3  Fertilizer runoff or just poorly treated waste...    113.0    4.0     0.0   \n",
              "4  I think that when the natural plants were remo...    137.0    1.0     0.0   \n",
              "\n",
              "     DESSL   DESSLd_7  ...  WRDCNCc_97     WRDIMGc     WRDMEAc  WRDPOLc  \\\n",
              "0   50.333  36.116001  ...  405.877014  448.796997  438.750000    4.082   \n",
              "1   98.000  50.911999  ...  381.651001  406.821014  417.691986    3.887   \n",
              "2   58.000  35.355000  ...  442.282013  451.049011  475.463989    4.193   \n",
              "3   27.250   5.560000  ...  362.399994  381.919006  422.000000    3.817   \n",
              "4  133.000   0.000000  ...  421.898010  438.962006  430.341003    3.712   \n",
              "\n",
              "   WRDHYPn  WRDHYPv  WRDHYPnv      RDFRE     RDFKGL       RDL2  \n",
              "0    6.374    1.374     1.869  30.200001  21.634001   9.401000  \n",
              "1    5.646    1.460     1.677   0.000000  41.818001  17.868999  \n",
              "2    5.908    1.701     2.003  17.343000  25.372999   6.205000  \n",
              "3    5.833    1.630     1.714  47.116001  13.706000  11.451000  \n",
              "4    6.358    1.431     1.895   0.000000  55.327999   4.586000  \n",
              "\n",
              "[5 rows x 111 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a02d8896-f09e-4eca-aa92-f94a8db27671\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Semester</th>\n",
              "      <th>Research ID</th>\n",
              "      <th>Student ID</th>\n",
              "      <th>NGCI</th>\n",
              "      <th>Reponse</th>\n",
              "      <th>DESWC_3</th>\n",
              "      <th>DESPL</th>\n",
              "      <th>DESPLd</th>\n",
              "      <th>DESSL</th>\n",
              "      <th>DESSLd_7</th>\n",
              "      <th>...</th>\n",
              "      <th>WRDCNCc_97</th>\n",
              "      <th>WRDIMGc</th>\n",
              "      <th>WRDMEAc</th>\n",
              "      <th>WRDPOLc</th>\n",
              "      <th>WRDHYPn</th>\n",
              "      <th>WRDHYPv</th>\n",
              "      <th>WRDHYPnv</th>\n",
              "      <th>RDFRE</th>\n",
              "      <th>RDFKGL</th>\n",
              "      <th>RDL2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>1</td>\n",
              "      <td>6594</td>\n",
              "      <td>2</td>\n",
              "      <td>The runoff of the wastewater used to irrigate ...</td>\n",
              "      <td>152.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>50.333</td>\n",
              "      <td>36.116001</td>\n",
              "      <td>...</td>\n",
              "      <td>405.877014</td>\n",
              "      <td>448.796997</td>\n",
              "      <td>438.750000</td>\n",
              "      <td>4.082</td>\n",
              "      <td>6.374</td>\n",
              "      <td>1.374</td>\n",
              "      <td>1.869</td>\n",
              "      <td>30.200001</td>\n",
              "      <td>21.634001</td>\n",
              "      <td>9.401000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>2</td>\n",
              "      <td>6463</td>\n",
              "      <td>2</td>\n",
              "      <td>I think that since corn is being grown near by...</td>\n",
              "      <td>200.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>98.000</td>\n",
              "      <td>50.911999</td>\n",
              "      <td>...</td>\n",
              "      <td>381.651001</td>\n",
              "      <td>406.821014</td>\n",
              "      <td>417.691986</td>\n",
              "      <td>3.887</td>\n",
              "      <td>5.646</td>\n",
              "      <td>1.460</td>\n",
              "      <td>1.677</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>41.818001</td>\n",
              "      <td>17.868999</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>3</td>\n",
              "      <td>2489</td>\n",
              "      <td>2</td>\n",
              "      <td>the corn that has been growing with the treate...</td>\n",
              "      <td>117.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>58.000</td>\n",
              "      <td>35.355000</td>\n",
              "      <td>...</td>\n",
              "      <td>442.282013</td>\n",
              "      <td>451.049011</td>\n",
              "      <td>475.463989</td>\n",
              "      <td>4.193</td>\n",
              "      <td>5.908</td>\n",
              "      <td>1.701</td>\n",
              "      <td>2.003</td>\n",
              "      <td>17.343000</td>\n",
              "      <td>25.372999</td>\n",
              "      <td>6.205000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>4</td>\n",
              "      <td>1718</td>\n",
              "      <td>2</td>\n",
              "      <td>Fertilizer runoff or just poorly treated waste...</td>\n",
              "      <td>113.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>27.250</td>\n",
              "      <td>5.560000</td>\n",
              "      <td>...</td>\n",
              "      <td>362.399994</td>\n",
              "      <td>381.919006</td>\n",
              "      <td>422.000000</td>\n",
              "      <td>3.817</td>\n",
              "      <td>5.833</td>\n",
              "      <td>1.630</td>\n",
              "      <td>1.714</td>\n",
              "      <td>47.116001</td>\n",
              "      <td>13.706000</td>\n",
              "      <td>11.451000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>5</td>\n",
              "      <td>9064</td>\n",
              "      <td>2</td>\n",
              "      <td>I think that when the natural plants were remo...</td>\n",
              "      <td>137.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>133.000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>421.898010</td>\n",
              "      <td>438.962006</td>\n",
              "      <td>430.341003</td>\n",
              "      <td>3.712</td>\n",
              "      <td>6.358</td>\n",
              "      <td>1.431</td>\n",
              "      <td>1.895</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>55.327999</td>\n",
              "      <td>4.586000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 111 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a02d8896-f09e-4eca-aa92-f94a8db27671')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a02d8896-f09e-4eca-aa92-f94a8db27671 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a02d8896-f09e-4eca-aa92-f94a8db27671');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Feature Preprocessing**"
      ],
      "metadata": {
        "id": "BsuY__wvD5mP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#remove missing value\n",
        "df.dropna(subset=['Reponse'],inplace=True)\n",
        "df.reset_index(inplace=True, drop=True)"
      ],
      "metadata": {
        "id": "ySYVEhT5EAF8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 698
        },
        "id": "LbMYe3yaEk_n",
        "outputId": "4c9ec37e-bf6b-48d5-ef1d-304fa246e33e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Semester  Research ID  Student ID  NGCI  \\\n",
              "0  Spring22            1        6594     2   \n",
              "1  Spring22            2        6463     2   \n",
              "2  Spring22            3        2489     2   \n",
              "3  Spring22            4        1718     2   \n",
              "4  Spring22            5        9064     2   \n",
              "\n",
              "                                             Reponse  DESWC_3  DESPL  DESPLd  \\\n",
              "0  The runoff of the wastewater used to irrigate ...    152.0    3.0     0.0   \n",
              "1  I think that since corn is being grown near by...    200.0    2.0     0.0   \n",
              "2  the corn that has been growing with the treate...    117.0    2.0     0.0   \n",
              "3  Fertilizer runoff or just poorly treated waste...    113.0    4.0     0.0   \n",
              "4  I think that when the natural plants were remo...    137.0    1.0     0.0   \n",
              "\n",
              "     DESSL   DESSLd_7  ...  WRDCNCc_97     WRDIMGc     WRDMEAc  WRDPOLc  \\\n",
              "0   50.333  36.116001  ...  405.877014  448.796997  438.750000    4.082   \n",
              "1   98.000  50.911999  ...  381.651001  406.821014  417.691986    3.887   \n",
              "2   58.000  35.355000  ...  442.282013  451.049011  475.463989    4.193   \n",
              "3   27.250   5.560000  ...  362.399994  381.919006  422.000000    3.817   \n",
              "4  133.000   0.000000  ...  421.898010  438.962006  430.341003    3.712   \n",
              "\n",
              "   WRDHYPn  WRDHYPv  WRDHYPnv      RDFRE     RDFKGL       RDL2  \n",
              "0    6.374    1.374     1.869  30.200001  21.634001   9.401000  \n",
              "1    5.646    1.460     1.677   0.000000  41.818001  17.868999  \n",
              "2    5.908    1.701     2.003  17.343000  25.372999   6.205000  \n",
              "3    5.833    1.630     1.714  47.116001  13.706000  11.451000  \n",
              "4    6.358    1.431     1.895   0.000000  55.327999   4.586000  \n",
              "\n",
              "[5 rows x 111 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-920b6725-6be3-4086-a8bb-6f7d741d3f3e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Semester</th>\n",
              "      <th>Research ID</th>\n",
              "      <th>Student ID</th>\n",
              "      <th>NGCI</th>\n",
              "      <th>Reponse</th>\n",
              "      <th>DESWC_3</th>\n",
              "      <th>DESPL</th>\n",
              "      <th>DESPLd</th>\n",
              "      <th>DESSL</th>\n",
              "      <th>DESSLd_7</th>\n",
              "      <th>...</th>\n",
              "      <th>WRDCNCc_97</th>\n",
              "      <th>WRDIMGc</th>\n",
              "      <th>WRDMEAc</th>\n",
              "      <th>WRDPOLc</th>\n",
              "      <th>WRDHYPn</th>\n",
              "      <th>WRDHYPv</th>\n",
              "      <th>WRDHYPnv</th>\n",
              "      <th>RDFRE</th>\n",
              "      <th>RDFKGL</th>\n",
              "      <th>RDL2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>1</td>\n",
              "      <td>6594</td>\n",
              "      <td>2</td>\n",
              "      <td>The runoff of the wastewater used to irrigate ...</td>\n",
              "      <td>152.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>50.333</td>\n",
              "      <td>36.116001</td>\n",
              "      <td>...</td>\n",
              "      <td>405.877014</td>\n",
              "      <td>448.796997</td>\n",
              "      <td>438.750000</td>\n",
              "      <td>4.082</td>\n",
              "      <td>6.374</td>\n",
              "      <td>1.374</td>\n",
              "      <td>1.869</td>\n",
              "      <td>30.200001</td>\n",
              "      <td>21.634001</td>\n",
              "      <td>9.401000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>2</td>\n",
              "      <td>6463</td>\n",
              "      <td>2</td>\n",
              "      <td>I think that since corn is being grown near by...</td>\n",
              "      <td>200.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>98.000</td>\n",
              "      <td>50.911999</td>\n",
              "      <td>...</td>\n",
              "      <td>381.651001</td>\n",
              "      <td>406.821014</td>\n",
              "      <td>417.691986</td>\n",
              "      <td>3.887</td>\n",
              "      <td>5.646</td>\n",
              "      <td>1.460</td>\n",
              "      <td>1.677</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>41.818001</td>\n",
              "      <td>17.868999</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>3</td>\n",
              "      <td>2489</td>\n",
              "      <td>2</td>\n",
              "      <td>the corn that has been growing with the treate...</td>\n",
              "      <td>117.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>58.000</td>\n",
              "      <td>35.355000</td>\n",
              "      <td>...</td>\n",
              "      <td>442.282013</td>\n",
              "      <td>451.049011</td>\n",
              "      <td>475.463989</td>\n",
              "      <td>4.193</td>\n",
              "      <td>5.908</td>\n",
              "      <td>1.701</td>\n",
              "      <td>2.003</td>\n",
              "      <td>17.343000</td>\n",
              "      <td>25.372999</td>\n",
              "      <td>6.205000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>4</td>\n",
              "      <td>1718</td>\n",
              "      <td>2</td>\n",
              "      <td>Fertilizer runoff or just poorly treated waste...</td>\n",
              "      <td>113.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>27.250</td>\n",
              "      <td>5.560000</td>\n",
              "      <td>...</td>\n",
              "      <td>362.399994</td>\n",
              "      <td>381.919006</td>\n",
              "      <td>422.000000</td>\n",
              "      <td>3.817</td>\n",
              "      <td>5.833</td>\n",
              "      <td>1.630</td>\n",
              "      <td>1.714</td>\n",
              "      <td>47.116001</td>\n",
              "      <td>13.706000</td>\n",
              "      <td>11.451000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Spring22</td>\n",
              "      <td>5</td>\n",
              "      <td>9064</td>\n",
              "      <td>2</td>\n",
              "      <td>I think that when the natural plants were remo...</td>\n",
              "      <td>137.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>133.000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>421.898010</td>\n",
              "      <td>438.962006</td>\n",
              "      <td>430.341003</td>\n",
              "      <td>3.712</td>\n",
              "      <td>6.358</td>\n",
              "      <td>1.431</td>\n",
              "      <td>1.895</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>55.327999</td>\n",
              "      <td>4.586000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 111 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-920b6725-6be3-4086-a8bb-6f7d741d3f3e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-920b6725-6be3-4086-a8bb-6f7d741d3f3e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-920b6725-6be3-4086-a8bb-6f7d741d3f3e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# select CM features\n",
        "# DESWC_3: total word written\n",
        "# DESSLd_7: sentence length, number of words, SD\n",
        "# DESWLsy_8: Word length number of syllables, mean\n",
        "# PCVERBp_18: Text Easability PC deep cohesion\n",
        "# LSASS1_38: LSA overlap, adjacent sentences, mean\n",
        "# LDTTRc_46: Lexical diversity, TTR\n",
        "# LDMTLD_48: Lexical diversity, MTLD\n",
        "# CNCAll_50\n",
        "# CNCCaus_51\n",
        "# CNCLogic_52\n",
        "# CNCADC_53\n",
        "# CNCTempx_55\n",
        "# CNCAdd_56\n",
        "# CNCPos_57\n",
        "# CNCNeg_58\n",
        "# SMCAUSv_59\n",
        "# SMCAUSvp_60\n",
        "# SMINTEp_61\n",
        "# SMCAUSr_62\n",
        "# SMINTEr_63\n",
        "# SMCAUSlsa_64\n",
        "# SMCAUSwn_65\n",
        "# SMTEMP_66\n",
        "# SYNLE_67\n",
        "# SYNSTRUTa_73\n",
        "# WRDFRQc_92\n",
        "# WRDCNCc_97\n",
        "#====================================27 measures\n",
        "# WRDNOUN\n",
        "# WRDVERB\n",
        "# WRDADJ\n",
        "# WRDADV\n",
        "# WRDPRO\n",
        "# WRDPRP1s\n",
        "# WRDPRP3s\n",
        "# WRDFAMc\n",
        "# WRDIMGc\n",
        "# WRDMEAc\n",
        "# CRFCWO1\n",
        "# CRFCWO1d\n",
        "# CRFAO1\n",
        "# CRFAOa\n",
        "# CRFNO1\n",
        "# CRFSOa\n",
        "#====================================43 measures\n",
        "\n",
        "CM_df = df.loc[:, [\"DESWC_3\",\"DESSLd_7\", \"DESWLsy_8\", \"PCVERBp_18\", \"LSASS1_38\", \"LDTTRc_46\",\n",
        "                      \"LDMTLD_48\", \"CNCAll_50\", \"CNCCaus_51\",\"CNCLogic_52\", \"CNCADC_53\", \"CNCTempx_55\",\n",
        "                      \"CNCAdd_56\", \"CNCPos_57\", \"CNCNeg_58\", \"SMCAUSv_59\", \"SMCAUSvp_60\",\n",
        "                      \"SMINTEp_61\", \"SMCAUSr_62\", \"SMINTEr_63\", \"SMCAUSlsa_64\", \"SMCAUSwn_65\",\n",
        "                      \"SMTEMP_66\", \"SYNLE_67\", \"SYNSTRUTa_73\", \"WRDFRQc_92\", \"WRDCNCc_97\"]]\n",
        "                      #\"WRDNOUN\", \"WRDVERB\", \"WRDADJ\", \"WRDADV\", \"WRDPRO\", \"WRDPRP1s\",\n",
        "                      #\"WRDPRP3s\", 'WRDFAMc', \"WRDIMGc\", \"WRDMEAc\", \"CRFCWO1\", \"CRFCWO1d\",\n",
        "                      #\"CRFAO1\", \"CRFAOa\", \"CRFNO1\", \"CRFSOa\"\n",
        "CM_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 487
        },
        "id": "XgrJ6Ad0FEON",
        "outputId": "42c821a7-4b8c-4631-fd06-0baf86d9a2dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      DESWC_3   DESSLd_7  DESWLsy_8  PCVERBp_18  LSASS1_38  LDTTRc_46  \\\n",
              "0       152.0  36.116001      1.480   76.730003      0.237      0.733   \n",
              "1       200.0  50.911999      1.560   21.480000      0.589      0.788   \n",
              "2       117.0  35.355000      1.538   70.540001      0.589      0.857   \n",
              "3       113.0   5.560000      1.549    8.230000      0.090      0.851   \n",
              "4       137.0   0.000000      1.482  100.000000      0.000      0.846   \n",
              "...       ...        ...        ...         ...        ...        ...   \n",
              "1716     72.0  15.000000      1.361   15.870000      0.470      0.775   \n",
              "1717    136.0  15.076000      1.500   59.480000      0.305      0.778   \n",
              "1718    121.0  21.221001      1.463   36.320000      0.520      0.726   \n",
              "1719    118.0  10.035000      1.305   94.290001      0.142      0.788   \n",
              "1720     34.0   1.155000      1.559    6.940000      0.587      0.800   \n",
              "\n",
              "      LDMTLD_48   CNCAll_50  CNCCaus_51  CNCLogic_52  ...  SMINTEp_61  \\\n",
              "0     41.801998   85.526001   32.895000    26.316000  ...    0.000000   \n",
              "1     66.801002  100.000000   50.000000    50.000000  ...    0.000000   \n",
              "2     46.641998   85.470001   17.094000     8.547000  ...    0.000000   \n",
              "3     90.954002   88.496002   17.698999    79.646004  ...    8.850000   \n",
              "4     68.040001   87.591003   14.599000    14.599000  ...    0.000000   \n",
              "...         ...         ...         ...          ...  ...         ...   \n",
              "1716  36.924000   41.667000   13.889000    27.778000  ...    0.000000   \n",
              "1717  45.776001   95.587997   44.118000    36.764999  ...    7.353000   \n",
              "1718  59.000000  165.289002   90.908997    82.644997  ...    0.000000   \n",
              "1719  60.500000  110.169998   25.424000    25.424000  ...    8.475000   \n",
              "1720  42.591000   88.235001   29.412001    29.412001  ...   29.412001   \n",
              "\n",
              "      SMCAUSr_62  SMINTEr_63  SMCAUSlsa_64  SMCAUSwn_65  SMTEMP_66  SYNLE_67  \\\n",
              "0          2.000         4.0         0.153        0.353      0.750     2.000   \n",
              "1          1.333         7.0         0.103        0.385      1.000    22.500   \n",
              "2          1.000         1.0         0.071        0.446      0.500    41.000   \n",
              "3          0.250         0.5         0.058        0.468      0.500     5.000   \n",
              "4          0.500         1.0         0.107        0.339     -1.999     1.000   \n",
              "...          ...         ...           ...          ...        ...       ...   \n",
              "1716       0.000         1.0         0.184        0.320      0.750     2.333   \n",
              "1717       0.667         2.0         0.092        0.477      0.750     6.800   \n",
              "1718       3.000         5.0         0.193        0.333      0.750     5.000   \n",
              "1719       0.200         1.5         0.070        0.627      1.000     0.800   \n",
              "1720       0.000         0.5         0.175        0.255      1.000     0.333   \n",
              "\n",
              "      SYNSTRUTa_73  WRDFRQc_92  WRDCNCc_97  \n",
              "0            0.042       2.176  405.877014  \n",
              "1            0.023       2.042  381.651001  \n",
              "2            0.017       2.006  442.282013  \n",
              "3            0.045       2.156  362.399994  \n",
              "4            0.000       2.235  421.898010  \n",
              "...            ...         ...         ...  \n",
              "1716         0.077       2.042  407.522003  \n",
              "1717         0.066       2.180  426.031006  \n",
              "1718         0.052       2.242  388.160004  \n",
              "1719         0.047       2.396  396.675995  \n",
              "1720         0.155       2.130  423.545013  \n",
              "\n",
              "[1721 rows x 27 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fe80d366-d6af-43d5-87f6-1d68493f1def\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DESWC_3</th>\n",
              "      <th>DESSLd_7</th>\n",
              "      <th>DESWLsy_8</th>\n",
              "      <th>PCVERBp_18</th>\n",
              "      <th>LSASS1_38</th>\n",
              "      <th>LDTTRc_46</th>\n",
              "      <th>LDMTLD_48</th>\n",
              "      <th>CNCAll_50</th>\n",
              "      <th>CNCCaus_51</th>\n",
              "      <th>CNCLogic_52</th>\n",
              "      <th>...</th>\n",
              "      <th>SMINTEp_61</th>\n",
              "      <th>SMCAUSr_62</th>\n",
              "      <th>SMINTEr_63</th>\n",
              "      <th>SMCAUSlsa_64</th>\n",
              "      <th>SMCAUSwn_65</th>\n",
              "      <th>SMTEMP_66</th>\n",
              "      <th>SYNLE_67</th>\n",
              "      <th>SYNSTRUTa_73</th>\n",
              "      <th>WRDFRQc_92</th>\n",
              "      <th>WRDCNCc_97</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>152.0</td>\n",
              "      <td>36.116001</td>\n",
              "      <td>1.480</td>\n",
              "      <td>76.730003</td>\n",
              "      <td>0.237</td>\n",
              "      <td>0.733</td>\n",
              "      <td>41.801998</td>\n",
              "      <td>85.526001</td>\n",
              "      <td>32.895000</td>\n",
              "      <td>26.316000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.000</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.153</td>\n",
              "      <td>0.353</td>\n",
              "      <td>0.750</td>\n",
              "      <td>2.000</td>\n",
              "      <td>0.042</td>\n",
              "      <td>2.176</td>\n",
              "      <td>405.877014</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>200.0</td>\n",
              "      <td>50.911999</td>\n",
              "      <td>1.560</td>\n",
              "      <td>21.480000</td>\n",
              "      <td>0.589</td>\n",
              "      <td>0.788</td>\n",
              "      <td>66.801002</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>50.000000</td>\n",
              "      <td>50.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.333</td>\n",
              "      <td>7.0</td>\n",
              "      <td>0.103</td>\n",
              "      <td>0.385</td>\n",
              "      <td>1.000</td>\n",
              "      <td>22.500</td>\n",
              "      <td>0.023</td>\n",
              "      <td>2.042</td>\n",
              "      <td>381.651001</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>117.0</td>\n",
              "      <td>35.355000</td>\n",
              "      <td>1.538</td>\n",
              "      <td>70.540001</td>\n",
              "      <td>0.589</td>\n",
              "      <td>0.857</td>\n",
              "      <td>46.641998</td>\n",
              "      <td>85.470001</td>\n",
              "      <td>17.094000</td>\n",
              "      <td>8.547000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.071</td>\n",
              "      <td>0.446</td>\n",
              "      <td>0.500</td>\n",
              "      <td>41.000</td>\n",
              "      <td>0.017</td>\n",
              "      <td>2.006</td>\n",
              "      <td>442.282013</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>113.0</td>\n",
              "      <td>5.560000</td>\n",
              "      <td>1.549</td>\n",
              "      <td>8.230000</td>\n",
              "      <td>0.090</td>\n",
              "      <td>0.851</td>\n",
              "      <td>90.954002</td>\n",
              "      <td>88.496002</td>\n",
              "      <td>17.698999</td>\n",
              "      <td>79.646004</td>\n",
              "      <td>...</td>\n",
              "      <td>8.850000</td>\n",
              "      <td>0.250</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.058</td>\n",
              "      <td>0.468</td>\n",
              "      <td>0.500</td>\n",
              "      <td>5.000</td>\n",
              "      <td>0.045</td>\n",
              "      <td>2.156</td>\n",
              "      <td>362.399994</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>137.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.482</td>\n",
              "      <td>100.000000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.846</td>\n",
              "      <td>68.040001</td>\n",
              "      <td>87.591003</td>\n",
              "      <td>14.599000</td>\n",
              "      <td>14.599000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.500</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.107</td>\n",
              "      <td>0.339</td>\n",
              "      <td>-1.999</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>2.235</td>\n",
              "      <td>421.898010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1716</th>\n",
              "      <td>72.0</td>\n",
              "      <td>15.000000</td>\n",
              "      <td>1.361</td>\n",
              "      <td>15.870000</td>\n",
              "      <td>0.470</td>\n",
              "      <td>0.775</td>\n",
              "      <td>36.924000</td>\n",
              "      <td>41.667000</td>\n",
              "      <td>13.889000</td>\n",
              "      <td>27.778000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.184</td>\n",
              "      <td>0.320</td>\n",
              "      <td>0.750</td>\n",
              "      <td>2.333</td>\n",
              "      <td>0.077</td>\n",
              "      <td>2.042</td>\n",
              "      <td>407.522003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1717</th>\n",
              "      <td>136.0</td>\n",
              "      <td>15.076000</td>\n",
              "      <td>1.500</td>\n",
              "      <td>59.480000</td>\n",
              "      <td>0.305</td>\n",
              "      <td>0.778</td>\n",
              "      <td>45.776001</td>\n",
              "      <td>95.587997</td>\n",
              "      <td>44.118000</td>\n",
              "      <td>36.764999</td>\n",
              "      <td>...</td>\n",
              "      <td>7.353000</td>\n",
              "      <td>0.667</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.092</td>\n",
              "      <td>0.477</td>\n",
              "      <td>0.750</td>\n",
              "      <td>6.800</td>\n",
              "      <td>0.066</td>\n",
              "      <td>2.180</td>\n",
              "      <td>426.031006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1718</th>\n",
              "      <td>121.0</td>\n",
              "      <td>21.221001</td>\n",
              "      <td>1.463</td>\n",
              "      <td>36.320000</td>\n",
              "      <td>0.520</td>\n",
              "      <td>0.726</td>\n",
              "      <td>59.000000</td>\n",
              "      <td>165.289002</td>\n",
              "      <td>90.908997</td>\n",
              "      <td>82.644997</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>3.000</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.193</td>\n",
              "      <td>0.333</td>\n",
              "      <td>0.750</td>\n",
              "      <td>5.000</td>\n",
              "      <td>0.052</td>\n",
              "      <td>2.242</td>\n",
              "      <td>388.160004</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1719</th>\n",
              "      <td>118.0</td>\n",
              "      <td>10.035000</td>\n",
              "      <td>1.305</td>\n",
              "      <td>94.290001</td>\n",
              "      <td>0.142</td>\n",
              "      <td>0.788</td>\n",
              "      <td>60.500000</td>\n",
              "      <td>110.169998</td>\n",
              "      <td>25.424000</td>\n",
              "      <td>25.424000</td>\n",
              "      <td>...</td>\n",
              "      <td>8.475000</td>\n",
              "      <td>0.200</td>\n",
              "      <td>1.5</td>\n",
              "      <td>0.070</td>\n",
              "      <td>0.627</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.800</td>\n",
              "      <td>0.047</td>\n",
              "      <td>2.396</td>\n",
              "      <td>396.675995</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1720</th>\n",
              "      <td>34.0</td>\n",
              "      <td>1.155000</td>\n",
              "      <td>1.559</td>\n",
              "      <td>6.940000</td>\n",
              "      <td>0.587</td>\n",
              "      <td>0.800</td>\n",
              "      <td>42.591000</td>\n",
              "      <td>88.235001</td>\n",
              "      <td>29.412001</td>\n",
              "      <td>29.412001</td>\n",
              "      <td>...</td>\n",
              "      <td>29.412001</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.5</td>\n",
              "      <td>0.175</td>\n",
              "      <td>0.255</td>\n",
              "      <td>1.000</td>\n",
              "      <td>0.333</td>\n",
              "      <td>0.155</td>\n",
              "      <td>2.130</td>\n",
              "      <td>423.545013</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1721 rows × 27 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fe80d366-d6af-43d5-87f6-1d68493f1def')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-fe80d366-d6af-43d5-87f6-1d68493f1def button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-fe80d366-d6af-43d5-87f6-1d68493f1def');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#standardize/Normalize Data\n",
        "num_cols = CM_df.columns[(CM_df.dtypes == 'float64') | (CM_df.dtypes == 'int64')]\n",
        "num_cols"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HuQS1oqHGBnE",
        "outputId": "f497c5a9-81c5-4eb3-83c8-f2538d38642b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['DESWC_3', 'DESSLd_7', 'DESWLsy_8', 'PCVERBp_18', 'LSASS1_38',\n",
              "       'LDTTRc_46', 'LDMTLD_48', 'CNCAll_50', 'CNCCaus_51', 'CNCLogic_52',\n",
              "       'CNCADC_53', 'CNCTempx_55', 'CNCAdd_56', 'CNCPos_57', 'CNCNeg_58',\n",
              "       'SMCAUSv_59', 'SMCAUSvp_60', 'SMINTEp_61', 'SMCAUSr_62', 'SMINTEr_63',\n",
              "       'SMCAUSlsa_64', 'SMCAUSwn_65', 'SMTEMP_66', 'SYNLE_67', 'SYNSTRUTa_73',\n",
              "       'WRDFRQc_92', 'WRDCNCc_97'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "scaler = StandardScaler()\n",
        "scaler.fit(CM_df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OuqHNcwwNU-n",
        "outputId": "00644a28-801b-4f0b-ae14-a29f8044401d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "StandardScaler()"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scaled_data = scaler.transform(CM_df)"
      ],
      "metadata": {
        "id": "91lvh9W4NbOL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scaled_data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lfjuz7iCiBxc",
        "outputId": "f8c55042-9278-4d2c-b457-0c40654f4ae2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 4.46057931e-01,  1.12006431e+00, -2.89585147e-01, ...,\n",
              "        -1.86097139e-01, -7.98638147e-02, -3.18366813e-01],\n",
              "       [ 1.13841534e+00,  1.97726662e+00,  1.64883287e-01, ...,\n",
              "        -5.47944636e-01, -6.89443724e-01, -9.55943434e-01],\n",
              "       [-5.87860117e-02,  1.07597588e+00,  3.99046043e-02, ...,\n",
              "        -6.62212254e-01, -8.53211443e-01,  6.39734542e-01],\n",
              "       ...,\n",
              "       [-1.08956109e-03,  2.57126306e-01, -3.86159554e-01, ...,\n",
              "         4.34895962e-03,  2.20376827e-01, -7.84640467e-01],\n",
              "       [-4.43618990e-02, -3.90931765e-01, -1.28373620e+00, ...,\n",
              "        -9.08741280e-02,  9.20937596e-01, -5.60517874e-01],\n",
              "       [-1.25598736e+00, -9.05392266e-01,  1.59202835e-01, ...,\n",
              "         1.96594339e+00, -2.89122509e-01,  1.46616969e-01]])"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Perform PCA on Coh-Metrix**"
      ],
      "metadata": {
        "id": "sLTlGXM6C7ML"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import PCA\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline"
      ],
      "metadata": {
        "id": "8VJki3uHDjh_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pca = PCA(n_components=3)\n",
        "pca.fit(scaled_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oi5T1u7_Dwnu",
        "outputId": "ae8c76ab-5d5a-41b1-8377-146a8f3cedde"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PCA(n_components=3)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "CM_pca = pca.transform(scaled_data)"
      ],
      "metadata": {
        "id": "dX13VrykUjAm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scaled_data.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8N5QanhCO4XD",
        "outputId": "e8314df6-2e7d-4eaf-ad73-6a461e5c0076"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1721, 27)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "CM_pca.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uan2kf_VikX6",
        "outputId": "0958ca4d-9cde-4034-eaac-8f977e590605"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1721, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "CM_pca"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vkz56qxSQYhf",
        "outputId": "0de755ca-a05b-4f5c-8762-86f3f0b43497"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.25378   , -0.81946538, -0.90945604],\n",
              "       [ 2.42998894,  0.23158181, -1.32035096],\n",
              "       [-1.04398033,  0.00516287, -1.93353615],\n",
              "       ...,\n",
              "       [ 4.36710824, -1.88620211,  1.26142077],\n",
              "       [-0.39437541, -0.48948648,  0.71328235],\n",
              "       [-1.74983146,  2.98827027,  1.92251067]])"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(3):\n",
        "    print('Component', i+1)\n",
        "    for j in range(len(CM_df.columns)):\n",
        "        print(CM_df.columns[j], ':', pca.components_[i][j])\n",
        "    print()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vexWzhPsU2Gf",
        "outputId": "8cf12c63-35c9-4abc-801e-02ff63bbe6f1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Component 1\n",
            "DESWC_3 : 0.2361777988709615\n",
            "DESSLd_7 : 0.19218344504055093\n",
            "DESWLsy_8 : -0.0932187804548323\n",
            "PCVERBp_18 : -0.16067409733462687\n",
            "LSASS1_38 : 0.19807506928404767\n",
            "LDTTRc_46 : -0.1264493510024721\n",
            "LDMTLD_48 : 0.10126055192947316\n",
            "CNCAll_50 : 0.3901238969126121\n",
            "CNCCaus_51 : 0.28054890237834557\n",
            "CNCLogic_52 : 0.35818820277350005\n",
            "CNCADC_53 : 0.17416643936057363\n",
            "CNCTempx_55 : 0.18588850327471929\n",
            "CNCAdd_56 : 0.21153835179048025\n",
            "CNCPos_57 : 0.34411546887373723\n",
            "CNCNeg_58 : 0.16790189529093805\n",
            "SMCAUSv_59 : -0.051134501133765546\n",
            "SMCAUSvp_60 : 0.10038714921105425\n",
            "SMINTEp_61 : -0.0592046513566492\n",
            "SMCAUSr_62 : 0.19788190729741034\n",
            "SMINTEr_63 : 0.277826430976181\n",
            "SMCAUSlsa_64 : 0.015768375165552777\n",
            "SMCAUSwn_65 : 0.1427450358309237\n",
            "SMTEMP_66 : 0.17469385688036573\n",
            "SYNLE_67 : -0.0017584037470775532\n",
            "SYNSTRUTa_73 : 0.046785264124872314\n",
            "WRDFRQc_92 : 0.0038146180150230614\n",
            "WRDCNCc_97 : -0.07723137234464186\n",
            "\n",
            "Component 2\n",
            "DESWC_3 : 0.14283656802789868\n",
            "DESSLd_7 : 0.17903467170358778\n",
            "DESWLsy_8 : 0.19790881041647695\n",
            "PCVERBp_18 : -0.4052548852159344\n",
            "LSASS1_38 : 0.3150586961311849\n",
            "LDTTRc_46 : 0.008890434781871147\n",
            "LDMTLD_48 : 0.09826964785852212\n",
            "CNCAll_50 : -0.11691463683937832\n",
            "CNCCaus_51 : -0.18850206453817572\n",
            "CNCLogic_52 : -0.08249755615887264\n",
            "CNCADC_53 : 0.08165967016601504\n",
            "CNCTempx_55 : -0.11345718403895463\n",
            "CNCAdd_56 : 0.006625265472311086\n",
            "CNCPos_57 : -0.1486825852764025\n",
            "CNCNeg_58 : 0.0737626126936304\n",
            "SMCAUSv_59 : 0.2520824548346151\n",
            "SMCAUSvp_60 : 0.08915827071222912\n",
            "SMINTEp_61 : 0.17015527518038778\n",
            "SMCAUSr_62 : -0.2329446046182067\n",
            "SMINTEr_63 : -0.1316890614786554\n",
            "SMCAUSlsa_64 : -0.10860385608787232\n",
            "SMCAUSwn_65 : -0.012806021852740816\n",
            "SMTEMP_66 : 0.4118642684595091\n",
            "SYNLE_67 : -0.17652081174714615\n",
            "SYNSTRUTa_73 : 0.34087625846336317\n",
            "WRDFRQc_92 : -0.17556087704409182\n",
            "WRDCNCc_97 : 0.013611764411416146\n",
            "\n",
            "Component 3\n",
            "DESWC_3 : -0.2529282039706082\n",
            "DESSLd_7 : -0.19463266839011653\n",
            "DESWLsy_8 : 0.17181014653290677\n",
            "PCVERBp_18 : 0.02471375144080431\n",
            "LSASS1_38 : -0.14723111520097965\n",
            "LDTTRc_46 : 0.28390122834666004\n",
            "LDMTLD_48 : -0.009186890305978645\n",
            "CNCAll_50 : 0.12697487746597286\n",
            "CNCCaus_51 : 0.3172658954797228\n",
            "CNCLogic_52 : 0.01316780050643439\n",
            "CNCADC_53 : -0.23020591332948073\n",
            "CNCTempx_55 : 0.16547310563843917\n",
            "CNCAdd_56 : -0.06335514066175207\n",
            "CNCPos_57 : 0.21089666183783518\n",
            "CNCNeg_58 : -0.22528757970511265\n",
            "SMCAUSv_59 : 0.3842374879956903\n",
            "SMCAUSvp_60 : 0.481519846146894\n",
            "SMINTEp_61 : 0.18987000557119182\n",
            "SMCAUSr_62 : 0.02630978441994623\n",
            "SMINTEr_63 : -0.0032138471047051835\n",
            "SMCAUSlsa_64 : -0.1370581848558102\n",
            "SMCAUSwn_65 : 0.008563709419973874\n",
            "SMTEMP_66 : -0.006191119828951762\n",
            "SYNLE_67 : -0.10791777527670822\n",
            "SYNSTRUTa_73 : 0.1282490797560053\n",
            "WRDFRQc_92 : -0.0709455678167142\n",
            "WRDCNCc_97 : -0.0283228434168488\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "dj9dYMcqVA5i"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}