Alzheimer's_text_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Alzheimer's text classification.ipynb",
      "provenance": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ae76753485d4476a9c3e45d1eafff3e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_69891a56516a4b8da857c1dfd3dfc108",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dd42dbcd604d419291df1bf22ac780ae",
              "IPY_MODEL_6ac527a0cc964eeda664c413977e9be1"
            ]
          }
        },
        "69891a56516a4b8da857c1dfd3dfc108": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd42dbcd604d419291df1bf22ac780ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_63d0bc56232d4b60b7d417dcd40988a4",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9f49646c30b9435fb97dd718c14d6e50"
          }
        },
        "6ac527a0cc964eeda664c413977e9be1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c257d181473c4bac80d6a477a226b715",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 232k/232k [00:00&lt;00:00, 2.60MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_08374b1ddc0e49c89d15cb8706dc44be"
          }
        },
        "63d0bc56232d4b60b7d417dcd40988a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9f49646c30b9435fb97dd718c14d6e50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c257d181473c4bac80d6a477a226b715": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "08374b1ddc0e49c89d15cb8706dc44be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "46aab3adf1fe4b29a67dd7c3a2510936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_730951b0665c4351a8c6ea90cc920222",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_36ff95f18e9845af98f80435c6f7ff33",
              "IPY_MODEL_3d426fed81c84e97af4a0e468c387d82"
            ]
          }
        },
        "730951b0665c4351a8c6ea90cc920222": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "36ff95f18e9845af98f80435c6f7ff33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4ed82c627db6425c937611968fd73321",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 546,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 546,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5cf51a8751d2474482d3f9d5d323f226"
          }
        },
        "3d426fed81c84e97af4a0e468c387d82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_34ec65d95a944e9cbc141004d80035fa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 546/546 [00:00&lt;00:00, 26.4kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f037181761474dab8bf06040f556bfbc"
          }
        },
        "4ed82c627db6425c937611968fd73321": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5cf51a8751d2474482d3f9d5d323f226": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "34ec65d95a944e9cbc141004d80035fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f037181761474dab8bf06040f556bfbc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c04fab4dbe546b993aa601a4382169f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b17875957d854245aec23730fd46b168",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2bd5bb3397074280aea2a0c17902a33a",
              "IPY_MODEL_2efc747c0ab1419e9a78919cb085456e"
            ]
          }
        },
        "b17875957d854245aec23730fd46b168": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2bd5bb3397074280aea2a0c17902a33a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_eb3429c52d5640918b5b649b4353c388",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 267967963,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 267967963,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4d09b5745e254f07a7f938a3fc5c474b"
          }
        },
        "2efc747c0ab1419e9a78919cb085456e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7882a14e00be48adb3dae130f675f771",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 268M/268M [00:04&lt;00:00, 58.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a014957c2ef4fbba33531b88d2e3d1d"
          }
        },
        "eb3429c52d5640918b5b649b4353c388": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4d09b5745e254f07a7f938a3fc5c474b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7882a14e00be48adb3dae130f675f771": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a014957c2ef4fbba33531b88d2e3d1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/petroslamb/nlp-notebooks/blob/master/Alzheimer's_text_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iI2kPze0gCqZ",
        "colab_type": "text"
      },
      "source": [
        "*Much of this notebook is based on work by Jay Alammar*\n",
        "\n",
        "## Introduction\n",
        "\n",
        "This is an attempt to use a toy (30 positive + 30 negative examples) dataset of text descriptions from a picture, to detect signs of dementia.\n",
        "\n",
        "The subjects have described in their own words a simple picture, from the coherence of their speech we can elicit possible signs of Alzheimers. \n",
        "\n",
        "This is called, the cookie theft description task.\n",
        "\n",
        "## Method\n",
        "\n",
        "Using modern NLP and in particular BERT based models, a complex set of features will be generated for the given text snippets.\n",
        "\n",
        "Using these generated sentence embeddings, a simple classification of the snippets will be attempted."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "b25c55ca-ca82-4bf8-f9ba-e6bdd51fe104",
        "id": "EDPfEuJ1gCUM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 671
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/13/33/ffb67897a6985a7b7d8e5e7878c3628678f553634bd3836404fef06ef19b/transformers-2.5.1-py3-none-any.whl (499kB)\n",
            "\r\u001b[K     |▋                               | 10kB 16.5MB/s eta 0:00:01\r\u001b[K     |█▎                              | 20kB 2.2MB/s eta 0:00:01\r\u001b[K     |██                              | 30kB 3.2MB/s eta 0:00:01\r\u001b[K     |██▋                             | 40kB 2.1MB/s eta 0:00:01\r\u001b[K     |███▎                            | 51kB 2.6MB/s eta 0:00:01\r\u001b[K     |████                            | 61kB 3.1MB/s eta 0:00:01\r\u001b[K     |████▋                           | 71kB 3.6MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 81kB 4.0MB/s eta 0:00:01\r\u001b[K     |██████                          | 92kB 4.5MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 102kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████▏                        | 112kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 122kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 133kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 143kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 153kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 163kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 174kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 184kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 194kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 204kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 215kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 225kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 235kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 245kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 256kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 266kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 276kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 286kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 296kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 307kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 317kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 327kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 337kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 348kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 358kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 368kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 378kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 389kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 399kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 409kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 419kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 430kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 440kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 450kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 460kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▏ | 471kB 3.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 481kB 3.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 491kB 3.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 501kB 3.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.11.15)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 62.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
            "Collecting tokenizers==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
            "\u001b[K     |████████████████████████████████| 3.7MB 40.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.5)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 42.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.3)\n",
            "Requirement already satisfied: botocore<1.15.0,>=1.14.15 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.14.15)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers) (2.6.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=6936e30b5d75a87419adf243f7b0c589a0b227a2ccb1663165a5f360fa092c61\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
            "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.5.2 transformers-2.5.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "076b351a-8944-4328-b089-fbeb3e07f974",
        "id": "9rwFULfCgCUE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        }
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import cross_val_score\n",
        "import torch\n",
        "import transformers as ppb\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "0smP9eXMgCUC"
      },
      "source": [
        "## Importing the dataset\n",
        "We'll use pandas to read the dataset and load it into a dataframe."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pd-wMpXVmFcu",
        "colab_type": "code",
        "outputId": "0371e825-15c2-4bb0-ff54-bc1b9443aa63",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Make data directory if it doesn't exist\n",
        "!mkdir -p data\n",
        "!unzip -n -d data ./AssignmentData.zip\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  ./AssignmentData.zip\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zJ9HwIt6b5O3",
        "colab_type": "code",
        "outputId": "9d758cd9-ea28-43fe-8749-5ebafbf898d2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "!ls ./data/AssignmentData/Assigment/AD/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "01.txt\t04.txt\t07.txt\t10.txt\t13.txt\t16.txt\t19.txt\t22.txt\t25.txt\t28.txt\n",
            "02.txt\t05.txt\t08.txt\t11.txt\t14.txt\t17.txt\t20.txt\t23.txt\t26.txt\t29.txt\n",
            "03.txt\t06.txt\t09.txt\t12.txt\t15.txt\t18.txt\t21.txt\t24.txt\t27.txt\t30.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "moMt5EKZjg0u",
        "colab_type": "text"
      },
      "source": [
        "Load data and create dataframe for those tested as AD positive."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9kcSPlnvgCT7",
        "outputId": "1075366e-c12a-478e-da3a-ab818f1d6db5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "import glob\n",
        "import os\n",
        "\n",
        "ad_path = './data/AssignmentData/Assigment/AD'\n",
        "ad_filenames = glob.glob(os.path.join(ad_path, '*.txt'))\n",
        "ad_data = []\n",
        "\n",
        "for fn in ad_filenames:\n",
        "  with open(fn, 'r') as f:\n",
        "    ad_data.append(f.read().splitlines())\n",
        "\n",
        "# See the form of the imported data\n",
        "print(ad_data[0])\n",
        "# Count the number of words per patient file\n",
        "print([sum([len(s.split(' ')) for s in p]) for p in ad_data])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['oh boy .', 'alright .', 'family is in the kitchen .', \"the mother's washing dishes .\", 'and her sink is overflowing .', \"and she's looking out the window .\", \"and the two kids are taking they're stealing cookies off the out_of the cupboard .\", \"and the boy looks like he's gonna  fall down and hurt himself or fall against his mother .\", \"and the girl is whispering “don't make too much noise” to him .\", \"she's &let or else she's laughin at him .\", 'they got the cookies .', \"alright now though the window, let's see .\", \"there's a nice look outside, real nice .\", 'I told you the water was running over and splashing onto the floor .', \"and the mother doesn't seem too too affected by it .\", \"she's dryin a dish or wiping it .\", \"let's see .\", \"I guess the girl is laughing at her brother because he's going to fall .\", 'looks like a nice house .', \"there is a little bit of very little  but I don't think that's meant for this .\", 'the corner that got to the corner .', 'so this is a corner here .', \"and that goes back into there but that's  .\", \"do you see what I'm making reference to ?  \", 'the carpentry ?  \\t', '\\t']\n",
            "[223, 304, 138, 145, 245, 107, 265, 121, 338, 106, 64, 102, 300, 87, 105, 120, 93, 286, 125, 125, 105, 157, 65, 33, 200, 194, 93, 154, 72, 91]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "QHopFMSigCT5"
      },
      "source": [
        "All the above passages are within the 512 Bert input limit. Which is good as we can use Bert easily."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nN5NlCWFkIRa",
        "colab_type": "code",
        "outputId": "5b2cec34-0b02-4526-d255-3b4a90c8d3e4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 240
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "ad_data_concat = [(\" \").join(s) for s in ad_data]\n",
        "print(ad_data_concat[0])\n",
        "\n",
        "ad_df = pd.DataFrame(ad_data_concat, columns=['text'])\n",
        "ad_df['label'] = pd.Series([1 for x in range(len(ad_df.index))], index=ad_df.index)\n",
        "ad_df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "oh boy . alright . family is in the kitchen . the mother's washing dishes . and her sink is overflowing . and she's looking out the window . and the two kids are taking they're stealing cookies off the out_of the cupboard . and the boy looks like he's gonna  fall down and hurt himself or fall against his mother . and the girl is whispering “don't make too much noise” to him . she's &let or else she's laughin at him . they got the cookies . alright now though the window, let's see . there's a nice look outside, real nice . I told you the water was running over and splashing onto the floor . and the mother doesn't seem too too affected by it . she's dryin a dish or wiping it . let's see . I guess the girl is laughing at her brother because he's going to fall . looks like a nice house . there is a little bit of very little  but I don't think that's meant for this . the corner that got to the corner . so this is a corner here . and that goes back into there but that's  . do you see what I'm making reference to ?   the carpentry ?  \t \t\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>oh boy . alright . family is in the kitchen . ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>what's happening there ?    oh my . poor kids ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>oh little boy's in the cookie jar . the girl's...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>there's a little girl  reaching for the cookie...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>here's a cookie jar . and the lid is off the c...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label\n",
              "0  oh boy . alright . family is in the kitchen . ...      1\n",
              "1  what's happening there ?    oh my . poor kids ...      1\n",
              "2  oh little boy's in the cookie jar . the girl's...      1\n",
              "3  there's a little girl  reaching for the cookie...      1\n",
              "4  here's a cookie jar . and the lid is off the c...      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "US9OZoMq_u2L",
        "colab_type": "text"
      },
      "source": [
        "Let's do the same for the negative samples with label 0 this time."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "3e1d9f27-cb8f-42dd-8926-f3c184271438",
        "id": "B0bGGQnh_sPx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "import glob\n",
        "import os\n",
        "\n",
        "nc_path = './data/AssignmentData/Assigment/NC'\n",
        "nc_filenames = glob.glob(os.path.join(nc_path, '*.txt'))\n",
        "nc_data = []\n",
        "\n",
        "for fn in nc_filenames:\n",
        "  with open(fn, 'r') as f:\n",
        "    nc_data.append(f.read().splitlines())\n",
        "\n",
        "# See the form of the imported data\n",
        "print(nc_data[0])\n",
        "# Count the number of words per patient file\n",
        "print([sum([len(s.split(' ')) for s in p]) for p in nc_data])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['a boy is getting cookies out of the cookie jar . ', \"he's standing on a stool that's gonna fall . \", 'the girl is reaching for a cookie . ', \"the mother's drying dishes . \", \"the faucet's running water . \", \"it's dripping out of the sink . \", 'spilling onto the floor . ', 'dishes are on the counter . ', 'window is open .', 'must be summertime . ', 'the girl is laughing . ', \"looks like she's laughing . \", \"that's about it . \"]\n",
            "[90, 120, 89, 303, 180, 165, 62, 140, 120, 78, 197, 185, 305, 199, 180, 198, 78, 111, 93, 70, 212, 221, 277, 122, 123, 124, 203, 146, 117, 131]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "3bc66b78-b616-45f7-8678-97d6d0ec8d20",
        "id": "BrtU2iToALJz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 240
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "nc_data_concat = [(\" \").join(s) for s in nc_data]\n",
        "print(nc_data_concat[0])\n",
        "\n",
        "nc_df = pd.DataFrame(nc_data_concat, columns=['text'])\n",
        "nc_df['label'] = pd.Series([0 for x in range(len(nc_df.index))], index=nc_df.index)\n",
        "nc_df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "a boy is getting cookies out of the cookie jar .  he's standing on a stool that's gonna fall .  the girl is reaching for a cookie .  the mother's drying dishes .  the faucet's running water .  it's dripping out of the sink .  spilling onto the floor .  dishes are on the counter .  window is open . must be summertime .  the girl is laughing .  looks like she's laughing .  that's about it . \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a boy is getting cookies out of the cookie jar...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>the boy's getting cookies out o  the cookie ja...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>oh goody .  that boy's climbed on a stool that...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>are you ready ? well the sink is overflowing ....</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>a boy and a girl are in the kitchen with their...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label\n",
              "0  a boy is getting cookies out of the cookie jar...      0\n",
              "1  the boy's getting cookies out o  the cookie ja...      0\n",
              "2  oh goody .  that boy's climbed on a stool that...      0\n",
              "3  are you ready ? well the sink is overflowing ....      0\n",
              "4  a boy and a girl are in the kitchen with their...      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-VNFRoULgCT0",
        "colab": {}
      },
      "source": [
        "batch_1 = ad_df.append(nc_df, ignore_index=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "mPdgyehhgCTy"
      },
      "source": [
        "We can ask pandas how many sentences are labeled as \"positive\" (value 1) and how many are labeled \"negative\" (having the value 0)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "b326309d-565f-4dc2-c68b-4a55b38d6a4e",
        "id": "5EzgqfVOgCTs",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "batch_1['label'].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    30\n",
              "0    30\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "meNgXuvqgCTq"
      },
      "source": [
        "## Loading the Pre-trained BERT model\n",
        "Let's now load a pre-trained BERT model. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "e4f3b422-2184-4a73-8356-5e2096ccdd28",
        "id": "IYnZavPCgCTk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "ae76753485d4476a9c3e45d1eafff3e0",
            "69891a56516a4b8da857c1dfd3dfc108",
            "dd42dbcd604d419291df1bf22ac780ae",
            "6ac527a0cc964eeda664c413977e9be1",
            "63d0bc56232d4b60b7d417dcd40988a4",
            "9f49646c30b9435fb97dd718c14d6e50",
            "c257d181473c4bac80d6a477a226b715",
            "08374b1ddc0e49c89d15cb8706dc44be",
            "46aab3adf1fe4b29a67dd7c3a2510936",
            "730951b0665c4351a8c6ea90cc920222",
            "36ff95f18e9845af98f80435c6f7ff33",
            "3d426fed81c84e97af4a0e468c387d82",
            "4ed82c627db6425c937611968fd73321",
            "5cf51a8751d2474482d3f9d5d323f226",
            "34ec65d95a944e9cbc141004d80035fa",
            "f037181761474dab8bf06040f556bfbc",
            "2c04fab4dbe546b993aa601a4382169f",
            "b17875957d854245aec23730fd46b168",
            "2bd5bb3397074280aea2a0c17902a33a",
            "2efc747c0ab1419e9a78919cb085456e",
            "eb3429c52d5640918b5b649b4353c388",
            "4d09b5745e254f07a7f938a3fc5c474b",
            "7882a14e00be48adb3dae130f675f771",
            "6a014957c2ef4fbba33531b88d2e3d1d"
          ]
        }
      },
      "source": [
        "# For DistilBERT:\n",
        "model_class, tokenizer_class, pretrained_weights = (ppb.DistilBertModel, ppb.DistilBertTokenizer, 'distilbert-base-uncased')\n",
        "\n",
        "## Want BERT instead of distilBERT? Uncomment the following line:\n",
        "#model_class, tokenizer_class, pretrained_weights = (ppb.BertModel, ppb.BertTokenizer, 'bert-base-uncased')\n",
        "\n",
        "# Load pretrained model/tokenizer\n",
        "tokenizer = tokenizer_class.from_pretrained(pretrained_weights)\n",
        "model = model_class.from_pretrained(pretrained_weights)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ae76753485d4476a9c3e45d1eafff3e0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "46aab3adf1fe4b29a67dd7c3a2510936",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=546, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2c04fab4dbe546b993aa601a4382169f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=267967963, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "P3lzO5epgCTi"
      },
      "source": [
        "Right now, the variable `model` holds a pretrained distilBERT model -- a version of BERT that is smaller, but much faster and requiring a lot less memory.\n",
        "\n",
        "## Model #1: Preparing the Dataset\n",
        "Before we can hand our sentences to BERT, we need to so some minimal processing to put them in the format it requires.\n",
        "\n",
        "### Tokenization\n",
        "Our first step is to tokenize the sentences -- break them up into word and subwords in the format BERT is comfortable with."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "8msRmcu7gCTa",
        "outputId": "b77f5b6e-992f-4996-e2e4-d8c4ff709184",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120
        }
      },
      "source": [
        "tokenized = batch_1['text'].apply((lambda x: tokenizer.encode(x, add_special_tokens=True)))\n",
        "tokenized.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    [101, 2821, 2879, 1012, 10303, 1012, 2155, 200...\n",
              "1    [101, 2054, 1005, 1055, 6230, 2045, 1029, 2821...\n",
              "2    [101, 2821, 2210, 2879, 1005, 1055, 1999, 1996...\n",
              "3    [101, 2045, 1005, 1055, 1037, 2210, 2611, 4285...\n",
              "4    [101, 2182, 1005, 1055, 1037, 17387, 15723, 10...\n",
              "Name: text, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "a7EYkdufgCTY"
      },
      "source": [
        "### Padding\n",
        "After tokenization, `tokenized` is a list of sentences -- each sentences is represented as a list of tokens. We want BERT to process our examples all at once (as one batch). It's just faster that way. For that reason, we need to pad all lists to the same size, so we can represent the input as one 2-d array, rather than a list of lists (of different lengths)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "om30g15ygCTR",
        "outputId": "0c81944b-d335-414e-a57c-08c95f1b57d2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 703
        }
      },
      "source": [
        "max_len = 0\n",
        "for i in tokenized.values:\n",
        "    if len(i) > max_len:\n",
        "        max_len = len(i)\n",
        "\n",
        "padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])\n",
        "padded[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([  101,  2821,  2879,  1012, 10303,  1012,  2155,  2003,  1999,\n",
              "        1996,  3829,  1012,  1996,  2388,  1005,  1055, 12699, 10447,\n",
              "        1012,  1998,  2014,  7752,  2003,  2058, 12314,  2075,  1012,\n",
              "        1998,  2016,  1005,  1055,  2559,  2041,  1996,  3332,  1012,\n",
              "        1998,  1996,  2048,  4268,  2024,  2635,  2027,  1005,  2128,\n",
              "       11065, 16324,  2125,  1996,  2041,  1035,  1997,  1996, 25337,\n",
              "        1012,  1998,  1996,  2879,  3504,  2066,  2002,  1005,  1055,\n",
              "        6069,  2991,  2091,  1998,  3480,  2370,  2030,  2991,  2114,\n",
              "        2010,  2388,  1012,  1998,  1996,  2611,  2003, 13550,  1523,\n",
              "        2123,  1005,  1056,  2191,  2205,  2172,  5005,  1524,  2000,\n",
              "        2032,  1012,  2016,  1005,  1055,  1004,  2292,  2030,  2842,\n",
              "        2016,  1005,  1055,  4756,  2378,  2012,  2032,  1012,  2027,\n",
              "        2288,  1996, 16324,  1012, 10303,  2085,  2295,  1996,  3332,\n",
              "        1010,  2292,  1005,  1055,  2156,  1012,  2045,  1005,  1055,\n",
              "        1037,  3835,  2298,  2648,  1010,  2613,  3835,  1012,  1045,\n",
              "        2409,  2017,  1996,  2300,  2001,  2770,  2058,  1998, 17624,\n",
              "        2075,  3031,  1996,  2723,  1012,  1998,  1996,  2388,  2987,\n",
              "        1005,  1056,  4025,  2205,  2205,  5360,  2011,  2009,  1012,\n",
              "        2016,  1005,  1055,  4318,  2378,  1037,  9841,  2030, 14612,\n",
              "        2009,  1012,  2292,  1005,  1055,  2156,  1012,  1045,  3984,\n",
              "        1996,  2611,  2003,  5870,  2012,  2014,  2567,  2138,  2002,\n",
              "        1005,  1055,  2183,  2000,  2991,  1012,  3504,  2066,  1037,\n",
              "        3835,  2160,  1012,  2045,  2003,  1037,  2210,  2978,  1997,\n",
              "        2200,  2210,  2021,  1045,  2123,  1005,  1056,  2228,  2008,\n",
              "        1005,  1055,  3214,  2005,  2023,  1012,  1996,  3420,  2008,\n",
              "        2288,  2000,  1996,  3420,  1012,  2061,  2023,  2003,  1037,\n",
              "        3420,  2182,  1012,  1998,  2008,  3632,  2067,  2046,  2045,\n",
              "        2021,  2008,  1005,  1055,  1012,  2079,  2017,  2156,  2054,\n",
              "        1045,  1005,  1049,  2437,  4431,  2000,  1029,  1996, 29267,\n",
              "        4765,  2854,  1029,   102,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "huxZkb_tgCTP"
      },
      "source": [
        "Our dataset is now in the `padded` variable, we can view its dimensions below:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "17c9acf6-691d-4e06-a4d3-1485b02bf83e",
        "id": "KGYOCZXTgCTJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "np.array(padded).shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60, 357)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "vXqWL794gCTH"
      },
      "source": [
        "### Masking\n",
        "If we directly send `padded` to BERT, that would slightly confuse it. We need to create another variable to tell it to ignore (mask) the padding we've added when it's processing its input. That's what attention_mask is:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "db291ca0-f872-4b34-86b2-d90fc633242e",
        "id": "YghcLZ6HgCTB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 308
        }
      },
      "source": [
        "attention_mask = np.where(padded != 0, 1, 0)\n",
        "attention_mask[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s4qrEa9RNnDs",
        "colab_type": "code",
        "outputId": "7dee0cf7-41b7-4cb1-a0a1-ca169eaf7547",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "attention_mask.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60, 357)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "OVvPfk9ggCTA"
      },
      "source": [
        "## Model #1: And Now, Deep Learning!\n",
        "Now that we have our model and inputs ready, let's run our model!\n",
        "\n",
        "The `model()` function runs our sentences through BERT. The results of the processing will be returned into `last_hidden_states`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tx7E8CTqgCS5",
        "colab": {}
      },
      "source": [
        "input_ids = torch.tensor(padded)  \n",
        "attention_mask = torch.tensor(attention_mask)\n",
        "\n",
        "with torch.no_grad():\n",
        "    last_hidden_states = model(input_ids, attention_mask=attention_mask)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SZ8fd8t5O8io",
        "colab_type": "code",
        "outputId": "2e9ed0c7-6174-4be1-e954-0e53cb86d8f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "last_hidden_states[0].size()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([60, 357, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "V-PjwP4OgCS3"
      },
      "source": [
        "Let's slice only the part of the output that we need. That is the output corresponding the first token of each sentence. The way BERT does sentence classification, is that it adds a token called `[CLS]` (for classification) at the beginning of every sentence. The output corresponding to that token can be thought of as an embedding for the entire sentence.\n",
        "\n",
        "\n",
        "We'll save those in the `features` variable, as they'll serve as the features to our logitics regression model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "uzNbPNFMgCSw",
        "colab": {}
      },
      "source": [
        "features = last_hidden_states[0][:,0,:].numpy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "2vfqUR47gCSu"
      },
      "source": [
        "The labels indicating which sentence is positive and negative now go into the `labels` variable"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Lr2-3TdSgCSn",
        "colab": {}
      },
      "source": [
        "labels = batch_1['label']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "JKvSBrk4gCSl"
      },
      "source": [
        "## Model #2: Train/Test Split\n",
        "Let's now split our datset into a training set and testing set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Fwybze0agCSf",
        "colab": {}
      },
      "source": [
        "train_features, test_features, train_labels, test_labels = train_test_split(features, labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "fKnwGJBmgCSd"
      },
      "source": [
        "\n",
        "### [Bonus] Grid Search for Parameters\n",
        "We can dive into Logistic regression directly with the Scikit Learn default parameters, but sometimes it's worth searching for the best value of the C parameter, which determines regularization strength."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IwsaKV9igCSW",
        "colab": {}
      },
      "source": [
        "# parameters = {'C': np.linspace(0.0001, 100, 20)}\n",
        "# grid_search = GridSearchCV(LogisticRegression(), parameters)\n",
        "# grid_search.fit(train_features, train_labels)\n",
        "\n",
        "# print('best parameters: ', grid_search.best_params_)\n",
        "# print('best scrores: ', grid_search.best_score_)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "xKmPVaTUgCSU"
      },
      "source": [
        "We now train the LogisticRegression model. If you've chosen to do the gridsearch, you can plug the value of C into the model declaration (e.g. `LogisticRegression(C=5.2)`)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "bee16546-258f-4ed8-b3a3-d9862a924830",
        "id": "g8Hps4PYgCSK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 123
        }
      },
      "source": [
        "lr_clf = LogisticRegression()\n",
        "lr_clf.fit(train_features, train_labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
              "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
              "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
              "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
              "                   warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "i500IhYOgCSE"
      },
      "source": [
        "\n",
        "## Evaluating Model #2\n",
        "So how well does our model do in classifying sentences? One way is to check the accuracy against the testing dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "6b96679a-eb09-453c-801f-013b704b85bb",
        "id": "YyASo9_ogCRw",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "lr_clf.score(test_features, test_labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7333333333333333"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "0KuG9GBVgCRp"
      },
      "source": [
        "How good is this score? What can we compare it against? Let's first look at a dummy classifier:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "eed1c506-bfa9-48a6-9043-99e6d6804f19",
        "id": "en9G4EYTgCRV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from sklearn.dummy import DummyClassifier\n",
        "clf = DummyClassifier()\n",
        "\n",
        "scores = cross_val_score(clf, train_features, train_labels)\n",
        "print(\"Dummy classifier score: %0.3f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dummy classifier score: 0.489 (+/- 0.36)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "VNOQxRQhgCRR"
      },
      "source": [
        "So our model clearly does better than a dummy classifier. But how does it compare against the best models? We need to go to the relevant literature for that."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fE0a1pW0gCRA",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}