Aula02_nltk.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Aula02_nltk",
      "provenance": [],
      "authorship_tag": "ABX9TyOln5outrJDDTCzd05WSug2",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Muramatsu2602/nlp-intro/blob/main/Aula02_nltk.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y-Q2a4etyDnV",
        "outputId": "9c2c5eea-631b-404e-dd24-ca5ec4883d7b"
      },
      "source": [
        "import nltk\r\n",
        "nltk.download()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "NLTK Downloader\n",
            "---------------------------------------------------------------------------\n",
            "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
            "---------------------------------------------------------------------------\n",
            "Downloader> q\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sbbTkh0s1WMK"
      },
      "source": [
        "**Tokenization**\r\n",
        "diz respeito ao nivel lexico. \r\n",
        "\r\n",
        "Token --> uma palavra, numero ou pontuacao\r\n",
        "\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LiXMgwxg1VId"
      },
      "source": [
        "texto = \" o jogador, que esta com a camisa 10, marcou o gol da vitoria!\""
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FCld1kZf15CC"
      },
      "source": [
        "nltk.word_tokenize(texto) # devolve lista"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NQe5OaHA2K8k"
      },
      "source": [
        "from nltk.tokenize import RegexpTokenizer\r\n",
        "tokenizer = RegexpTokenizer(r'\\w+') # regex --> so caracteres\r\n",
        "tokens = tokenizer.tokenize(texto)\r\n",
        "\r\n",
        "tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyKWuDfW4S5z"
      },
      "source": [
        "from nltk.tokenize import RegexpTokenizer\r\n",
        "\r\n",
        "tokenizer = RegexpTokenizer(r'[0-9]\\w*') \r\n",
        "tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qVPHE0rC5KhU"
      },
      "source": [
        "from google.colab import drive\r\n",
        "drive.mount('/content/drive')\r\n",
        "infile = open('/content/drive/My Drive/Colab Notebooks/arquivos_text/corpus.txt')\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tDkOJGIu5pLJ"
      },
      "source": [
        "tokenizer = RegexpTokenizer()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZwJLEoEl5xF-"
      },
      "source": [
        "frequencia = nltk.FreqDist(tokens)\r\n",
        "frequencia.most_common() # frequencia GERAL do corpus"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7xS-4GgP52VN"
      },
      "source": [
        "frequencia.most_common(5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mldToSUK6Viz"
      },
      "source": [
        "frequencia = nltk.FreqDist(w.lower() for w in tokens)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}