NLP_Lab_1_Tokenizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "### The Natural Language Toolkit, or more commonly NLTK, is a suite of libraries and programs for symbolic and statistical natural language processing (NLP) for English written in the Python programming language.\n",
        "### It supports classification, tokenization, stemming, tagging, parsing, and semantic reasoning functionalities"
      ],
      "metadata": {
        "id": "vmBaIMi3DVSU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Experiment 1\n",
        "## Convert the text into tokens."
      ],
      "metadata": {
        "id": "G6fRP0-uRaCn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Importing Required Library\n",
        "import nltk"
      ],
      "metadata": {
        "id": "WxmIw1F4RhN3"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Downloading All from Nltk\n",
        "# We are downloading all because we need functions for tokenization so instead of importing one by one we are directly downloading all functions\n",
        "nltk.download('all')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kNh6dRmu64HL",
        "outputId": "495dc352-a477-4cc0-d85b-3842864c5802"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading collection 'all'\n",
            "[nltk_data]    | \n",
            "[nltk_data]    | Downloading package abc to /root/nltk_data...\n",
            "[nltk_data]    |   Package abc is already up-to-date!\n",
            "[nltk_data]    | Downloading package alpino to /root/nltk_data...\n",
            "[nltk_data]    |   Package alpino is already up-to-date!\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package averaged_perceptron_tagger is already up-\n",
            "[nltk_data]    |       to-date!\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package averaged_perceptron_tagger_ru is already\n",
            "[nltk_data]    |       up-to-date!\n",
            "[nltk_data]    | Downloading package basque_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package basque_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package bcp47 to /root/nltk_data...\n",
            "[nltk_data]    |   Package bcp47 is already up-to-date!\n",
            "[nltk_data]    | Downloading package biocreative_ppi to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package biocreative_ppi is already up-to-date!\n",
            "[nltk_data]    | Downloading package bllip_wsj_no_aux to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package bllip_wsj_no_aux is already up-to-date!\n",
            "[nltk_data]    | Downloading package book_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package book_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]    |   Package brown is already up-to-date!\n",
            "[nltk_data]    | Downloading package brown_tei to /root/nltk_data...\n",
            "[nltk_data]    |   Package brown_tei is already up-to-date!\n",
            "[nltk_data]    | Downloading package cess_cat to /root/nltk_data...\n",
            "[nltk_data]    |   Package cess_cat is already up-to-date!\n",
            "[nltk_data]    | Downloading package cess_esp to /root/nltk_data...\n",
            "[nltk_data]    |   Package cess_esp is already up-to-date!\n",
            "[nltk_data]    | Downloading package chat80 to /root/nltk_data...\n",
            "[nltk_data]    |   Package chat80 is already up-to-date!\n",
            "[nltk_data]    | Downloading package city_database to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package city_database is already up-to-date!\n",
            "[nltk_data]    | Downloading package cmudict to /root/nltk_data...\n",
            "[nltk_data]    |   Package cmudict is already up-to-date!\n",
            "[nltk_data]    | Downloading package comparative_sentences to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package comparative_sentences is already up-to-\n",
            "[nltk_data]    |       date!\n",
            "[nltk_data]    | Downloading package comtrans to /root/nltk_data...\n",
            "[nltk_data]    |   Package comtrans is already up-to-date!\n",
            "[nltk_data]    | Downloading package conll2000 to /root/nltk_data...\n",
            "[nltk_data]    |   Package conll2000 is already up-to-date!\n",
            "[nltk_data]    | Downloading package conll2002 to /root/nltk_data...\n",
            "[nltk_data]    |   Package conll2002 is already up-to-date!\n",
            "[nltk_data]    | Downloading package conll2007 to /root/nltk_data...\n",
            "[nltk_data]    |   Package conll2007 is already up-to-date!\n",
            "[nltk_data]    | Downloading package crubadan to /root/nltk_data...\n",
            "[nltk_data]    |   Package crubadan is already up-to-date!\n",
            "[nltk_data]    | Downloading package dependency_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package dependency_treebank is already up-to-date!\n",
            "[nltk_data]    | Downloading package dolch to /root/nltk_data...\n",
            "[nltk_data]    |   Package dolch is already up-to-date!\n",
            "[nltk_data]    | Downloading package europarl_raw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package europarl_raw is already up-to-date!\n",
            "[nltk_data]    | Downloading package extended_omw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package extended_omw is already up-to-date!\n",
            "[nltk_data]    | Downloading package floresta to /root/nltk_data...\n",
            "[nltk_data]    |   Package floresta is already up-to-date!\n",
            "[nltk_data]    | Downloading package framenet_v15 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package framenet_v15 is already up-to-date!\n",
            "[nltk_data]    | Downloading package framenet_v17 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package framenet_v17 is already up-to-date!\n",
            "[nltk_data]    | Downloading package gazetteers to /root/nltk_data...\n",
            "[nltk_data]    |   Package gazetteers is already up-to-date!\n",
            "[nltk_data]    | Downloading package genesis to /root/nltk_data...\n",
            "[nltk_data]    |   Package genesis is already up-to-date!\n",
            "[nltk_data]    | Downloading package gutenberg to /root/nltk_data...\n",
            "[nltk_data]    |   Package gutenberg is already up-to-date!\n",
            "[nltk_data]    | Downloading package ieer to /root/nltk_data...\n",
            "[nltk_data]    |   Package ieer is already up-to-date!\n",
            "[nltk_data]    | Downloading package inaugural to /root/nltk_data...\n",
            "[nltk_data]    |   Package inaugural is already up-to-date!\n",
            "[nltk_data]    | Downloading package indian to /root/nltk_data...\n",
            "[nltk_data]    |   Package indian is already up-to-date!\n",
            "[nltk_data]    | Downloading package jeita to /root/nltk_data...\n",
            "[nltk_data]    |   Package jeita is already up-to-date!\n",
            "[nltk_data]    | Downloading package kimmo to /root/nltk_data...\n",
            "[nltk_data]    |   Package kimmo is already up-to-date!\n",
            "[nltk_data]    | Downloading package knbc to /root/nltk_data...\n",
            "[nltk_data]    |   Package knbc is already up-to-date!\n",
            "[nltk_data]    | Downloading package large_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package large_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package lin_thesaurus to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package lin_thesaurus is already up-to-date!\n",
            "[nltk_data]    | Downloading package mac_morpho to /root/nltk_data...\n",
            "[nltk_data]    |   Package mac_morpho is already up-to-date!\n",
            "[nltk_data]    | Downloading package machado to /root/nltk_data...\n",
            "[nltk_data]    |   Package machado is already up-to-date!\n",
            "[nltk_data]    | Downloading package masc_tagged to /root/nltk_data...\n",
            "[nltk_data]    |   Package masc_tagged is already up-to-date!\n",
            "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package maxent_ne_chunker is already up-to-date!\n",
            "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package maxent_treebank_pos_tagger is already up-\n",
            "[nltk_data]    |       to-date!\n",
            "[nltk_data]    | Downloading package moses_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package moses_sample is already up-to-date!\n",
            "[nltk_data]    | Downloading package movie_reviews to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package movie_reviews is already up-to-date!\n",
            "[nltk_data]    | Downloading package mte_teip5 to /root/nltk_data...\n",
            "[nltk_data]    |   Package mte_teip5 is already up-to-date!\n",
            "[nltk_data]    | Downloading package mwa_ppdb to /root/nltk_data...\n",
            "[nltk_data]    |   Package mwa_ppdb is already up-to-date!\n",
            "[nltk_data]    | Downloading package names to /root/nltk_data...\n",
            "[nltk_data]    |   Package names is already up-to-date!\n",
            "[nltk_data]    | Downloading package nombank.1.0 to /root/nltk_data...\n",
            "[nltk_data]    |   Package nombank.1.0 is already up-to-date!\n",
            "[nltk_data]    | Downloading package nonbreaking_prefixes to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package nonbreaking_prefixes is already up-to-date!\n",
            "[nltk_data]    | Downloading package nps_chat to /root/nltk_data...\n",
            "[nltk_data]    |   Package nps_chat is already up-to-date!\n",
            "[nltk_data]    | Downloading package omw to /root/nltk_data...\n",
            "[nltk_data]    |   Package omw is already up-to-date!\n",
            "[nltk_data]    | Downloading package omw-1.4 to /root/nltk_data...\n",
            "[nltk_data]    |   Package omw-1.4 is already up-to-date!\n",
            "[nltk_data]    | Downloading package opinion_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package opinion_lexicon is already up-to-date!\n",
            "[nltk_data]    | Downloading package panlex_swadesh to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package panlex_swadesh is already up-to-date!\n",
            "[nltk_data]    | Downloading package paradigms to /root/nltk_data...\n",
            "[nltk_data]    |   Package paradigms is already up-to-date!\n",
            "[nltk_data]    | Downloading package pe08 to /root/nltk_data...\n",
            "[nltk_data]    |   Package pe08 is already up-to-date!\n",
            "[nltk_data]    | Downloading package perluniprops to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package perluniprops is already up-to-date!\n",
            "[nltk_data]    | Downloading package pil to /root/nltk_data...\n",
            "[nltk_data]    |   Package pil is already up-to-date!\n",
            "[nltk_data]    | Downloading package pl196x to /root/nltk_data...\n",
            "[nltk_data]    |   Package pl196x is already up-to-date!\n",
            "[nltk_data]    | Downloading package porter_test to /root/nltk_data...\n",
            "[nltk_data]    |   Package porter_test is already up-to-date!\n",
            "[nltk_data]    | Downloading package ppattach to /root/nltk_data...\n",
            "[nltk_data]    |   Package ppattach is already up-to-date!\n",
            "[nltk_data]    | Downloading package problem_reports to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package problem_reports is already up-to-date!\n",
            "[nltk_data]    | Downloading package product_reviews_1 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package product_reviews_1 is already up-to-date!\n",
            "[nltk_data]    | Downloading package product_reviews_2 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package product_reviews_2 is already up-to-date!\n",
            "[nltk_data]    | Downloading package propbank to /root/nltk_data...\n",
            "[nltk_data]    |   Package propbank is already up-to-date!\n",
            "[nltk_data]    | Downloading package pros_cons to /root/nltk_data...\n",
            "[nltk_data]    |   Package pros_cons is already up-to-date!\n",
            "[nltk_data]    | Downloading package ptb to /root/nltk_data...\n",
            "[nltk_data]    |   Package ptb is already up-to-date!\n",
            "[nltk_data]    | Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]    |   Package punkt is already up-to-date!\n",
            "[nltk_data]    | Downloading package qc to /root/nltk_data...\n",
            "[nltk_data]    |   Package qc is already up-to-date!\n",
            "[nltk_data]    | Downloading package reuters to /root/nltk_data...\n",
            "[nltk_data]    |   Package reuters is already up-to-date!\n",
            "[nltk_data]    | Downloading package rslp to /root/nltk_data...\n",
            "[nltk_data]    |   Package rslp is already up-to-date!\n",
            "[nltk_data]    | Downloading package rte to /root/nltk_data...\n",
            "[nltk_data]    |   Package rte is already up-to-date!\n",
            "[nltk_data]    | Downloading package sample_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package sample_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package semcor to /root/nltk_data...\n",
            "[nltk_data]    |   Package semcor is already up-to-date!\n",
            "[nltk_data]    | Downloading package senseval to /root/nltk_data...\n",
            "[nltk_data]    |   Package senseval is already up-to-date!\n",
            "[nltk_data]    | Downloading package sentence_polarity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package sentence_polarity is already up-to-date!\n",
            "[nltk_data]    | Downloading package sentiwordnet to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package sentiwordnet is already up-to-date!\n",
            "[nltk_data]    | Downloading package shakespeare to /root/nltk_data...\n",
            "[nltk_data]    |   Package shakespeare is already up-to-date!\n",
            "[nltk_data]    | Downloading package sinica_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package sinica_treebank is already up-to-date!\n",
            "[nltk_data]    | Downloading package smultron to /root/nltk_data...\n",
            "[nltk_data]    |   Package smultron is already up-to-date!\n",
            "[nltk_data]    | Downloading package snowball_data to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package snowball_data is already up-to-date!\n",
            "[nltk_data]    | Downloading package spanish_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package spanish_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package state_union to /root/nltk_data...\n",
            "[nltk_data]    |   Package state_union is already up-to-date!\n",
            "[nltk_data]    | Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]    |   Package stopwords is already up-to-date!\n",
            "[nltk_data]    | Downloading package subjectivity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package subjectivity is already up-to-date!\n",
            "[nltk_data]    | Downloading package swadesh to /root/nltk_data...\n",
            "[nltk_data]    |   Package swadesh is already up-to-date!\n",
            "[nltk_data]    | Downloading package switchboard to /root/nltk_data...\n",
            "[nltk_data]    |   Package switchboard is already up-to-date!\n",
            "[nltk_data]    | Downloading package tagsets to /root/nltk_data...\n",
            "[nltk_data]    |   Package tagsets is already up-to-date!\n",
            "[nltk_data]    | Downloading package timit to /root/nltk_data...\n",
            "[nltk_data]    |   Package timit is already up-to-date!\n",
            "[nltk_data]    | Downloading package toolbox to /root/nltk_data...\n",
            "[nltk_data]    |   Package toolbox is already up-to-date!\n",
            "[nltk_data]    | Downloading package treebank to /root/nltk_data...\n",
            "[nltk_data]    |   Package treebank is already up-to-date!\n",
            "[nltk_data]    | Downloading package twitter_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package twitter_samples is already up-to-date!\n",
            "[nltk_data]    | Downloading package udhr to /root/nltk_data...\n",
            "[nltk_data]    |   Package udhr is already up-to-date!\n",
            "[nltk_data]    | Downloading package udhr2 to /root/nltk_data...\n",
            "[nltk_data]    |   Package udhr2 is already up-to-date!\n",
            "[nltk_data]    | Downloading package unicode_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package unicode_samples is already up-to-date!\n",
            "[nltk_data]    | Downloading package universal_tagset to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package universal_tagset is already up-to-date!\n",
            "[nltk_data]    | Downloading package universal_treebanks_v20 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package universal_treebanks_v20 is already up-to-\n",
            "[nltk_data]    |       date!\n",
            "[nltk_data]    | Downloading package vader_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package vader_lexicon is already up-to-date!\n",
            "[nltk_data]    | Downloading package verbnet to /root/nltk_data...\n",
            "[nltk_data]    |   Package verbnet is already up-to-date!\n",
            "[nltk_data]    | Downloading package verbnet3 to /root/nltk_data...\n",
            "[nltk_data]    |   Package verbnet3 is already up-to-date!\n",
            "[nltk_data]    | Downloading package webtext to /root/nltk_data...\n",
            "[nltk_data]    |   Package webtext is already up-to-date!\n",
            "[nltk_data]    | Downloading package wmt15_eval to /root/nltk_data...\n",
            "[nltk_data]    |   Package wmt15_eval is already up-to-date!\n",
            "[nltk_data]    | Downloading package word2vec_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package word2vec_sample is already up-to-date!\n",
            "[nltk_data]    | Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]    |   Package wordnet is already up-to-date!\n",
            "[nltk_data]    | Downloading package wordnet2021 to /root/nltk_data...\n",
            "[nltk_data]    |   Package wordnet2021 is already up-to-date!\n",
            "[nltk_data]    | Downloading package wordnet2022 to /root/nltk_data...\n",
            "[nltk_data]    |   Package wordnet2022 is already up-to-date!\n",
            "[nltk_data]    | Downloading package wordnet31 to /root/nltk_data...\n",
            "[nltk_data]    |   Package wordnet31 is already up-to-date!\n",
            "[nltk_data]    | Downloading package wordnet_ic to /root/nltk_data...\n",
            "[nltk_data]    |   Package wordnet_ic is already up-to-date!\n",
            "[nltk_data]    | Downloading package words to /root/nltk_data...\n",
            "[nltk_data]    |   Package words is already up-to-date!\n",
            "[nltk_data]    | Downloading package ycoe to /root/nltk_data...\n",
            "[nltk_data]    |   Package ycoe is already up-to-date!\n",
            "[nltk_data]    | \n",
            "[nltk_data]  Done downloading collection all\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Tokenization in NLP is the process of breaking text into individual units or tokens, such as words or subwords, to facilitate language analysis and understanding in natural language processing tasks."
      ],
      "metadata": {
        "id": "LdIz2MspDO5j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Declaring text\n",
        "text1 = \"My Name is Xyz\"\n",
        "print(\"Original Text is: \",text1)\n",
        "# Using Word tokenize\n",
        "# This will tokenize the sentence into words and these words will be called tokens\n",
        "tokens1 = nltk.word_tokenize(text1)\n",
        "print(\"After Using Word Tokenize: \",tokens1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e6FHOHt266Yx",
        "outputId": "24d50994-10b4-45de-8563-a0e407a1d360"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Text is:  My Name is Xyz\n",
            "After Using Word Tokenize:  ['My', 'Name', 'is', 'Xyz']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Declaring text\n",
        "text2 = \"My Name is Xyz.I am learning NLP. We are doing first experiment\"\n",
        "print(\"Original Text is: \",text2)\n",
        "# Using Sent tokenize\n",
        "# We need to give space after full stop and it will act as delimiter else it will treat all the sentence as one\n",
        "# This will tokenize the sentence into words and these words will be called tokens\n",
        "tokens2 = nltk.sent_tokenize(text2)\n",
        "print(\"After Using Sent Tokenize: \",tokens2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3gqVn20Q73Te",
        "outputId": "9b134e01-9d2e-44b3-cc88-556a00d324dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Text is:  My Name is Xyz.I am learning NLP. We are doing first experiment\n",
            "After Using Sent Tokenize:  ['My Name is Xyz.I am learning NLP.', 'We are doing first experiment']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Now if we do not want to download entire nltk library functions we can import tokenizers using nltk.tokenizer\n",
        "from nltk.tokenize import word_tokenize,sent_tokenize"
      ],
      "metadata": {
        "id": "nepNsEmCERJA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### We can perform the same functions using this as well"
      ],
      "metadata": {
        "id": "DqlKuXf7FC7d"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Declaring text\n",
        "text1 = \"My Name is Xyz\"\n",
        "print(\"Original Text is: \",text1)\n",
        "# Using Word tokenize\n",
        "# This will tokenize the sentence into words and these words will be called tokens\n",
        "tokens1 = nltk.word_tokenize(text1)\n",
        "print(\"After Using Word Tokenize: \",tokens1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2d821243-7e56-43d7-dbf2-f03a7344b7af",
        "id": "qbLMU6RcEv5y"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Text is:  My Name is Xyz\n",
            "After Using Word Tokenize:  ['My', 'Name', 'is', 'Xyz']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Declaring text\n",
        "text2 = \"My Name is Xyz. I am learning NLP. We are doing first experiment\"\n",
        "print(\"Original Text is: \",text2)\n",
        "# Using Sent tokenize\n",
        "# Sent Tokenize stands for Sentence Tokenize\n",
        "# We need to give space after full stop and it will act as delimiter else it will treat all the sentence as one\n",
        "# This will tokenize the sentence into words and these words will be called tokens\n",
        "tokens2 = nltk.sent_tokenize(text2)\n",
        "print(\"After Using Sent Tokenize: \",tokens2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d1a73715-2505-48b7-e89d-36d8a862961e",
        "id": "C8fkPX11FBNa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Text is:  My Name is Xyz. I am learning NLP. We are doing first experiment\n",
            "After Using Sent Tokenize:  ['My Name is Xyz.', 'I am learning NLP.', 'We are doing first experiment']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Trying More Tokenizers available in NLP\n",
        "For This we have to import required librarires as well\n",
        "1. **Whitespace Tokenizer**\n",
        "   - **Library**: Python's `split()` or NLTK's `WhitespaceTokenizer`\n",
        "   - Splits text on whitespace to create tokens.\n",
        "\n",
        "2. **Word Tokenizer**\n",
        "   - **Library**: NLTK's `word_tokenize` or spaCy\n",
        "   - Splits text into words based on language-specific rules.\n",
        "\n",
        "3. **Sentence Tokenizer**\n",
        "   - **Library**: NLTK's `sent_tokenize` or spaCy\n",
        "   - Splits text into sentences based on punctuation and grammar.\n",
        "\n",
        "4. **Subword Tokenizer**\n",
        "   - **Library**: Hugging Face's `tokenizers` or SentencePiece\n",
        "   - Segments text into subword units, useful for languages with complex morphology.\n",
        "\n",
        "5. **BERT Tokenizer**\n",
        "   - **Library**: Hugging Face's `transformers`\n",
        "   - Tokenizes text for BERT-based models, preserving subword information.\n",
        "\n",
        "6. **Byte-Pair Encoding (BPE) Tokenizer**\n",
        "   - **Library**: Hugging Face's `tokenizers` or GPT-2's `BPE`\n",
        "   - Splits text into variable-length subword units, often used in neural machine translation.\n",
        "\n",
        "7. **Regular Expression Tokenizer**\n",
        "   - **Library**: Python's `re`\n",
        "   - Tokenizes text based on custom-defined regular expressions.\n",
        "\n",
        "11. **Penn Treebank Tokenizer**\n",
        "    - **Library**: NLTK's `TreebankWordTokenizer`\n",
        "    - Tokenizes text according to the Penn Treebank tokenization rules, often used in parsing tasks.\n",
        "\n",
        "12. **Moses Tokenizer**\n",
        "    - **Library**: NLTK's `MosesTokenizer` or the Moses SMT toolkit\n",
        "    - Tokenizes text with a focus on statistical machine translation, handling complex tokenization tasks.\n",
        "\n",
        "8. **Custom Tokenizer**\n",
        "   - **Library**: Depends on the implementation\n",
        "   - Allows for user-defined tokenization rules tailored to specific tasks or languages."
      ],
      "metadata": {
        "id": "nPUzhOdcFVZx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# We have already implemented word and sentence tokenizer now we can use the rest\n",
        "# Declaring text\n",
        "text = \"My Name is Xyz. I am learning NLP. We are doing first experiment\"\n",
        "print(\"Original Text is: \",text)\n",
        "\n",
        "# Using Whitespace tokenizer\n",
        "# First we create an instance of class whitespacetokenizer and then we can use it\n",
        "wh_tk_instance = nltk.WhitespaceTokenizer()\n",
        "\n",
        "# Now using whitespace tokenizer\n",
        "wh_tk = wh_tk_instance.tokenize(text)\n",
        "print(\"After Using Whitespace Tokenize: \",wh_tk)"
      ],
      "metadata": {
        "id": "70bYYhTXG39t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff028d12-d9a5-4611-c70b-9381d37b5936"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Text is:  My Name is Xyz. I am learning NLP. We are doing first experiment\n",
            "After Using Whitespace Tokenize:  ['My', 'Name', 'is', 'Xyz.', 'I', 'am', 'learning', 'NLP.', 'We', 'are', 'doing', 'first', 'experiment']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# We have already implemented word and sentence tokenizer now we can use the rest\n",
        "# Declaring text\n",
        "text = \"My Name is Xyz. I am learning NLP. We are doing first experiment\"\n",
        "print(\"Original Text is: \",text)\n",
        "\n",
        "# Using Penn Treebank Tokenizer\n",
        "# First we create an instance of class Penn Treebank and then we can use it\n",
        "penn_tk_instance = nltk.TreebankWordTokenizer()\n",
        "\n",
        "# Now using whitespace tokenizer\n",
        "penn_tk = penn_tk_instance.tokenize(text)\n",
        "print(\"After Using Penn Treebank Tokenize: \",penn_tk)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BUVXWtuBV7Pn",
        "outputId": "359c523b-0762-422b-ac78-0313e1db4832"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Text is:  My Name is Xyz. I am learning NLP. We are doing first experiment\n",
            "After Using Penn Treebank Tokenize:  ['My', 'Name', 'is', 'Xyz.', 'I', 'am', 'learning', 'NLP.', 'We', 'are', 'doing', 'first', 'experiment']\n"
          ]
        }
      ]
    }
  ]
}