IPU_roberta_pretraining.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iebqEdlNGzCA"
   },
   "source": [
    "# Pre-training Tiny-RoBERTa on BabyLM data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gWV1X0uaGzCE"
   },
   "outputs": [],
   "source": [
    "%pip install -q evaluate\n",
    "%pip install -q \"optimum-graphcore>=0.5, <0.6\" # Version must be under < 0.6\n",
    "%pip install huggingface_hub==0.10.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "u7gO7Zu2GzCG"
   },
   "outputs": [],
   "source": [
    "!apt install git-lfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Qh7KUZokGzCH"
   },
   "outputs": [],
   "source": [
    "import yaml\n",
    "import random\n",
    "from typing import List, Dict, Any, Tuple\n",
    "from pathlib import Path\n",
    "import os\n",
    "from tokenizers import Tokenizer\n",
    "from datasets import load_dataset\n",
    "from optimum.graphcore import IPUConfig, IPUTrainer, IPUTrainingArguments\n",
    "from transformers import RobertaTokenizerFast,DataCollatorForLanguageModeling,AutoConfig, AutoModelForMaskedLM\n",
    "from transformers import RobertaConfig, RobertaForMaskedLM,pipeline,AutoTokenizer\n",
    "from huggingface_hub import notebook_login"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IVa74bKFGzCI"
   },
   "source": [
    "Pre-training RoBERTa on BabyLM requires:\n",
    "\n",
    "* BabyLM data that can be loaded via:  \n",
    "```\n",
    "git clone https://github.com/upunaprosk/small-language-models.git\n",
    "cd small-language-models\n",
    "bash download_data.sh\n",
    "```\n",
    "\n",
    "* RoBERTa Tokenizer pre-trained on BabyLM data using [that code](https://github.com/upunaprosk/BabyBERTa)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VNZZs-r6iKAV",
    "outputId": "61192782-6210-46f5-d42d-6cb5dab4b60f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, 'pod4')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_ipu = int(os.getenv(\"NUM_AVAILABLE_IPU\", 4))\n",
    "executable_cache_dir = os.getenv(\"POPLAR_EXECUTABLE_CACHE_DIR\", \"/tmp/exe_cache/\") + \"/roberta\"\n",
    "pod_type = os.getenv(\"GRAPHCORE_POD_TYPE\", \"pod4\")\n",
    "path_tokenizer_config = 'trained-tokenizer/custom_tokenizer.json' # Path to pre-trained tokenizer\n",
    "n_ipu,pod_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "v1dNs14dGzCK"
   },
   "outputs": [],
   "source": [
    "def load_tokenizer(config_path: Path,\n",
    "                   max_input_length: int,\n",
    "                   ) -> Tokenizer:\n",
    "\n",
    "    tokenizer = Tokenizer.from_file(str(config_path))\n",
    "    tokenizer.enable_truncation(max_length=max_input_length)\n",
    "\n",
    "    return tokenizer\n",
    "\n",
    "tokenizer = load_tokenizer(path_tokenizer_config, max_input_length=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sjglGL17GzCL"
   },
   "outputs": [],
   "source": [
    "tokenizer = RobertaTokenizerFast(vocab_file=None,\n",
    "                                     merges_file=None,\n",
    "                                     tokenizer_file=path_tokenizer_config,\n",
    "                                     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "l2Uxh6g8GzCM",
    "outputId": "ca254fcc-e6e4-4606-ebb2-b4da499f0c41"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data/qed.test',\n",
       " 'data/switchboard.test',\n",
       " 'data/cbt.test',\n",
       " 'data/simple_wikipedia.test',\n",
       " 'data/aochildes.test',\n",
       " 'data/children_stories.test',\n",
       " 'data/gutenberg.test',\n",
       " 'data/wikipedia.test',\n",
       " 'data/bnc_spoken.test',\n",
       " 'data/open_subtitles.test']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = Path('./babylm_data/babylm_10M/')\n",
    "files=[p.as_posix() for p in data_path.glob('*.train')]\n",
    "files_dev = [p.as_posix() for p in data_path.glob('*.dev')]\n",
    "files_test = [p.as_posix() for p in data_path.glob('*.test')]\n",
    "files_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sSrFghGZGzCM"
   },
   "outputs": [],
   "source": [
    "d = load_dataset('text', data_files={'train': list(files)})\n",
    "d_dev = load_dataset('text', data_files={'train': list(files_dev)})\n",
    "d_test = load_dataset('text', data_files={'train': list(files_test)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "j_nxu2iaGzCN"
   },
   "outputs": [],
   "source": [
    "block_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "v4Os-N0JGzCN",
    "outputId": "f543a2d2-02a3-49d2-8020-142dd0adef02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of train data=1015494\n"
     ]
    }
   ],
   "source": [
    "def tokenize_function(examples):\n",
    "    # Remove empty lines\n",
    "    examples[\"text\"] = [line for line in examples[\"text\"] if len(line) > 0 and not line.isspace()]\n",
    "    return tokenizer(\n",
    "        examples[\"text\"],\n",
    "        padding=True,\n",
    "        truncation=True,\n",
    "        max_length=128,\n",
    "        # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it\n",
    "        # receives the `special_tokens_mask`.\n",
    "        # return_special_tokens_mask=True,\n",
    "    )\n",
    "tokenized_datasets = d.map(tokenize_function, batched=True, num_proc=4, remove_columns=[\"text\"])\n",
    "tokenized_datasets_dev = d_dev.map(tokenize_function, batched=True, num_proc=4, remove_columns=[\"text\"])\n",
    "tokenized_datasets_test = d_test.map(tokenize_function, batched=True, num_proc=4, remove_columns=[\"text\"])\n",
    "\n",
    "print(f'Length of train data={len(tokenized_datasets[\"train\"])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fcZWacxYGzCN"
   },
   "outputs": [],
   "source": [
    "def group_texts(examples):\n",
    "    # Concatenate all texts.\n",
    "    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
    "    total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
    "    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n",
    "    # customize this part to your needs.\n",
    "    total_length = (total_length // block_size) * block_size\n",
    "    # Split by chunks of max_len.\n",
    "    result = {\n",
    "        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n",
    "        for k, t in concatenated_examples.items()\n",
    "    }\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_fRA4iXaGzCO",
    "outputId": "b287a994-5359-4bdb-ebac-accd6dc4eb4c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8192"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2LBavcO_GzCO"
   },
   "outputs": [],
   "source": [
    "ipu_config_name = \"Graphcore/roberta-base-ipu\"\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=tokenizer, mlm=True, mlm_probability=0.135\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sHK6S5XaGzCO"
   },
   "outputs": [],
   "source": [
    "DEFAULT_ROBERTA_CONFIG={\n",
    "  \"attention_probs_dropout_prob\": 0.1,\n",
    "  \"bos_token_id\": 0,\n",
    "  \"classifier_dropout\": None,\n",
    "  \"eos_token_id\": 2,\n",
    "  \"hidden_act\": \"gelu\",\n",
    "  \"hidden_dropout_prob\": 0.1,\n",
    "  \"hidden_size\": 768,\n",
    "  \"initializer_range\": 0.02,\n",
    "  \"intermediate_size\": 3072,\n",
    "  \"layer_norm_eps\": 1e-12,\n",
    "  \"max_position_embeddings\": 512,\n",
    "  \"model_type\": \"roberta\",\n",
    "  \"num_attention_heads\": 12,\n",
    "  \"num_hidden_layers\": 12,\n",
    "  \"pad_token_id\": 1,\n",
    "  \"position_embedding_type\": \"absolute\",\n",
    "  \"transformers_version\": \"4.20.1\",\n",
    "  \"type_vocab_size\": 2,\n",
    "  \"use_cache\": True,\n",
    "  \"vocab_size\": 30522\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cMgT2ypuGzCO"
   },
   "outputs": [],
   "source": [
    "# param_id  trial_id  param_name                   param_value  distribution_json\n",
    "# 318       40        hidden_size_multiplier       87.000000    {\"name\": \"IntDistribution\", \"attributes\": {\"log\": false, \"step\": 1, \"low\": 1, \"high\": 100}}                           1\n",
    "# 319       40        hidden_layers                9.000000     {\"name\": \"IntDistribution\", \"attributes\": {\"log\": false, \"step\": 1, \"low\": 1, \"high\": 12}}                            1\n",
    "# 320       40        attention_heads              12.000000    {\"name\": \"IntDistribution\", \"attributes\": {\"log\": false, \"step\": 1, \"low\": 1, \"high\": 18}}                            1\n",
    "# 321       40        intermediate_size            2048.000000  {\"name\": \"IntDistribution\", \"attributes\": {\"log\": false, \"step\": 1, \"low\": 1, \"high\": 3072}}                          1\n",
    "# 322       40        hidden_act                   3.000000     {\"name\": \"CategoricalDistribution\", \"attributes\": {\"choices\": [\"gelu\", \"relu\", \"silu\", \"gelu_new\"]}}                  1\n",
    "# 323       40        hidden_dropout_prob          0.146995     {\"name\": \"FloatDistribution\", \"attributes\": {\"step\": null, \"low\": 0.1, \"high\": 1.0, \"log\": false}}                    1\n",
    "# 324       40        attention_prob_dropout_prog  0.995935     {\"name\": \"FloatDistribution\", \"attributes\": {\"step\": null, \"low\": 0.1, \"high\": 1.0, \"log\": false}}                    1\n",
    "# 325       40        position_embedding_type      2.000000     {\"name\": \"CategoricalDistribution\", \"attributes\": {\"choices\": [\"absolute\", \"relative_key\", \"relative_key_query\"]}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "88mULj47GzCP"
   },
   "outputs": [],
   "source": [
    "# hidden_size_multiplier meaning\n",
    "# hidden_size=model_parameters['hidden_size_multiplier'] * model_parameters['num_attention_heads'],\n",
    "# 87*12=1044\n",
    "# by DEFAULT:\"hidden_size\": 768, hidden_size_multiplier = 64\n",
    "# optimum value:\n",
    "# hidden_size_multiplier=70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KVrb7kssGzCP"
   },
   "outputs": [],
   "source": [
    "opt_roberta_config={\n",
    "  \"pad_token_id\":tokenizer.convert_tokens_to_ids ('<pad>'),\n",
    "  \"bos_token_id\":tokenizer.convert_tokens_to_ids('<s>'),\n",
    "  \"eos_token_id\":tokenizer.convert_tokens_to_ids('</s>'),\n",
    "  \"attention_probs_dropout_prob\": 0.3,\n",
    "  \"hidden_act\": \"gelu_new\",\n",
    "  \"hidden_dropout_prob\": 0.15,\n",
    "  \"hidden_size\": 70*8,\n",
    "  \"initializer_range\": 0.02,\n",
    "  \"intermediate_size\": 1412,\n",
    "  \"layer_norm_eps\": 1e-12,\n",
    "  \"max_position_embeddings\": 128,\n",
    "  \"model_type\": \"roberta\",\n",
    "  \"num_attention_heads\": 8,\n",
    "  \"num_hidden_layers\": 4,\n",
    "  \"position_embedding_type\": \"relative_key_query\",\n",
    "  \"type_vocab_size\": 1,\n",
    "  \"vocab_size\": tokenizer.vocab_size\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "s27Vq0vvGzCP"
   },
   "outputs": [],
   "source": [
    "config = RobertaConfig(**opt_roberta_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kD5BqiWkGzCP",
    "outputId": "7ba858f8-b9e0-46a0-98c6-1e11c8467d9d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting replicated_tensor_sharding to False when replication_factor=1\n"
     ]
    }
   ],
   "source": [
    "model = RobertaForMaskedLM(config)\n",
    "ipu_config = IPUConfig.from_pretrained(\n",
    "    \"Graphcore/roberta-base-ipu\",\n",
    "    executable_cache_dir=\"/tmp/exe_cache/3.2.1/roberta\",\n",
    "    ipus_per_replica= 3,\n",
    "    layers_per_ipu= [\n",
    "        1,\n",
    "        1,\n",
    "        1,\n",
    "        1\n",
    "    ],\n",
    "    inference_layers_per_ipu=[-1]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RucHDSHKGzCP",
    "outputId": "976603ec-f5bd-47d3-ac38-c6f3145af65c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘roberta_toddler’: File exists\n"
     ]
    }
   ],
   "source": [
    "!mkdir roberta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "n8iF33BCGzCQ"
   },
   "outputs": [],
   "source": [
    "training_args = IPUTrainingArguments(output_dir=\"roberta\",\n",
    "                                     do_train=False,\n",
    "                                     do_eval=True,\n",
    "                                     per_device_train_batch_size=4,\n",
    "                                     per_device_eval_batch_size=4,\n",
    "                                     gradient_accumulation_steps=128,\n",
    "                                     learning_rate=1e-4,\n",
    "                                     num_train_epochs=5,\n",
    "                                     # fp32=True,\n",
    "                                     # logging_steps=25,\n",
    "                                     dataloader_num_workers=52,\n",
    "                                     weight_decay=0.1,\n",
    "                                     dataloader_drop_last=True,\n",
    "                                     prediction_loss_only=True,\n",
    "                                     # resume_from_checkpoint=\"./roberta/checkpoint-142803/\",\n",
    "                                     # pad_on_batch_axis=True,\n",
    "                                     # # pod_type=pod_type,\n",
    "                                     # pad_on_batch_axis = True,\n",
    "                                     save_strategy=\"epoch\",\n",
    "                                     # lamb=True,\n",
    "                                     auto_loss_scaling=True,\n",
    "                                     overwrite_output_dir=True,\n",
    "                                     # report_to=\"none\",\n",
    "                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "y7GJ5qKPGzCQ",
    "outputId": "81b45cc9-b561-4521-9dbe-ef3b184e9e0f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/optimum/graphcore/ipu_configuration.py:140: UserWarning: The \"sharded_execution_for_inference\" parameter is deprecated, sharded execution is always used during inference\n",
      "  warnings.warn(\n",
      "Overriding IPU config: gradient_accumulation_steps=128,auto_loss_scaling=True\n",
      "-------------------- Device Allocation --------------------\n",
      "Embedding  --> IPU 0\n",
      "Encoder 0  --> IPU 1\n",
      "Encoder 1  --> IPU 1\n",
      "Encoder 2  --> IPU 1\n",
      "Encoder 3  --> IPU 2\n",
      "LM Head    --> IPU 0\n",
      "-----------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "trainer1 = IPUTrainer(\n",
    "    model=model,\n",
    "    ipu_config=ipu_config,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_datasets['train'],\n",
    "    data_collator=data_collator,\n",
    "    eval_dataset=tokenized_datasets_dev['train'],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eCAbTESZGzCQ",
    "outputId": "ceb67b18-8a3d-49b9-c407-744b6e507af9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'attention_mask'],\n",
       "        num_rows: 1015494\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "96e4a036173f4d5d86713cb9121bd014"
     ]
    },
    "id": "G_Jzuy6-GzCQ",
    "outputId": "25fbf90e-c4fa-467c-e620-2f26a3705b90"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Compiling Model...\n",
      "Graph compilation: 100%|██████████| 100/100 [02:28<00:00]\n",
      "Compiled/Loaded model in 192.6328711300157 secs\n",
      "***** Running training *****\n",
      "  Num examples = 1015494\n",
      "  Num Epochs = 5\n",
      "  Instantaneous batch size per device = 1\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 64\n",
      "  Total optimization steps = 79335\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96e4a036173f4d5d86713cb9121bd014",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/79335 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.7565, 'learning_rate': 9.936976113947186e-05, 'epoch': 0.03}\n",
      "{'loss': 2.7209, 'learning_rate': 9.873952227894372e-05, 'epoch': 0.06}\n",
      "{'loss': 2.7275, 'learning_rate': 9.810928341841559e-05, 'epoch': 0.09}\n",
      "{'loss': 2.6101, 'learning_rate': 9.747904455788744e-05, 'epoch': 0.13}\n",
      "{'loss': 2.7243, 'learning_rate': 9.68488056973593e-05, 'epoch': 0.16}\n",
      "{'loss': 2.7339, 'learning_rate': 9.621856683683115e-05, 'epoch': 0.19}\n",
      "{'loss': 2.7137, 'learning_rate': 9.558832797630302e-05, 'epoch': 0.22}\n",
      "{'loss': 2.5916, 'learning_rate': 9.495808911577488e-05, 'epoch': 0.25}\n",
      "{'loss': 2.6203, 'learning_rate': 9.432785025524674e-05, 'epoch': 0.28}\n",
      "{'loss': 2.5641, 'learning_rate': 9.369761139471859e-05, 'epoch': 0.32}\n",
      "{'loss': 2.4523, 'learning_rate': 9.306737253419046e-05, 'epoch': 0.35}\n",
      "{'loss': 2.5155, 'learning_rate': 9.243713367366232e-05, 'epoch': 0.38}\n",
      "{'loss': 2.3701, 'learning_rate': 9.180689481313419e-05, 'epoch': 0.41}\n",
      "{'loss': 2.6104, 'learning_rate': 9.117665595260604e-05, 'epoch': 0.44}\n",
      "{'loss': 2.3118, 'learning_rate': 9.05464170920779e-05, 'epoch': 0.47}\n",
      "{'loss': 2.5063, 'learning_rate': 8.991617823154977e-05, 'epoch': 0.5}\n",
      "{'loss': 2.4941, 'learning_rate': 8.928593937102163e-05, 'epoch': 0.54}\n",
      "{'loss': 2.4519, 'learning_rate': 8.865570051049348e-05, 'epoch': 0.57}\n",
      "{'loss': 2.3964, 'learning_rate': 8.802546164996535e-05, 'epoch': 0.6}\n",
      "{'loss': 2.5229, 'learning_rate': 8.739522278943721e-05, 'epoch': 0.63}\n",
      "{'loss': 2.3828, 'learning_rate': 8.676498392890906e-05, 'epoch': 0.66}\n",
      "{'loss': 2.5381, 'learning_rate': 8.613474506838092e-05, 'epoch': 0.69}\n",
      "{'loss': 2.2383, 'learning_rate': 8.550450620785278e-05, 'epoch': 0.72}\n",
      "{'loss': 2.1456, 'learning_rate': 8.487426734732464e-05, 'epoch': 0.76}\n",
      "{'loss': 2.5176, 'learning_rate': 8.42440284867965e-05, 'epoch': 0.79}\n",
      "{'loss': 2.4587, 'learning_rate': 8.361378962626836e-05, 'epoch': 0.82}\n",
      "{'loss': 2.4446, 'learning_rate': 8.298355076574021e-05, 'epoch': 0.85}\n",
      "{'loss': 2.1336, 'learning_rate': 8.235331190521208e-05, 'epoch': 0.88}\n",
      "{'loss': 2.2984, 'learning_rate': 8.172307304468394e-05, 'epoch': 0.91}\n",
      "{'loss': 2.3352, 'learning_rate': 8.10928341841558e-05, 'epoch': 0.95}\n",
      "{'loss': 2.0509, 'learning_rate': 8.046259532362765e-05, 'epoch': 0.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to roberta_5/checkpoint-15867\n",
      "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\n",
      "-------------------- Device Allocation --------------------\n",
      "Embedding  --> IPU 0\n",
      "Encoder 0  --> IPU 1\n",
      "Encoder 1  --> IPU 1\n",
      "Encoder 2  --> IPU 1\n",
      "Encoder 3  --> IPU 2\n",
      "LM Head    --> IPU 0\n",
      "-----------------------------------------------------------\n",
      "Configuration saved in roberta_5/checkpoint-15867/ipu_config.json\n",
      "/usr/local/lib/python3.8/dist-packages/optimum/graphcore/ipu_configuration.py:140: UserWarning: The \"sharded_execution_for_inference\" parameter is deprecated, sharded execution is always used during inference\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.1439, 'learning_rate': 7.983235646309952e-05, 'epoch': 1.01}\n",
      "{'loss': 2.2581, 'learning_rate': 7.920211760257138e-05, 'epoch': 1.04}\n",
      "{'loss': 2.1662, 'learning_rate': 7.857187874204323e-05, 'epoch': 1.07}\n",
      "{'loss': 2.1791, 'learning_rate': 7.794163988151509e-05, 'epoch': 1.1}\n",
      "{'loss': 2.184, 'learning_rate': 7.731140102098696e-05, 'epoch': 1.13}\n",
      "{'loss': 2.0694, 'learning_rate': 7.668116216045881e-05, 'epoch': 1.17}\n",
      "{'loss': 2.3353, 'learning_rate': 7.605092329993067e-05, 'epoch': 1.2}\n",
      "{'loss': 2.2295, 'learning_rate': 7.542068443940254e-05, 'epoch': 1.23}\n",
      "{'loss': 2.1829, 'learning_rate': 7.47904455788744e-05, 'epoch': 1.26}\n",
      "{'loss': 2.125, 'learning_rate': 7.416020671834627e-05, 'epoch': 1.29}\n",
      "{'loss': 2.0657, 'learning_rate': 7.352996785781812e-05, 'epoch': 1.32}\n",
      "{'loss': 2.4687, 'learning_rate': 7.289972899728998e-05, 'epoch': 1.36}\n",
      "{'loss': 2.226, 'learning_rate': 7.226949013676185e-05, 'epoch': 1.39}\n",
      "{'loss': 2.3861, 'learning_rate': 7.16392512762337e-05, 'epoch': 1.42}\n",
      "{'loss': 2.1069, 'learning_rate': 7.100901241570556e-05, 'epoch': 1.45}\n",
      "{'loss': 2.3141, 'learning_rate': 7.037877355517742e-05, 'epoch': 1.48}\n",
      "{'loss': 2.1017, 'learning_rate': 6.974853469464927e-05, 'epoch': 1.51}\n",
      "{'loss': 2.1594, 'learning_rate': 6.911829583412114e-05, 'epoch': 1.54}\n",
      "{'loss': 1.8987, 'learning_rate': 6.8488056973593e-05, 'epoch': 1.58}\n",
      "{'loss': 2.1233, 'learning_rate': 6.785781811306485e-05, 'epoch': 1.61}\n",
      "{'loss': 2.37, 'learning_rate': 6.722757925253671e-05, 'epoch': 1.64}\n",
      "{'loss': 2.0824, 'learning_rate': 6.659734039200858e-05, 'epoch': 1.67}\n",
      "{'loss': 2.1713, 'learning_rate': 6.596710153148043e-05, 'epoch': 1.7}\n",
      "{'loss': 2.0865, 'learning_rate': 6.533686267095229e-05, 'epoch': 1.73}\n",
      "{'loss': 2.161, 'learning_rate': 6.470662381042415e-05, 'epoch': 1.76}\n",
      "{'loss': 2.2659, 'learning_rate': 6.407638494989602e-05, 'epoch': 1.8}\n",
      "{'loss': 2.0936, 'learning_rate': 6.344614608936787e-05, 'epoch': 1.83}\n",
      "{'loss': 2.247, 'learning_rate': 6.281590722883973e-05, 'epoch': 1.86}\n",
      "{'loss': 1.9906, 'learning_rate': 6.218566836831158e-05, 'epoch': 1.89}\n",
      "{'loss': 2.1091, 'learning_rate': 6.155542950778345e-05, 'epoch': 1.92}\n",
      "{'loss': 2.0023, 'learning_rate': 6.092519064725532e-05, 'epoch': 1.95}\n",
      "{'loss': 2.0195, 'learning_rate': 6.029495178672717e-05, 'epoch': 1.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to roberta_5/checkpoint-31734\n",
      "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\n",
      "-------------------- Device Allocation --------------------\n",
      "Embedding  --> IPU 0\n",
      "Encoder 0  --> IPU 1\n",
      "Encoder 1  --> IPU 1\n",
      "Encoder 2  --> IPU 1\n",
      "Encoder 3  --> IPU 2\n",
      "LM Head    --> IPU 0\n",
      "-----------------------------------------------------------\n",
      "Configuration saved in roberta_5/checkpoint-31734/ipu_config.json\n",
      "/usr/local/lib/python3.8/dist-packages/optimum/graphcore/ipu_configuration.py:140: UserWarning: The \"sharded_execution_for_inference\" parameter is deprecated, sharded execution is always used during inference\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.1167, 'learning_rate': 5.966471292619903e-05, 'epoch': 2.02}\n",
      "{'loss': 2.2098, 'learning_rate': 5.90344740656709e-05, 'epoch': 2.05}\n",
      "{'loss': 1.9882, 'learning_rate': 5.8404235205142755e-05, 'epoch': 2.08}\n",
      "{'loss': 2.2684, 'learning_rate': 5.777399634461461e-05, 'epoch': 2.11}\n",
      "{'loss': 2.0564, 'learning_rate': 5.714375748408647e-05, 'epoch': 2.14}\n",
      "{'loss': 2.1961, 'learning_rate': 5.651351862355832e-05, 'epoch': 2.17}\n",
      "{'loss': 2.0022, 'learning_rate': 5.588327976303019e-05, 'epoch': 2.21}\n",
      "{'loss': 2.0053, 'learning_rate': 5.525304090250205e-05, 'epoch': 2.24}\n",
      "{'loss': 2.1097, 'learning_rate': 5.462280204197391e-05, 'epoch': 2.27}\n",
      "{'loss': 2.0055, 'learning_rate': 5.399256318144577e-05, 'epoch': 2.3}\n",
      "{'loss': 2.0297, 'learning_rate': 5.336232432091763e-05, 'epoch': 2.33}\n",
      "{'loss': 1.922, 'learning_rate': 5.2732085460389493e-05, 'epoch': 2.36}\n",
      "{'loss': 2.0509, 'learning_rate': 5.210184659986135e-05, 'epoch': 2.39}\n",
      "{'loss': 1.9075, 'learning_rate': 5.1471607739333206e-05, 'epoch': 2.43}\n",
      "{'loss': 2.0719, 'learning_rate': 5.0841368878805075e-05, 'epoch': 2.46}\n",
      "{'loss': 2.0836, 'learning_rate': 5.021113001827693e-05, 'epoch': 2.49}\n",
      "{'loss': 2.0642, 'learning_rate': 4.958089115774879e-05, 'epoch': 2.52}\n",
      "{'loss': 2.0716, 'learning_rate': 4.895065229722065e-05, 'epoch': 2.55}\n",
      "{'loss': 2.1809, 'learning_rate': 4.8320413436692506e-05, 'epoch': 2.58}\n",
      "{'loss': 1.9687, 'learning_rate': 4.769017457616437e-05, 'epoch': 2.62}\n",
      "{'loss': 2.1657, 'learning_rate': 4.7059935715636225e-05, 'epoch': 2.65}\n",
      "{'loss': 1.9955, 'learning_rate': 4.642969685510809e-05, 'epoch': 2.68}\n",
      "{'loss': 2.1529, 'learning_rate': 4.579945799457995e-05, 'epoch': 2.71}\n",
      "{'loss': 2.1502, 'learning_rate': 4.516921913405181e-05, 'epoch': 2.74}\n",
      "{'loss': 2.0869, 'learning_rate': 4.453898027352367e-05, 'epoch': 2.77}\n",
      "{'loss': 2.0933, 'learning_rate': 4.390874141299553e-05, 'epoch': 2.8}\n",
      "{'loss': 2.0465, 'learning_rate': 4.327850255246739e-05, 'epoch': 2.84}\n",
      "{'loss': 1.8772, 'learning_rate': 4.264826369193925e-05, 'epoch': 2.87}\n",
      "{'loss': 2.0451, 'learning_rate': 4.201802483141111e-05, 'epoch': 2.9}\n",
      "{'loss': 2.0784, 'learning_rate': 4.138778597088296e-05, 'epoch': 2.93}\n",
      "{'loss': 2.1077, 'learning_rate': 4.0757547110354826e-05, 'epoch': 2.96}\n",
      "{'loss': 2.0725, 'learning_rate': 4.012730824982668e-05, 'epoch': 2.99}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to roberta_5/checkpoint-47601\n",
      "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\n",
      "-------------------- Device Allocation --------------------\n",
      "Embedding  --> IPU 0\n",
      "Encoder 0  --> IPU 1\n",
      "Encoder 1  --> IPU 1\n",
      "Encoder 2  --> IPU 1\n",
      "Encoder 3  --> IPU 2\n",
      "LM Head    --> IPU 0\n",
      "-----------------------------------------------------------\n",
      "Configuration saved in roberta_5/checkpoint-47601/ipu_config.json\n",
      "/usr/local/lib/python3.8/dist-packages/optimum/graphcore/ipu_configuration.py:140: UserWarning: The \"sharded_execution_for_inference\" parameter is deprecated, sharded execution is always used during inference\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.0404, 'learning_rate': 3.9497069389298545e-05, 'epoch': 3.03}\n",
      "{'loss': 2.0917, 'learning_rate': 3.88668305287704e-05, 'epoch': 3.06}\n",
      "{'loss': 1.8917, 'learning_rate': 3.8236591668242264e-05, 'epoch': 3.09}\n",
      "{'loss': 1.9019, 'learning_rate': 3.760635280771413e-05, 'epoch': 3.12}\n",
      "{'loss': 2.0237, 'learning_rate': 3.697611394718599e-05, 'epoch': 3.15}\n",
      "{'loss': 1.9214, 'learning_rate': 3.6345875086657846e-05, 'epoch': 3.18}\n",
      "{'loss': 2.0049, 'learning_rate': 3.571563622612971e-05, 'epoch': 3.21}\n",
      "{'loss': 2.0326, 'learning_rate': 3.5085397365601565e-05, 'epoch': 3.25}\n",
      "{'loss': 2.0372, 'learning_rate': 3.445515850507343e-05, 'epoch': 3.28}\n",
      "{'loss': 1.9194, 'learning_rate': 3.3824919644545283e-05, 'epoch': 3.31}\n",
      "{'loss': 1.7669, 'learning_rate': 3.3194680784017146e-05, 'epoch': 3.34}\n",
      "{'loss': 1.8913, 'learning_rate': 3.2564441923489e-05, 'epoch': 3.37}\n",
      "{'loss': 2.037, 'learning_rate': 3.1934203062960865e-05, 'epoch': 3.4}\n",
      "{'loss': 2.0412, 'learning_rate': 3.130396420243272e-05, 'epoch': 3.43}\n",
      "{'loss': 2.0392, 'learning_rate': 3.0673725341904584e-05, 'epoch': 3.47}\n",
      "{'loss': 1.9034, 'learning_rate': 3.0043486481376443e-05, 'epoch': 3.5}\n",
      "{'loss': 1.938, 'learning_rate': 2.9413247620848306e-05, 'epoch': 3.53}\n",
      "{'loss': 2.0209, 'learning_rate': 2.8783008760320162e-05, 'epoch': 3.56}\n",
      "{'loss': 1.7796, 'learning_rate': 2.8152769899792025e-05, 'epoch': 3.59}\n",
      "{'loss': 1.8907, 'learning_rate': 2.752253103926388e-05, 'epoch': 3.62}\n",
      "{'loss': 1.8614, 'learning_rate': 2.689229217873574e-05, 'epoch': 3.66}\n",
      "{'loss': 2.0136, 'learning_rate': 2.6262053318207603e-05, 'epoch': 3.69}\n",
      "{'loss': 1.9108, 'learning_rate': 2.563181445767946e-05, 'epoch': 3.72}\n",
      "{'loss': 2.2307, 'learning_rate': 2.5001575597151322e-05, 'epoch': 3.75}\n",
      "{'loss': 1.9708, 'learning_rate': 2.4371336736623182e-05, 'epoch': 3.78}\n",
      "{'loss': 2.0375, 'learning_rate': 2.374109787609504e-05, 'epoch': 3.81}\n",
      "{'loss': 2.0121, 'learning_rate': 2.31108590155669e-05, 'epoch': 3.84}\n",
      "{'loss': 1.8148, 'learning_rate': 2.2480620155038764e-05, 'epoch': 3.88}\n",
      "{'loss': 2.0779, 'learning_rate': 2.1850381294510623e-05, 'epoch': 3.91}\n",
      "{'loss': 2.0311, 'learning_rate': 2.1220142433982482e-05, 'epoch': 3.94}\n",
      "{'loss': 2.0233, 'learning_rate': 2.058990357345434e-05, 'epoch': 3.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to roberta_5/checkpoint-63468\n",
      "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\n",
      "-------------------- Device Allocation --------------------\n",
      "Embedding  --> IPU 0\n",
      "Encoder 0  --> IPU 1\n",
      "Encoder 1  --> IPU 1\n",
      "Encoder 2  --> IPU 1\n",
      "Encoder 3  --> IPU 2\n",
      "LM Head    --> IPU 0\n",
      "-----------------------------------------------------------\n",
      "Configuration saved in roberta_5/checkpoint-63468/ipu_config.json\n",
      "/usr/local/lib/python3.8/dist-packages/optimum/graphcore/ipu_configuration.py:140: UserWarning: The \"sharded_execution_for_inference\" parameter is deprecated, sharded execution is always used during inference\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.0347, 'learning_rate': 1.9959664712926198e-05, 'epoch': 4.0}\n",
      "{'loss': 2.0159, 'learning_rate': 1.9329425852398057e-05, 'epoch': 4.03}\n",
      "{'loss': 1.8853, 'learning_rate': 1.869918699186992e-05, 'epoch': 4.07}\n",
      "{'loss': 1.7826, 'learning_rate': 1.806894813134178e-05, 'epoch': 4.1}\n",
      "{'loss': 1.8884, 'learning_rate': 1.743870927081364e-05, 'epoch': 4.13}\n",
      "{'loss': 1.9225, 'learning_rate': 1.68084704102855e-05, 'epoch': 4.16}\n",
      "{'loss': 2.2672, 'learning_rate': 1.6178231549757358e-05, 'epoch': 4.19}\n",
      "{'loss': 2.0474, 'learning_rate': 1.5547992689229217e-05, 'epoch': 4.22}\n",
      "{'loss': 1.7927, 'learning_rate': 1.4917753828701079e-05, 'epoch': 4.25}\n",
      "{'loss': 1.9031, 'learning_rate': 1.4287514968172938e-05, 'epoch': 4.29}\n",
      "{'loss': 2.0115, 'learning_rate': 1.3657276107644797e-05, 'epoch': 4.32}\n",
      "{'loss': 1.9207, 'learning_rate': 1.3027037247116659e-05, 'epoch': 4.35}\n",
      "{'loss': 1.9431, 'learning_rate': 1.2396798386588518e-05, 'epoch': 4.38}\n",
      "{'loss': 2.0146, 'learning_rate': 1.1766559526060377e-05, 'epoch': 4.41}\n",
      "{'loss': 1.7806, 'learning_rate': 1.1136320665532239e-05, 'epoch': 4.44}\n",
      "{'loss': 1.7852, 'learning_rate': 1.0506081805004098e-05, 'epoch': 4.47}\n",
      "{'loss': 2.1235, 'learning_rate': 9.875842944475956e-06, 'epoch': 4.51}\n",
      "{'loss': 1.7803, 'learning_rate': 9.245604083947817e-06, 'epoch': 4.54}\n",
      "{'loss': 1.7271, 'learning_rate': 8.615365223419676e-06, 'epoch': 4.57}\n",
      "{'loss': 1.9863, 'learning_rate': 7.985126362891536e-06, 'epoch': 4.6}\n",
      "{'loss': 2.0077, 'learning_rate': 7.354887502363396e-06, 'epoch': 4.63}\n",
      "{'loss': 2.1312, 'learning_rate': 6.7246486418352555e-06, 'epoch': 4.66}\n",
      "{'loss': 1.7127, 'learning_rate': 6.094409781307116e-06, 'epoch': 4.7}\n",
      "{'loss': 1.7341, 'learning_rate': 5.464170920778976e-06, 'epoch': 4.73}\n",
      "{'loss': 1.8594, 'learning_rate': 4.8339320602508355e-06, 'epoch': 4.76}\n",
      "{'loss': 1.7508, 'learning_rate': 4.203693199722695e-06, 'epoch': 4.79}\n",
      "{'loss': 2.0275, 'learning_rate': 3.573454339194555e-06, 'epoch': 4.82}\n",
      "{'loss': 1.7411, 'learning_rate': 2.9432154786664147e-06, 'epoch': 4.85}\n",
      "{'loss': 1.8448, 'learning_rate': 2.3129766181382745e-06, 'epoch': 4.88}\n",
      "{'loss': 1.8822, 'learning_rate': 1.6827377576101342e-06, 'epoch': 4.92}\n",
      "{'loss': 1.8026, 'learning_rate': 1.052498897081994e-06, 'epoch': 4.95}\n",
      "{'loss': 2.047, 'learning_rate': 4.22260036553854e-07, 'epoch': 4.98}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to roberta_5/checkpoint-79335\n",
      "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\n",
      "-------------------- Device Allocation --------------------\n",
      "Embedding  --> IPU 0\n",
      "Encoder 0  --> IPU 1\n",
      "Encoder 1  --> IPU 1\n",
      "Encoder 2  --> IPU 1\n",
      "Encoder 3  --> IPU 2\n",
      "LM Head    --> IPU 0\n",
      "-----------------------------------------------------------\n",
      "Configuration saved in roberta_5/checkpoint-79335/ipu_config.json\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "/usr/local/lib/python3.8/dist-packages/optimum/graphcore/ipu_configuration.py:140: UserWarning: The \"sharded_execution_for_inference\" parameter is deprecated, sharded execution is always used during inference\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_runtime': 13996.3023, 'train_samples_per_second': 362.77, 'train_steps_per_second': 5.668, 'train_loss': 2.1173090456263393, 'epoch': 5.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=79335, training_loss=2.1173090456263393, metrics={'train_runtime': 13996.3023, 'train_samples_per_second': 362.77, 'train_steps_per_second': 5.668, 'train_loss': 2.1173090456263393, 'epoch': 5.0})"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer1.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KSl9i7TuGzCQ"
   },
   "outputs": [],
   "source": [
    "trainer1.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vzxizGKiGzCR"
   },
   "outputs": [],
   "source": [
    "trainer1.save_model(\"roberta\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "J1sKoxpnGzCR"
   },
   "outputs": [],
   "source": [
    "tokenizer.save_pretrained(\"roberta\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "80846776069f4b88b8a54c3d1421f643"
     ]
    },
    "id": "B1rxUaICGzCR",
    "outputId": "f461500c-1337-462b-c54e-9982d138a5b4"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80846776069f4b88b8a54c3d1421f643",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "O14b18u8GzCR"
   },
   "outputs": [],
   "source": [
    "model = AutoModelForMaskedLM.from_pretrained(\"roberta\")\n",
    "tokenizer=AutoTokenizer.from_pretrained(\"roberta\")\n",
    "model.push_to_hub(\"roberta\")\n",
    "tokenizer.push_to_hub(\"roberta\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "D5DhzkrYGzCR",
    "outputId": "5067b4f8-d2f4-47a8-b716-ae15d1cdbc07"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to roberta_bebeshka\n",
      "-------------------- Device Allocation --------------------\n",
      "Embedding  --> IPU 0\n",
      "Encoder 0  --> IPU 0\n",
      "Encoder 1  --> IPU 1\n",
      "Encoder 2  --> IPU 2\n",
      "Encoder 3  --> IPU 3\n",
      "LM Head    --> IPU 0\n",
      "-----------------------------------------------------------\n",
      "Configuration saved in roberta_bebeshka/ipu_config.json\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'score': 0.05490488559007645,\n",
       "  'token': 228,\n",
       "  'token_str': ' it',\n",
       "  'sequence': 'the sun is it.'},\n",
       " {'score': 0.02775665931403637,\n",
       "  'token': 708,\n",
       "  'token_str': ' great',\n",
       "  'sequence': 'the sun is great.'},\n",
       " {'score': 0.024511652067303658,\n",
       "  'token': 415,\n",
       "  'token_str': ' here',\n",
       "  'sequence': 'the sun is here.'},\n",
       " {'score': 0.020964989438652992,\n",
       "  'token': 437,\n",
       "  'token_str': ' right',\n",
       "  'sequence': 'the sun is right.'},\n",
       " {'score': 0.019934361800551414,\n",
       "  'token': 503,\n",
       "  'token_str': ' good',\n",
       "  'sequence': 'the sun is good.'}]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask = pipeline(\n",
    "    \"fill-mask\",\n",
    "    model=\"roberta\",\n",
    "    tokenizer=\"roberta\"\n",
    ")\n",
    "\n",
    "# The sun <mask>.\n",
    "# => great\n",
    "\n",
    "fill_mask(\"The sun is <mask>.\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "oK7PPVm2XBgr"
   ],
   "gpuType": "A100",
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}