gpt2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "879b3b75-a974-4a87-bd12-d1216dfce30c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.cuda.empty_cache()\n",
    "from torch.utils.data import Dataset, random_split\n",
    "from transformers import (\n",
    "    GPT2LMHeadModel, \n",
    "    GPT2Tokenizer, \n",
    "    TrainingArguments, \n",
    "    Trainer, \n",
    "    DataCollatorForLanguageModeling,\n",
    "    TrainerCallback\n",
    ")\n",
    "from peft import get_peft_model, LoraConfig, TaskType\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "21007720-aa66-47ec-85a7-0e6b90cc665b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set seeds for reproducibility\n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "\n",
    "# Set the seed\n",
    "set_seed(42)  # You can change this to any integer value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a720f265-63b8-40a9-8e2f-bb54d3081d35",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextDataset(Dataset):\n",
    "    def __init__(self, file_path, tokenizer, block_size):\n",
    "        with open(file_path, 'r', encoding='utf-8') as f:\n",
    "            text = f.read()\n",
    "        tokenized_text = tokenizer.encode(text)\n",
    "        self.examples = [tokenized_text[i:i + block_size] for i in range(0, len(tokenized_text) - block_size + 1, block_size)]\n",
    "        print(f\"Loaded {len(self.examples)} examples.\")\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.examples)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return torch.tensor(self.examples[i], dtype=torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7b96b75f-1538-4e25-a390-ca8592bcf457",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomTrainer(Trainer):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super().__init__(*args, **kwargs)\n",
    "        self.train_loss = []\n",
    "        self.val_loss = []\n",
    "\n",
    "    def training_step(self, model, inputs):\n",
    "        loss = super().training_step(model, inputs)\n",
    "        self.train_loss.append(loss.item())\n",
    "        return loss\n",
    "    \n",
    "    def evaluation_loop(self, *args, **kwargs):\n",
    "        output = super().evaluation_loop(*args, **kwargs)\n",
    "        self.val_loss.append(output.metrics['eval_loss'])\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0673f108-e4b7-4366-9ca3-389084183c61",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SaveEpochCallback(TrainerCallback):\n",
    "    def __init__(self, save_epochs, output_dir, tokenizer):\n",
    "        self.save_epochs = save_epochs\n",
    "        self.output_dir = output_dir\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "    def on_epoch_end(self, args, state, control, **kwargs):\n",
    "        epoch = int(state.epoch)\n",
    "        print(f\"Callback triggered for epoch {epoch}\")\n",
    "        if epoch % self.save_epochs == 0:\n",
    "            checkpoint_dir = os.path.join(self.output_dir, f\"checkpoint-epoch-{epoch}\")\n",
    "            print(f\"Attempting to save checkpoint for epoch {epoch} to {checkpoint_dir}\")\n",
    "            if 'model' in kwargs:\n",
    "                kwargs['model'].save_pretrained(checkpoint_dir)\n",
    "                self.tokenizer.save_pretrained(checkpoint_dir)\n",
    "                print(f\"Saved checkpoint for epoch {epoch} to {checkpoint_dir}\")\n",
    "            else:\n",
    "                print(\"Model not found in kwargs, unable to save checkpoint\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b7c239d1-f21f-403e-ae58-3050e96a61bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_text(model, tokenizer, max_length=100):\n",
    "    input_ids = torch.tensor([[tokenizer.bos_token_id]]).to(model.device)\n",
    "    attention_mask = torch.ones_like(input_ids).to(model.device)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        output = model.generate(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            max_length=max_length,\n",
    "            num_return_sequences=1,\n",
    "            no_repeat_ngram_size=2,\n",
    "            do_sample=True,\n",
    "            top_k=50,\n",
    "            top_p=0.95,\n",
    "            temperature=0.75,\n",
    "            pad_token_id=tokenizer.eos_token_id,\n",
    "            bos_token_id=tokenizer.bos_token_id,\n",
    "            eos_token_id=tokenizer.eos_token_id\n",
    "        )\n",
    "    \n",
    "    return tokenizer.decode(output[0], skip_special_tokens=True).strip()\n",
    "\n",
    "def generate_text_with_prompt(model, tokenizer, prompt, max_length=200):\n",
    "    input_ids = tokenizer.encode(prompt, return_tensors='pt').to(model.device)\n",
    "    attention_mask = torch.ones_like(input_ids).to(model.device)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        output = model.generate(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            max_length=max_length + len(input_ids[0]),\n",
    "            num_return_sequences=1,\n",
    "            no_repeat_ngram_size=2,\n",
    "            do_sample=True,\n",
    "            top_k=50,\n",
    "            top_p=0.95,\n",
    "            temperature=0.75,\n",
    "            pad_token_id=tokenizer.eos_token_id,\n",
    "            bos_token_id=tokenizer.bos_token_id,\n",
    "            eos_token_id=tokenizer.eos_token_id\n",
    "        )\n",
    "    \n",
    "    return tokenizer.decode(output[0], skip_special_tokens=True).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "50ff08fd-1be5-405b-bb02-f2bc774d96d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/lora/layer.py:1091: UserWarning: fan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"Using device: {device}\")\n",
    "\n",
    "model_name = \"gpt2\"\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(model_name)\n",
    "model = GPT2LMHeadModel.from_pretrained(model_name)\n",
    "\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "model.config.pad_token_id = tokenizer.eos_token_id\n",
    "\n",
    "model.to(device)\n",
    "\n",
    "peft_config = LoraConfig(\n",
    "    task_type=TaskType.CAUSAL_LM,\n",
    "    inference_mode=False,\n",
    "    r=8,\n",
    "    lora_alpha=32,\n",
    "    lora_dropout=0.1,\n",
    "    target_modules=[\"c_attn\", \"c_proj\"]\n",
    ")\n",
    "\n",
    "model = get_peft_model(model, peft_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5a46842a-a34d-4ebd-9449-ed0021706ea3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1022810 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 7990 examples.\n"
     ]
    }
   ],
   "source": [
    "file_path = \"trump_speeches_combined_processed.txt\"\n",
    "block_size = 128\n",
    "full_dataset = TextDataset(file_path, tokenizer, block_size)\n",
    "\n",
    "train_size = int(0.9 * len(full_dataset))\n",
    "val_size = len(full_dataset) - train_size\n",
    "train_dataset, val_dataset = random_split(full_dataset, [train_size, val_size])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9d22b124-0b62-4b0a-8562-3e0139ec767d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Training setup\n",
    "total_epochs = 1000\n",
    "save_epochs = 10\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./results\",\n",
    "    num_train_epochs=total_epochs,\n",
    "    per_device_train_batch_size=4,\n",
    "    per_device_eval_batch_size=4,\n",
    "    gradient_accumulation_steps=8,\n",
    "    learning_rate=0.001,\n",
    "    warmup_steps=100,\n",
    "    logging_dir='./logs',\n",
    "    logging_steps=100,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"no\",  # We'll handle saving with our custom callback\n",
    "    fp16=True,\n",
    "    optim=\"adamw_torch\",\n",
    "    load_best_model_at_end=False,  # We're not using the default saving strategy\n",
    ")\n",
    "\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)\n",
    "\n",
    "# Pass the tokenizer to the callback\n",
    "save_callback = SaveEpochCallback(save_epochs, training_args.output_dir, tokenizer)\n",
    "\n",
    "trainer = CustomTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    data_collator=data_collator,\n",
    "    callbacks=[save_callback],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bddb706c-e3de-4932-8718-5b2c9756a0ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model\n",
    "print(\"Starting training...\")\n",
    "trainer.train()\n",
    "print(\"Training completed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "476318d1-efdf-4855-bd73-75da1959c887",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved final checkpoint to ./results/final-checkpoint\n"
     ]
    }
   ],
   "source": [
    "# Manually save the final state if needed\n",
    "final_checkpoint_dir = os.path.join(training_args.output_dir, \"final-checkpoint\")\n",
    "trainer.save_model(final_checkpoint_dir)\n",
    "tokenizer.save_pretrained(final_checkpoint_dir)\n",
    "print(f\"Saved final checkpoint to {final_checkpoint_dir}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "7dbdd89d-0f46-4900-b863-f8e39335390c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generated text after epoch 100:\n",
      "The U.S. military has been accused of deploying hundreds of troops to Iraq to bolster its offensive on Islamic State of Iraq and Syria (ISIS) militants in the east of the country.\n",
      "\n",
      "The White House released a statement on Monday that said the U.S.-led coalition was deploying more than 100 troops and a helicopter into northern Iraq, but denied it had deployed any troops. \"We have no information that any of this has happened or that there is any indication that the forces are\n",
      "\n",
      "Generated text with prompt after epoch 100:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans? If you're an individual, why do you want to see some form of healthcare? Do you have an issue with how you approach healthcare in the United States? Share your comments below.\n",
      "\n",
      "Do you prefer to do something with your life instead of relying on government for healthcare, or do your family members have it? Let us know your experiences in comments!\n",
      ".\n",
      "\n",
      "Generated text after epoch 200:\n",
      "\"The only way I know how to stop this is to see if we can stop it,\" she said. \"I have worked with the president and the Democrats on this issue and they've always supported me. I have no doubt they would do it.\"\n",
      "\n",
      "But with her campaign facing a major setback after the party announced it would not endorse Trump, Clinton did not say how she would handle the fallout.\n",
      ".@realDonaldTrump has said that his policies on immigration and trade are \"not what\n",
      "\n",
      "Generated text with prompt after epoch 200:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans?\n",
      "\n",
      "Source: Pew Research Center\n",
      ",\n",
      "\n",
      "Generated text after epoch 300:\n",
      "- Posted by: Jazmin on Tuesday, December 3, 2012\n",
      "\n",
      "This is just the tip of the iceberg.\n",
      ". I have a theory on how the internet works. It's just a bunch of people who want to know what the real internet is and what it's like to have an Internet that they can access and interact with. The internet goes crazy and they want someone to make an account, someone who is smart enough to understand what they are doing. And that person wants\n",
      "\n",
      "Generated text with prompt after epoch 300:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans? Let us know in the comments below!\n",
      "\n",
      "Image Credit: iStockphoto\n",
      "\n",
      "Generated text after epoch 400:\n",
      "\"If we go down to the level of 'what ifs'? What if we were given the right to say 'no' for the military?\"\n",
      "\n",
      "The minister added: \"The government will act to protect the interests of the people, and it will not be responsible for what happened.\"\n",
      "\n",
      "Generated text with prompt after epoch 400:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans? How would you like to see the government reform Medicare in the future?\n",
      "\n",
      "Click here for more from The Nation's Ezra Klein.\n",
      " (1) The government must give Americans the option of Medicare, or it can't afford to.\n",
      "\n",
      "Generated text after epoch 500:\n",
      "Crowdsourcing is a promising medium for helping companies create quality products that are quickly adopted by consumers. Crowdsourced products will not only help keep you motivated and your product in the hands of your customers, they will also help you build the community, which is why you may be interested in volunteering to help us design and develop the products you want.\n",
      "\n",
      "As a volunteer, we have the ability to provide a unique service for you and help with your projects in a variety of ways,\n",
      "\n",
      "Generated text with prompt after epoch 500:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans?\n",
      "\n",
      "The Affordable Care Act has a couple of big, important points. First of all, this is not a mandate for health care. It's a universal health insurance program that would cover all people. And it would not cover a lot of people who don't have health coverage. The second point is that we should not have a big government, big federal government that is putting money into our health system. We should have all of the government of our choosing.\n",
      " (Laughter.)\n",
      ", as well as having a single-payer system, which would be a massive boon to the American economy. If you look at the recent polls, Americans are split on whether the Affordable Plan is the best thing that could happen for the country. Forty-five percent of those surveyed say it's the worst thing. They are also somewhat divided on the idea that it is, and not very many say that they would prefer that. This is an important point, because it means that, in fact, the\n",
      "\n",
      "Generated text after epoch 600:\n",
      "A former U.S. Army sergeant who is facing charges that he sexually assaulted a woman and sexually harassed her on the job was arrested Friday.\n",
      "\n",
      "David B. Miller, 37, was indicted on one count of sexual assault and one counts of second-degree sexual abuse on March 26, according to a statement from the U..S Attorney's Office in Chicago. He is accused of assaulting the woman, a 21-year-old woman who lived in a house in the 100 block\n",
      "\n",
      "Generated text with prompt after epoch 600:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans? Let us know in the comments!\n",
      "\n",
      "Image Credit: Flickr user @bwg_salt\n",
      "\n",
      "Generated text after epoch 700:\n",
      "We are very proud to announce the signing of Kip Tharoor, a 21-year-old of West Bengal. He is a student in a public and private engineering and computer engineering school. A project manager in the area of project management, he is an active member of the Kanchal project team, which is currently looking at a new construction.\n",
      "\n",
      "Kip is very passionate about the work of technology. KIP is passionate in his work with technology, especially in robotics,\n",
      "\n",
      "Generated text with prompt after epoch 700:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans?\n",
      "\n",
      "If you're in the middle of the night and you want to watch an entire news program, there's no better time than right now to get your news straight to your phone. Your call to 911 is the best way to answer that call.\n",
      "...\n",
      " (This article originally appeared at The Daily Beast)\n",
      "\n",
      "Generated text after epoch 800:\n",
      "I love that you have such a nice collection of photos. It's a great way to celebrate a hobby that I think will forever be remembered as the greatest game ever.\n",
      "\n",
      "To see more of my photos of the game on Flickr click here\n",
      "\n",
      "Generated text with prompt after epoch 800:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans? Share them in the comments below!\n",
      "\n",
      "Generated text after epoch 900:\n",
      "I believe that if we all work together to make the world a better place, then we can all live a happier and healthier life together.\n",
      "\n",
      "I hope that I've proven my point by writing this blog post. If you're reading this post and still believe in the importance of good health and good job performance, please join me on this journey. I'm not saying that you should skip this step because you already know what it takes to have a healthy job. Instead, I want you\n",
      "\n",
      "Generated text with prompt after epoch 900:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans?\n",
      "\n",
      "H/T:\n",
      "-E-mail me at hkills@hillaryclinton.com\n",
      "Final Validation Perplexity: 15.28\n",
      "\n",
      "Final generated text:\n",
      "i don't know what to do.\n",
      "i don’t know how to make a statement. i just don “t want to put a name on this.'i“ve been with you for a long time. so we”re going to get the greatest possible. we have a great president that understands our borders, we know our laws. you can—t say that. okay? so, this is something i want. it‖s really a\n",
      "\n",
      "Final generated text with prompt:\n",
      "Prompt: What are your thoughts on free healthcare for all Americans?\n",
      "Response: What are your thoughts on free healthcare for all Americans? i don’t want to see the debate, i want the debates. i can‖t watch the speeches. you know, when i did it, it was like a joke. now, people are saying, \"oh, you can run. \" and you have to do it. we․re going to win it and we are going have a lot of fun.\n",
      "and you”ve got to remember this. this is an election, and i‣ve never had a debate where we had two people that were very dishonest and very smart and that was a very, very stupid debate. and by the way, one of the smartest people in the world got up and said, – but i think it‘ was good. so i would love to run because i will be honest with you. but we have people. it has people on the left that believe that are very dangerous and they think we don't want you anymore. they want guns are the worst\n"
     ]
    }
   ],
   "source": [
    "prompt = \"What are your thoughts on free healthcare for all Americans?\"\n",
    "interval = 100\n",
    "for epoch in range(interval, total_epochs + 1, interval):\n",
    "    checkpoint_dir = f\"./results/checkpoint-epoch-{epoch}\"\n",
    "    if os.path.exists(checkpoint_dir):\n",
    "        loaded_model = GPT2LMHeadModel.from_pretrained(checkpoint_dir)\n",
    "        loaded_model = get_peft_model(loaded_model, peft_config)\n",
    "        loaded_model.to(device)\n",
    "        \n",
    "        print(f\"\\nGenerated text after epoch {epoch}:\")\n",
    "        print(generate_text(loaded_model, tokenizer))\n",
    "\n",
    "        print(f\"\\nGenerated text with prompt after epoch {epoch}:\")\n",
    "        print(f\"Prompt: {prompt}\")\n",
    "        print(f\"Response: {generate_text_with_prompt(loaded_model, tokenizer, prompt)}\")\n",
    "\n",
    "# Calculate final perplexity on validation set\n",
    "val_loss = trainer.evaluate()['eval_loss']\n",
    "val_perplexity = math.exp(val_loss)\n",
    "print(f\"Final Validation Perplexity: {val_perplexity:.2f}\")\n",
    "\n",
    "# Generate text with final model\n",
    "print(\"\\nFinal generated text:\")\n",
    "print(generate_text(model, tokenizer))\n",
    "\n",
    "print(\"\\nFinal generated text with prompt:\")\n",
    "print(f\"Prompt: {prompt}\")\n",
    "print(f\"Response: {generate_text_with_prompt(model, tokenizer, prompt)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d519b847-df0a-4bb2-a44d-eefb7d32d4b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final model saved to ./fine_tuned_gpt2_final\n"
     ]
    }
   ],
   "source": [
    "final_output_dir = \"./fine_tuned_gpt2_final\"\n",
    "trainer.save_model(final_output_dir)\n",
    "tokenizer.save_pretrained(final_output_dir)\n",
    "print(f\"Final model saved to {final_output_dir}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e21c82b8-5e9f-4038-92c7-189c4aee0e2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Validation Perplexity: 15.28\n",
      "\n",
      "Final generated text:\n",
      "i am going to make america rich again. i'm going make our country safe again, and i will make it great again for all americans. thank you, everybody.\n",
      "thank you. in less than a year, the obama-clinton trade deal, signed by hillary clinton, will produce more than 60,000 jobs, save $150 billion, reduce our deficit and reduce the price of prescription drugs  all in one go. we've created 30, 000 new\n",
      "\n",
      "Final generated text with prompt:\n",
      "Prompt: What are your thoughts on China?\n",
      "Response: What are your thoughts on China? i'm the only one who's going to be able to answer that question. and i hope you'll agree with me. i think it was a very fair question, but i have to tell you, i really am. now, you know, this is a different kind of country, and you have a president who is very tough on illegal immigration, on trade deals, very, much, that have absolutely destroyed our jobs.\n",
      "you have people that are very dishonest and very bad, people who are terrible. they're on food stamps, they have terrible people. people are so much better off. but he's a good guy. so when i see him, he was on a television show, \"donald trump is the best in the world. \" i said, what's your opinion? \"he's the worst. he is. you don't know who he isn't? he doesn't have the greatest. do you remember jesse? and by the way, jessie ke\n"
     ]
    }
   ],
   "source": [
    "prompt = \"What are your thoughts on China?\"\n",
    "\n",
    "# Calculate final perplexity on validation set\n",
    "val_loss = trainer.evaluate()['eval_loss']\n",
    "val_perplexity = math.exp(val_loss)\n",
    "print(f\"Final Validation Perplexity: {val_perplexity:.2f}\")\n",
    "\n",
    "# Generate text with final model\n",
    "print(\"\\nFinal generated text:\")\n",
    "print(generate_text(model, tokenizer))\n",
    "\n",
    "print(\"\\nFinal generated text with prompt:\")\n",
    "print(f\"Prompt: {prompt}\")\n",
    "print(f\"Response: {generate_text_with_prompt(model, tokenizer, prompt)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5ef4a2e-b389-4325-ba83-3df4b4a60dc5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}