GPT2_adapter_generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "da6b6463-e3e4-4def-9414-431f366b0799",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from transformers import GPT2Tokenizer, TrainingArguments, set_seed\n",
    "from utils.helper import load_model_recursive\n",
    "from ppcm_models.pytorch_pretrained_bert.modeling_adapter import GPT2LMHeadModel, GPT2Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dcafe7cc-e742-4270-bcf8-4b62e6b8ff49",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2dbf6887-a0c8-4ff8-b3e0-d0ff9b659bb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataArguments():\n",
    "    def __init__(self):\n",
    "        self.dataset_path = '/home/bryan/datasets/bookcorpusopen/bookcorpusopen_chunked.arrow'\n",
    "        self.bookcorpusopen_story_column_name = 'chunk'\n",
    "        self.preprocessing_num_workers = 8\n",
    "        self.genre='Romance'\n",
    "        self.adapter_id=1\n",
    "        self.match_up_to_n_genres=3\n",
    "        self.sample_row=None\n",
    "        \n",
    "class ModelArguments():\n",
    "    def __init__(self):\n",
    "        self.model_size = 'small'\n",
    "        self.load_checkpoint_adapter = \"\"\n",
    "        self.max_seq_len=512\n",
    "        # self.lr = 2e-4 #, help=\"Learning rate\")\n",
    "\n",
    "class TrainingArguments(TrainingArguments):\n",
    "    def __init__(self):\n",
    "        self.output_dir = \"./save\"\n",
    "        self.eval_accumulation_steps = None\n",
    "        \n",
    "model_args = ModelArguments()\n",
    "data_args = DataArguments()\n",
    "training_args = TrainingArguments()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5dd69338-00ae-4e00-b2ac-1d19e981b9de",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_args.model_path = f'ppcm_models/dialoGPT/small/'\n",
    "config = GPT2Config.from_json_file(os.path.join(model_args.model_path, 'config.json'))\n",
    "\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(model_args.model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "608d510c-136b-483a-99e7-511548100827",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading finetuned model from ./save/GPT2small_adapterid0_genreThriller_matched3_sampleNone_maxseqlen512_bs8_lr0.001_10.0epoch_wd0.0_ws0/pytorch_model.bin\n",
      "Hello, my dog is cute. I'm not sure if he's a good dog, but he is. He's very cute and I think he likes me.\"\"I'm sorry, I don't know. But I can't help it. It's just that I've been so busy. You know, the last time I was here, you were at the hospital. And I didn't want to be there. So I went to the doctor.\"The man in the black suit was<|endoftext|> \n",
      "\n",
      "Loading finetuned model from ./save/GPT2small_adapterid0_genreMystery_matched3_sampleNone_maxseqlen512_bs8_lr0.001_10.0epoch_wd0.0_ws0/pytorch_model.bin\n",
      "Hello, my dog is cute. I'm not sure if he's a good dog, but he is a very good one. He's very smart and very strong. And he loves me.\"I'm sorry, I don't know. It's just that I was thinking about it. But I didn't want to get into that. You know, it's not like I want you to know anything. Just that you know what I mean.\"\"You're not going to be able<|endoftext|> \n",
      "\n",
      "Loading finetuned model from ./save/GPT2small_adapterid0_genreHistorical_matched3_sampleNone_maxseqlen512_bs8_lr0.001_10.0epoch_wd0.0_ws0/pytorch_model.bin\n",
      "Hello, my dog is cute. I'm not sure how much I love him. He's a good boy, but he's not very good at keeping his own thoughts to himself. And I don't know how he can be so good with his dogs.\"\"I'm sorry, I was just trying to get you to stop. It's just that I've been in a really bad mood lately. You know, the last time I saw you, you were in the hospital. That's<|endoftext|> \n",
      "\n",
      "CPU times: user 3h 36min 52s, sys: 1min 38s, total: 3h 38min 31s\n",
      "Wall time: 44min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model_run_names = ['GPT2small_adapterid0_genreThriller_matched3_sampleNone_maxseqlen512_bs8_lr0.001_10.0epoch_wd0.0_ws0'\n",
    "                  ,'GPT2small_adapterid0_genreMystery_matched3_sampleNone_maxseqlen512_bs8_lr0.001_10.0epoch_wd0.0_ws0'\n",
    "                  ,'GPT2small_adapterid0_genreHistorical_matched3_sampleNone_maxseqlen512_bs8_lr0.001_10.0epoch_wd0.0_ws0']\n",
    "\n",
    "for i, model_run_name in enumerate(model_run_names):\n",
    "\n",
    "    path = './save/'+model_run_name+'/pytorch_model.bin'\n",
    "    model = load_model_recursive(GPT2LMHeadModel(config), path, model_args, verbose=True)\n",
    "\n",
    "    # # use this to generate outputs\n",
    "    # inputs = tokenizer(\"Hello, my dog is cute\", return_tensors=\"pt\")\n",
    "    # outputs = model(inputs['input_ids'], task_id=[0])\n",
    "    # # or\n",
    "    # outputs = model.transformer(inputs['input_ids'], task_id=0)\n",
    "\n",
    "    length = 100\n",
    "    text = \"Hello, my dog is cute\"\n",
    "    generated = tokenizer.encode(text)\n",
    "    context = torch.tensor([generated])\n",
    "    \n",
    "    # Following https://github.com/huggingface/transformers/blob/main/src/transformers/generation_utils.py#L1096-L1100\n",
    "    torch.manual_seed(14045)\n",
    "    generation = model.generate(inputs=context,\n",
    "                               num_beams=3, \n",
    "                               length_penalty=3, \n",
    "                               early_stopping=1, \n",
    "                               num_beam_groups=3, \n",
    "                               do_sample=False, \n",
    "                               num_return_sequences=2, \n",
    "                               bos_token_id=50256,\n",
    "                               eos_token_id=50256,\n",
    "                               pad_token_id=50256,\n",
    "                               output_scores=True,\n",
    "                               output_attentions=True,\n",
    "                               output_hidden_states=True,\n",
    "                               return_dict_in_generate=True,\n",
    "                               repetition_penalty=1.1,\n",
    "                               min_length = 0,\n",
    "                               max_length = length,\n",
    "                               no_repeat_ngram_size=2,\n",
    "                               encoder_no_repeat_ngram_size=False,\n",
    "                               bad_words_ids=[[100]], # tokenizer.decode(100)\n",
    "                               diversity_penalty=0.2,\n",
    "                               forced_bos_token_id=50256,\n",
    "                               forced_eos_token_id=50256,\n",
    "                               remove_invalid_values=True,\n",
    "                               exponential_decay_length_penalty=[1.0, 1.2])\n",
    "\n",
    "    print(tokenizer.decode(generation[0][0]), '\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pix2story",
   "language": "python",
   "name": "pix2story"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}