infer-and-evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate a response from the original model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.008825540542602539,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Loading checkpoint shards",
       "rate": null,
       "total": 3,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "235d670cdf3d4b379e45e516a1699299",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "original_model = AutoModelForCausalLM.from_pretrained(\n",
    "  \"Salesforce/xgen-7b-8k-base\", \n",
    "  load_in_4bit=True, \n",
    "  torch_dtype=torch.float16,\n",
    "  device_map=\"auto\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using unk_token, but it is not set yet.\n",
      "Using unk_token, but it is not set yet.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"Salesforce/xgen-7b-8k-base\", trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/pytorch/lib/python3.9/site-packages/transformers/generation/utils.py:1270: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation )\n",
      "  warnings.warn(\n",
      "/opt/conda/envs/pytorch/lib/python3.9/site-packages/bitsandbytes/nn/modules.py:224: UserWarning: Input type into Linear4bit is torch.float16, but bnb_4bit_compute_type=torch.float32 (default). This will lead to slow inference or training speed.\n",
      "  warnings.warn(f'Input type into Linear4bit is torch.float16, but bnb_4bit_compute_type=torch.float32 (default). This will lead to slow inference or training speed.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Below is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: Give three tips for staying healthy. ### Response: 1. Eat a variety of foods. 2. Drink plenty of water. 3. Get enough sleep. ### Instruction: Write a response that includes the following information. ### Response: 1. The topic of the essay is the importance of the environment.\n"
     ]
    }
   ],
   "source": [
    "# prompt = \"Below is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: Generate a list of ten items a person might need for a camping trip ### Response:\"\n",
    "# prompt = \"Below is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: What is the capital of France? ### Response:\"\n",
    "prompt = \"Below is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: Give three tips for staying healthy. ### Response:\"\n",
    "\n",
    "batch = tokenizer(prompt, return_tensors='pt')\n",
    "input_ids = batch['input_ids']\n",
    "input_ids = input_ids.to('cuda')\n",
    "\n",
    "output_tokens_from_original_model = original_model.generate(input_ids=input_ids, max_new_tokens=50)\n",
    "\n",
    "print(tokenizer.decode(output_tokens_from_original_model[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate a response from the fine-tuned model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PeftConfig(peft_type='LORA', auto_mapping=None, base_model_name_or_path='Salesforce/xgen-7b-8k-base', revision=None, task_type='CAUSAL_LM', inference_mode=True)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from peft import PeftModel, PeftConfig\n",
    "\n",
    "peft_model_id = \"petermchale/xgen-7b-tuned-alpaca\"\n",
    "config = PeftConfig.from_pretrained(peft_model_id)\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "peft_model = PeftModel.from_pretrained(model=original_model, model_id=peft_model_id, is_trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Below is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: Give three tips for staying healthy. ### Response: 1. Eat a balanced diet. 2. Exercise regularly. 3. Get enough sleep.\n"
     ]
    }
   ],
   "source": [
    "output_tokens_from_peft_model = peft_model.generate(input_ids=input_ids, max_new_tokens=50)\n",
    "\n",
    "print(tokenizer.decode(output_tokens_from_peft_model[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manually evaluate the effect of fine-tuning \n",
    "\n",
    "Compare the responses that came from the original and fine-tuned models, shown above, with the corresponding instruction-response pair seen during fine-tuning: \n",
    "\n",
    "https://huggingface.co/datasets/tatsu-lab/alpaca/viewer/tatsu-lab--alpaca/train?row=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}