generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f4f97e56-39e5-41a4-b987-532d1174667f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-12 13:42:22.338386: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-12-12 13:42:22.523359: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2022-12-12 13:42:23.132207: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2022-12-12 13:42:23.132318: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2022-12-12 13:42:23.132326: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "from tqdm.auto import tqdm\n",
    "from transformers import pipeline, set_seed\n",
    "import torch\n",
    "from rdkit import Chem, RDLogger\n",
    "lg = RDLogger.logger()\n",
    "lg.setLevel(RDLogger.CRITICAL)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74d243d7-bb92-4497-a536-0dc8cfb297fb",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Setup tokenization"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1860f87-31c2-4b89-ad2e-d79bb4b9e317",
   "metadata": {},
   "source": [
    "## Atom-based tokenization & BPE class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "aa3091f2-bd65-43c6-bedc-9ac86eafdd08",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import logging\n",
    "import os\n",
    "import re\n",
    "import codecs\n",
    "import unicodedata\n",
    "from typing import List, Optional\n",
    "from transformers import PreTrainedTokenizer\n",
    "from SmilesPE.tokenizer import SPE_Tokenizer\n",
    "\n",
    "def load_vocab(vocab_file):\n",
    "    \"\"\"Loads a vocabulary file into a dictionary.\"\"\"\n",
    "    vocab = collections.OrderedDict()\n",
    "    with open(vocab_file, \"r\", encoding=\"utf-8\") as reader:\n",
    "        tokens = reader.readlines()\n",
    "    for index, token in enumerate(tokens):\n",
    "        token = token.rstrip(\"\\n\")\n",
    "        vocab[token] = index\n",
    "    return vocab\n",
    "\n",
    "class Atomwise_Tokenizer(object):\n",
    "    \"\"\"Run atom-level SMILES tokenization\"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\" Constructs a atom-level Tokenizer.\n",
    "        \"\"\"\n",
    "        self.regex_pattern = r\"(\\[[^\\]]+]|Br?|Cl?|N|O|S|P|F|I|b|c|n|o|s|p|\\(|\\)|\\.|=|#|-|\\+|\\\\|\\/|:|~|@|\\?|>>?|\\*|\\$|\\%[0-9]{2}|[0-9])\"\n",
    "        self.regex = re.compile(self.regex_pattern)\n",
    "\n",
    "    def tokenize(self, text):\n",
    "        \"\"\" Basic Tokenization of a SMILES.\n",
    "        \"\"\"\n",
    "        tokens = [token for token in self.regex.findall(text)]\n",
    "        return tokens\n",
    "\n",
    "class SMILES_SPE_Tokenizer(PreTrainedTokenizer):\n",
    "    r\"\"\"\n",
    "    Constructs a SMILES tokenizer. Based on SMILES Pair Encoding (https://github.com/XinhaoLi74/SmilesPE).\n",
    "    This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the methods. Users\n",
    "    should refer to the superclass for more information regarding methods.\n",
    "    Args:\n",
    "        vocab_file (:obj:`string`):\n",
    "            File containing the vocabulary.\n",
    "        spe_file (:obj:`string`):\n",
    "            File containing the trained SMILES Pair Encoding vocabulary.\n",
    "        unk_token (:obj:`string`, `optional`, defaults to \"[UNK]\"):\n",
    "            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this\n",
    "            token instead.\n",
    "        sep_token (:obj:`string`, `optional`, defaults to \"[SEP]\"):\n",
    "            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences\n",
    "            for sequence classification or for a text and a question for question answering.\n",
    "            It is also used as the last token of a sequence built with special tokens.\n",
    "        pad_token (:obj:`string`, `optional`, defaults to \"[PAD]\"):\n",
    "            The token used for padding, for example when batching sequences of different lengths.\n",
    "        cls_token (:obj:`string`, `optional`, defaults to \"[CLS]\"):\n",
    "            The classifier token which is used when doing sequence classification (classification of the whole\n",
    "            sequence instead of per-token classification). It is the first token of the sequence when built with\n",
    "            special tokens.\n",
    "        mask_token (:obj:`string`, `optional`, defaults to \"[MASK]\"):\n",
    "            The token used for masking values. This is the token used when training this model with masked language\n",
    "            modeling. This is the token which the model will try to predict.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        vocab_file,\n",
    "        spe_file,\n",
    "        unk_token=\"[UNK]\",\n",
    "        bos_token=\"[BOS]\",\n",
    "        eos_token=\"[EOS]\",\n",
    "        # sep_token=\"[SEP]\",\n",
    "        pad_token=\"[PAD]\",\n",
    "        # cls_token=\"[CLS]\",\n",
    "        # mask_token=\"[MASK]\",\n",
    "        **kwargs\n",
    "    ):\n",
    "        super().__init__(\n",
    "            unk_token=unk_token,\n",
    "            bos_token=bos_token,\n",
    "            eos_token=eos_token,\n",
    "            pad_token=pad_token,\n",
    "            # cls_token=cls_token,\n",
    "            # mask_token=mask_token,\n",
    "            **kwargs,\n",
    "        )\n",
    "\n",
    "        if not os.path.isfile(vocab_file):\n",
    "            raise ValueError(\n",
    "                \"Can't find a vocabulary file at path '{}'.\".format(vocab_file)\n",
    "            )\n",
    "        if not os.path.isfile(spe_file):\n",
    "            raise ValueError(\n",
    "                \"Can't find a SPE vocabulary file at path '{}'.\".format(spe_file)\n",
    "            )\n",
    "        self.vocab = load_vocab(vocab_file)\n",
    "        self.spe_vocab = codecs.open(spe_file)\n",
    "        self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])\n",
    "        self.spe_tokenizer = SPE_Tokenizer(self.spe_vocab)\n",
    "\n",
    "    @property\n",
    "    def vocab_size(self):\n",
    "        return len(self.vocab)\n",
    "\n",
    "    def get_vocab(self):\n",
    "        return dict(self.vocab, **self.added_tokens_encoder)\n",
    "\n",
    "    def _tokenize(self, text):\n",
    "        return self.spe_tokenizer.tokenize(text).split(' ')\n",
    "\n",
    "    def _convert_token_to_id(self, token):\n",
    "        \"\"\" Converts a token (str) in an id using the vocab. \"\"\"\n",
    "        return self.vocab.get(token, self.vocab.get(self.unk_token))\n",
    "\n",
    "    def _convert_id_to_token(self, index):\n",
    "        \"\"\"Converts an index (integer) in a token (str) using the vocab.\"\"\"\n",
    "        return self.ids_to_tokens.get(index, self.unk_token)\n",
    "\n",
    "    def convert_tokens_to_string(self, tokens):\n",
    "        \"\"\" Converts a sequence of tokens (string) in a single string. \"\"\"\n",
    "        out_string = \"\".join(tokens).replace(\" ##\", \"\").strip()\n",
    "        return out_string\n",
    "\n",
    "    def build_inputs_with_special_tokens(\n",
    "        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None\n",
    "    ) -> List[int]:\n",
    "        \"\"\"\n",
    "        Build model inputs from a sequence or a pair of sequence for sequence classification tasks\n",
    "        by concatenating and adding special tokens.\n",
    "        A BERT sequence has the following format:\n",
    "        - single sequence: ``[CLS] X [SEP]``\n",
    "        - pair of sequences: ``[CLS] A [SEP] B [SEP]``\n",
    "        Args:\n",
    "            token_ids_0 (:obj:`List[int]`):\n",
    "                List of IDs to which the special tokens will be added\n",
    "            token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):\n",
    "                Optional second list of IDs for sequence pairs.\n",
    "        Returns:\n",
    "            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.\n",
    "        \"\"\"\n",
    "        bos = [self.bos_token_id]\n",
    "        eos = [self.eos_token_id]\n",
    "        output = bos + token_ids_0 + eos\n",
    "        if token_ids_1 is None:\n",
    "            return output\n",
    "        return output + [bos] + token_ids_1 + [eos]\n",
    "\n",
    "    def get_special_tokens_mask(\n",
    "        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False\n",
    "    ) -> List[int]:\n",
    "        \"\"\"\n",
    "        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding\n",
    "        special tokens using the tokenizer ``prepare_for_model`` method.\n",
    "        Args:\n",
    "            token_ids_0 (:obj:`List[int]`):\n",
    "                List of ids.\n",
    "            token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):\n",
    "                Optional second list of IDs for sequence pairs.\n",
    "            already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):\n",
    "                Set to True if the token list is already formatted with special tokens for the model\n",
    "        Returns:\n",
    "            :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.\n",
    "        \"\"\"\n",
    "\n",
    "        if already_has_special_tokens:\n",
    "            if token_ids_1 is not None:\n",
    "                raise ValueError(\n",
    "                    \"You should not supply a second sequence if the provided sequence of \"\n",
    "                    \"ids is already formated with special tokens for the model.\"\n",
    "                )\n",
    "            return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))\n",
    "\n",
    "        if token_ids_1 is not None:\n",
    "            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]\n",
    "        return [1] + ([0] * len(token_ids_0)) + [1]\n",
    "\n",
    "\n",
    "    def save_vocabulary(self, vocab_path):\n",
    "        \"\"\"\n",
    "        Save the sentencepiece vocabulary (copy original file) and special tokens file to a directory.\n",
    "        Args:\n",
    "            vocab_path (:obj:`str`):\n",
    "                The directory in which to save the vocabulary.\n",
    "        Returns:\n",
    "            :obj:`Tuple(str)`: Paths to the files saved.\n",
    "        \"\"\"\n",
    "        index = 0\n",
    "        if os.path.isdir(vocab_path):\n",
    "            vocab_file = os.path.join(vocab_path, VOCAB_FILES_NAMES[\"vocab_file\"])\n",
    "        else:\n",
    "            vocab_file = vocab_path\n",
    "        with open(vocab_file, \"w\", encoding=\"utf-8\") as writer:\n",
    "            for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):\n",
    "                if index != token_index:\n",
    "                    logger.warning(\n",
    "                        \"Saving vocabulary to {}: vocabulary indices are not consecutive.\"\n",
    "                        \" Please check that the vocabulary is not corrupted!\".format(vocab_file)\n",
    "                    )\n",
    "                    index = token_index\n",
    "                writer.write(token + \"\\n\")\n",
    "                index += 1\n",
    "        return (vocab_file,)\n",
    "\n",
    "tokenizer = SMILES_SPE_Tokenizer(\n",
    "    vocab_file='./tokenization/SPE/vocab_spe.txt', \n",
    "    spe_file= './tokenization/SPE/SPE_ChEMBL.txt'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e26e517-4362-4cfb-9c1b-d2854d0f89ab",
   "metadata": {},
   "source": [
    "# Setup inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1f8f1f6e-cdcc-440a-9d9f-14bde6482df4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline, TextGenerationPipeline\n",
    "\n",
    "class MyPipeline(TextGenerationPipeline):\n",
    "    def postprocess(self, model_outputs):\n",
    "        output_tokens = np.squeeze(model_outputs['generated_sequence']).numpy()\n",
    "        output = tokenizer.decode(output_tokens, skip_special_tokens=True)\n",
    "        return {'generated': output}\n",
    "        \n",
    "model_path = './save/irak4-20221212-1339/checkpoint-84/'\n",
    "config_path = f\"{model_path}config.json\"\n",
    "device = 1 # -1 stands for using CPU\n",
    "\n",
    "generators = []\n",
    "\n",
    "# Top-K Sampling\n",
    "# ref: https://huggingface.co/blog/how-to-generate#top-k-sampling\n",
    "generators.append(\n",
    "    pipeline(\n",
    "    'text-generation', \n",
    "    model=model_path,\n",
    "    config=config_path,\n",
    "    tokenizer=tokenizer,\n",
    "    device=device,\n",
    "    max_length=100,\n",
    "    pad_token_id=0,\n",
    "    do_sample=True, \n",
    "    top_k=50, \n",
    "    early_stopping=True,\n",
    "    pipeline_class=MyPipeline\n",
    "))\n",
    "\n",
    "# Top-P Sampling\n",
    "# ref: https://huggingface.co/blog/how-to-generate#top-p-nucleus-sampling\n",
    "generators.append(\n",
    "    pipeline(\n",
    "    'text-generation', \n",
    "    model=model_path,\n",
    "    config=config_path,\n",
    "    tokenizer=tokenizer,\n",
    "    device=device,\n",
    "    max_length=100,\n",
    "    pad_token_id=0,\n",
    "    do_sample=True, \n",
    "    top_p=.92, \n",
    "    top_k=0, \n",
    "    early_stopping=True,\n",
    "    pipeline_class=MyPipeline\n",
    "))\n",
    "\n",
    "# Contrastive Search\n",
    "# https://huggingface.co/blog/introducing-csearch#contrastive_objective\n",
    "generators.append(\n",
    "    pipeline(\n",
    "    'text-generation', \n",
    "    model=model_path,\n",
    "    config=config_path,\n",
    "    tokenizer=tokenizer,\n",
    "    device=device,\n",
    "    max_length=100,\n",
    "    pad_token_id=0,\n",
    "    do_sample=True, \n",
    "    top_k=4, \n",
    "    penalty_alpha=0.6,\n",
    "    early_stopping=True,\n",
    "    pipeline_class=MyPipeline\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8f84937-ed09-4568-9bfb-e5a071ca5037",
   "metadata": {},
   "source": [
    "# Data generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6625cc5a-3981-4189-9e73-e1e710b856c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "gen_num = 30_000\n",
    "prompt = '[BOS]'\n",
    "\n",
    "def data(gen_num):\n",
    "    count = gen_num\n",
    "    while count:\n",
    "        count-=1\n",
    "        yield prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a44cfce8-6d42-43a3-be07-f52741704b10",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "694101ab99d14d07809b1b5e43a5f694",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generator#:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c63e1c43f40f44aab5eeeec62f5e8cd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "cmpd#:   0%|          | 0/30000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Non-parallel generation version\n",
    "results = []\n",
    "for generator in tqdm(generators, position=0, desc='generator#', leave=False):\n",
    "    results.append([\n",
    "        cmpd['generated'] \n",
    "        for cmpd in tqdm(\n",
    "            generator(data(gen_num)), \n",
    "            position=1, desc='cmpd#', leave=False, total=gen_num\n",
    "        )\n",
    "    ])\n",
    "    # print(out['generated'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d0ee4b7-5409-4a31-85bc-6e89383b1f78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10823113bcbd4c0d909575c61b89214d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-13 16:12:37.497063: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2022-12-13 16:12:38.793585: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2022-12-13 16:12:38.793829: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2022-12-13 16:12:38.804347: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      "2022-12-13 16:12:45.565748: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "  0%|          | 0/30000 [00:00<?, ?it/s]2022-12-13 16:12:46.273878: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2022-12-13 16:12:46.273976: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2022-12-13 16:12:46.273985: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      "2022-12-13 16:12:47.046938: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2022-12-13 16:12:48.342238: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2022-12-13 16:12:48.342372: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2022-12-13 16:12:48.342383: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
      " 93%|█████████▎| 27835/30000 [1:36:51<08:01,  4.50it/s]  "
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Parallel generation version\n",
    "# https://joblib.readthedocs.io/en/latest/parallel.html\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "def batch_generation(generator, data, gen_num):\n",
    "    return [\n",
    "        cmpd['generated'] \n",
    "        for cmpd in tqdm(generator(data(gen_num)), total=gen_num)\n",
    "    ]\n",
    "\n",
    "results = Parallel(n_jobs=128)(\n",
    "    delayed(batch_generation)\n",
    "    (generator, data, gen_num) \n",
    "    for generator in tqdm(generators)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "13a491fe-e243-43ae-87f9-fc64bc6dda79",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(results, index=None).T\n",
    "df.rename({0: 'top-k', 1: 'top-p', 2: 'cons'}, axis=1, inplace=True)\n",
    "df.to_csv('./generation/irak4_spe/checkpoint-70-30_000_gen_sample.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b8807a2-ae23-4428-af39-698ef6211efe",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "HF GPT-2",
   "language": "python",
   "name": "gpt"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}