Experiment 0 ALBERT Base Pre-training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow_gpu'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-d9ee477249e2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_logger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetLevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'WARNING'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_verbosity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mERROR\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow_gpu\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow_gpu'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "os.getcwd()\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
    "import tensorflow as tf\n",
    "tf.get_logger().setLevel('WARNING')\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-a652879afe3d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_gpu_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_built_with_cuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "assert tf.test.is_gpu_available()\n",
    "assert tf.test.is_built_with_cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "print(tf.test.is_gpu_available())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device mapping:\n",
      "/job:localhost/replica:0/task:0/device:XLA_CPU:0 -> device: XLA_CPU device\n",
      "/job:localhost/replica:0/task:0/device:XLA_GPU:0 -> device: XLA_GPU device\n",
      "\n",
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 2557779634678029685\n",
      ", name: \"/device:XLA_CPU:0\"\n",
      "device_type: \"XLA_CPU\"\n",
      "memory_limit: 17179869184\n",
      "locality {\n",
      "}\n",
      "incarnation: 15731681702213545963\n",
      "physical_device_desc: \"device: XLA_CPU device\"\n",
      ", name: \"/device:XLA_GPU:0\"\n",
      "device_type: \"XLA_GPU\"\n",
      "memory_limit: 17179869184\n",
      "locality {\n",
      "}\n",
      "incarnation: 10241760155805528355\n",
      "physical_device_desc: \"device: XLA_GPU device\"\n",
      "]\n",
      "Num GPUs Available:  1\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "from tensorflow.python.client import device_lib \n",
    "print(device_lib.list_local_devices())\n",
    "print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('XLA_GPU')))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file = COURPUS_DIR = '/home/bam/Desktop/ECHR Publication Experiments/DS/corpus.txt'\n",
    "BERT_BASE_DIR =\"/home/bam/Desktop/ECHR Publication Experiments/albert-master\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "!test -d albert || git clone https://github.com/google-research/albert albert\n",
    "if not 'albert' in sys.path:\n",
    "  sys.path += ['albert']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loading sentence piece model\n",
      "I0618 12:54:48.158270 140510101608256 tokenization.py:188] loading sentence piece model\n",
      "INFO:tensorflow:*** Reading from input files ***\n",
      "I0618 12:54:48.205111 140510101608256 create_pretraining_data.py:631] *** Reading from input files ***\n",
      "INFO:tensorflow:  /home/bam/Desktop/ECHR Publication Experiments/DS/corpus.txt\n",
      "I0618 12:54:48.205215 140510101608256 create_pretraining_data.py:633]   /home/bam/Desktop/ECHR Publication Experiments/DS/corpus.txt\n",
      "^C\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/bam/anaconda3/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n",
      "    \"__main__\", mod_spec)\n",
      "  File \"/home/bam/anaconda3/lib/python3.7/runpy.py\", line 85, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"/home/bam/Desktop/ECHR Publication Experiments/albert/create_pretraining_data.py\", line 656, in <module>\n",
      "    tf.app.run()\n",
      "  File \"/home/bam/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/platform/app.py\", line 40, in run\n",
      "    _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)\n",
      "  File \"/home/bam/anaconda3/lib/python3.7/site-packages/absl/app.py\", line 299, in run\n",
      "    _run_main(main, args)\n",
      "  File \"/home/bam/anaconda3/lib/python3.7/site-packages/absl/app.py\", line 250, in _run_main\n",
      "    sys.exit(main(argv))\n",
      "  File \"/home/bam/Desktop/ECHR Publication Experiments/albert/create_pretraining_data.py\", line 639, in main\n",
      "    rng)\n",
      "  File \"/home/bam/Desktop/ECHR Publication Experiments/albert/create_pretraining_data.py\", line 246, in create_training_instances\n",
      "    tokens = tokenizer.tokenize(line)\n",
      "  File \"/home/bam/Desktop/ECHR Publication Experiments/albert/tokenization.py\", line 226, in tokenize\n",
      "    split_tokens = encode_pieces(self.sp_model, text, return_unicode=False)\n",
      "  File \"/home/bam/Desktop/ECHR Publication Experiments/albert/tokenization.py\", line 66, in encode_pieces\n",
      "    piece = printable_text(piece)\n",
      "  File \"/home/bam/Desktop/ECHR Publication Experiments/albert/tokenization.py\", line 124, in printable_text\n",
      "    if isinstance(text, str):\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "!python -m albert.create_pretraining_data\\\n",
    "  --input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\DS\\corpus.txt\"\\\n",
    "  --output_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\further\\20dupetf_examples.tfrecord\"\\\n",
    "  --vocab_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\30k-clean.vocab\"\\\n",
    "  --spm_model_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\30k-clean.model\"\\\n",
    "  --do_lower_case=True\\\n",
    "  --max_seq_length=512\\\n",
    "  --max_predictions_per_seq=77\\\n",
    "  --masked_lm_prob=0.15\\\n",
    "  --random_seed=12345\\\n",
    "  --dupe_factor=20\\\n",
    "  --do_whole_word_mask=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m albert.run_pretraining \\\n",
    "  --input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\further\\tf_examples.tfrecord\" \\\n",
    "  --output_dir=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\further\" \\\n",
    "  --init_checkpoint= \"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\model.ckpt-best\"\\\n",
    "  --albert_config_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\albert_config.json\"\\\n",
    "  --do_train True \\\n",
    "  --do_eval True \\\n",
    "  --train_batch_size=6 \\\n",
    "  --eval_batch_size=6 \\\n",
    "  --max_seq_length=512 \\\n",
    "  --max_predictions_per_seq=77 \\\n",
    "  --optimizer=\"lamb\" \\\n",
    "  --learning_rate=.00176 \\\n",
    "  --num_train_steps=10000 \\\n",
    "  --num_warmup_steps=10 \\\n",
    "  --save_checkpoints_steps=500\\\n",
    "  --use_gpu=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "python -m albert.run_pretraining --input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\further\\20dupetf_examples.tfrecord\" --output_dir=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\further\" --init_checkpoint= \"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\model.ckpt-best\" --albert_config_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\albert_config.json\" --do_train True  --do_eval True --train_batch_size=8 --eval_batch_size=8 --max_seq_length=512 --max_predictions_per_seq=77 --optimizer=\"adamw\" --learning_rate=1e-4 --num_train_steps=10000 --num_warmup_steps=10 --save_checkpoints_steps=500 --use_gpu=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "python -m albert.create_pretraining_data --input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\DS\\corpus.txt\" --output_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\further\\20dupetf_examples.tfrecord\" --vocab_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\30k-clean.vocab\" --spm_model_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\albert-master\\pre-trained\\albert_base\\30k-clean.model\" --do_lower_case=True --max_seq_length=512 --max_predictions_per_seq=77 --masked_lm_prob=0.15 --random_seed=12345 --dupe_factor=20 --do_whole_word_mask=True"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}