Experiment 0 BERT Base Pre-training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/bam/Desktop/ECHR Publication Experiments'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "# os.chdir(\"/tmp/\")\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "COURPUS_DIR = '/home/bam/Desktop/ECHR Publication Experiments/DS/corpus.txt'\n",
    "BERT_BASE_DIR ='/home/bam/Desktop/ECHR Publication Experiments/bert_master'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# !cd \"/home/bam/Desktop/ECHR Publication Experiments\"\\\n",
    "!python \"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\create_pretraining_data.py\" \\\n",
    "  --input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\DS\\corpus.txt\" \\\n",
    "  --output_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\further\\20dupetf_examples.tfrecord\" \\\n",
    "  --vocab_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\vocab.txt\"\\\n",
    "  --do_lower_case=True \\\n",
    "  --max_seq_length=512 \\\n",
    "  --max_predictions_per_seq=77 \\\n",
    "  --masked_lm_prob=0.15 \\\n",
    "  --random_seed=12345 \\\n",
    "  --dupe_factor=20 \\\n",
    "  --do_whole_word_mask=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "python \"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\create_pretraining_data.py\" --input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\DS\\corpus.txt\" --output_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\further\\20dupetf_examples.tfrecord\" --vocab_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\vocab.txt\" --do_lower_case=True --max_seq_length=512 --max_predictions_per_seq=77 --masked_lm_prob=0.15 --random_seed=12345 --dupe_factor=20 --do_whole_word_mask=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python \"bert_master/run_pretraining.py\"\\\n",
    "--input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\further\\20dupetf_examples.tfrecord\"\\\n",
    "--output_dir=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\further\"\\\n",
    "--do_train=True\\\n",
    "--do_eval=True\\\n",
    "--bert_config_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\bert_config.json\"\\\n",
    "--init_checkpoint=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\bert_model.ckpt\"\\\n",
    "--train_batch_size=8\\\n",
    "--max_seq_length=512\\\n",
    "--max_predictions_per_seq=77\\\n",
    "--num_train_steps=10000\\\n",
    "--num_warmup_steps=1000\\\n",
    "--learning_rate=2e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "python \"bert_master/run_pretraining.py\" --input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\further\\20dupetf_examples.tfrecord\" --output_dir=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\further\" --do_train=True --do_eval=True --bert_config_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\bert_config.json\" --init_checkpoint=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\bert_model.ckpt\" --train_batch_size=4 --max_seq_length=512 --max_predictions_per_seq=77 --num_train_steps=10000 --num_warmup_steps=1000 --learning_rate=2e-5 --save_checkpoints_steps=500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final Pre-training Decision (Steps tbd)\n",
    "python \"bert_master/run_pretraining.py\" --input_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\further\\20dupetf_examples.tfrecord\" --output_dir=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\further\" --do_train=True --do_eval=True --bert_config_file=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\bert_config.json\" --init_checkpoint=\"C:\\Users\\Yifei\\Desktop\\ECHR Publication Experiments\\bert_master\\pre-trained\\uncased_L-12_H-768_A-12\\bert_model.ckpt\" --train_batch_size=4 --max_seq_length=512 --max_predictions_per_seq=77 --num_train_steps=100000 --num_warmup_steps=10000 --learning_rate=8e-5 --save_checkpoints_steps=10000"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}