Z01_LLaMA-Factory_HPC_v3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ORCx8i1o1ZK9",
    "tags": []
   },
   "source": [
    "# LLaMA_Factory (HPC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PAnZLFDg1ZLC"
   },
   "source": [
    "## 初始環境設定, 切換到 kernel pytorch_2.1.0-cuda11.8-cudnn8-devel.sif\n",
    "\n",
    "https://zhuanlan.zhihu.com/p/630734624"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "3UgaDyzu1ZLD",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 初始環境設定\n",
    "import os\n",
    "from pathlib import Path\n",
    "HOME = str(Path.home())\n",
    "Add_Binarry_Path=HOME+'/.local/bin:/usr/ubuntu_bin'\n",
    "os.environ['PATH']=os.environ['PATH']+':'+Add_Binarry_Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pip install deepspeed==0.12.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "git clone https://github.com/microsoft/DeepSpeed.git\n",
    "cd DeepSpeed\n",
    "DS_BUILD_FUSED_ADAM=1 DS_BUILD_CPU_ADAM=1  pip3 install .\n",
    "ds_report 查"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "git clone https://github.com/microsoft/DeepSpeed.git ${STAGE_DIR}/DeepSpeed\n",
    "RUN cd ${STAGE_DIR}/DeepSpeed && \\\n",
    "        git checkout . && \\\n",
    "        git checkout master && \\\n",
    "        ./install.sh --pip_sudo\n",
    "RUN rm -rf ${STAGE_DIR}/DeepSpeed\n",
    "RUN python -c \"import deepspeed; print(deepspeed.__version__)\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M3Wt3onj1ZLH"
   },
   "source": [
    "## 安裝套件, 切換到 kernel pytorch_2.1.0-cuda11.8-cudnn8-devel.sif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "pip uninstall torch -y\n",
    "pip uninstall xformers -y\n",
    "pip install --upgrade torch==2.1.2 --index-url https://download.pytorch.org/whl/cu118  -q\n",
    "pip install --upgrade xformers==0.0.23.post1 --index-url https://download.pytorch.org/whl/cu118 -q\n",
    "pip install bitsandbytes==0.42.0 accelerate==0.24.1 transformers==4.34.1 datasets==2.14.7 tiktoken==0.5.1 peft==0.6.2 trl==0.7.1 deepspeed==0.12.3 -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ls /home/g00cjz00/.cache/torch_extensions/py310_cu118 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "rm -rf LLaMA-Factory\n",
    "git clone https://github.com/hiyouga/LLaMA-Factory.git\n",
    "cd LLaMA-Factory\n",
    "pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "z0dGdgGD1ZLF"
   },
   "source": [
    "## 確認CUDA版本, 以及否能使用GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "xuM2aF8F1ZLG",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Feb 12 22:14:10 2024       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 510.47.03    Driver Version: 510.47.03    CUDA Version: 11.6     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla V100-SXM2...  Off  | 00000000:1B:00.0 Off |                    0 |\n",
      "| N/A   33C    P0    41W / 300W |      0MiB / 32768MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  Tesla V100-SXM2...  Off  | 00000000:1C:00.0 Off |                    0 |\n",
      "| N/A   33C    P0    42W / 300W |      0MiB / 32768MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  Tesla V100-SXM2...  Off  | 00000000:B1:00.0 Off |                    0 |\n",
      "| N/A   32C    P0    40W / 300W |      0MiB / 32768MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  Tesla V100-SXM2...  Off  | 00000000:B2:00.0 Off |                    0 |\n",
      "| N/A   32C    P0    41W / 300W |      0MiB / 32768MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n",
      "2.1.2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!nvidia-smi\n",
    "import torch\n",
    "print(torch.__version__)\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: The directory '/tmp/user-pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.\u001b[0m\u001b[33m\n",
      "\u001b[0mtorch                     2.1.2        /opt/conda/lib/python3.10/site-packages\n",
      "torchaudio                2.1.2        /opt/conda/lib/python3.10/site-packages\n",
      "torchelastic              0.2.2        /opt/conda/lib/python3.10/site-packages            pip\n",
      "torchvision               0.16.2       /opt/conda/lib/python3.10/site-packages\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip list -v |grep torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YzSyHE8B1ZLI"
   },
   "source": [
    "## 撰寫程式碼"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qHey8AHr1ZLJ",
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# IP\n",
    "node_ip=$(cat /etc/hosts |grep \"$(hostname -a)\" | awk '{print $1}')\n",
    "# PORT\n",
    "noed_port_genai=$(python -c \"import socket; s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(('', 0)); addr = s.getsockname(); s.close(); print(addr[1])\")\n",
    "# PROXY\n",
    "proxy_url=/rstudio/${node_ip}/${noed_port_genai}\n",
    "# URL\n",
    "https_url=https://node01.biobank.org.tw${proxy_url}/\n",
    "\n",
    "# SCRIPT FILE\n",
    "cat << EOF >  LLaMA-Factory/src/train_web_demo.py\n",
    "from llmtuner import create_ui\n",
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "print(\"${https_url}\")\n",
    "\n",
    "def main():\n",
    "    demo = create_ui()\n",
    "    demo.queue()\n",
    "    demo.launch(server_port=${noed_port_genai}, server_name=\"$(hostname -s)\", share=False, inbrowser=True, root_path=\"${proxy_url}\", auth=(\"nchc\", \"nchcorgtw\"))\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n",
    "EOF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 強制刪除運行中服務"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 強制刪除運行中服務\n",
    "!ps -ef |grep train_web_demo.py | awk '{print $2}' | xargs kill -9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LZqt2QBg1ZLK"
   },
   "source": [
    "## 開啟WebUI\n",
    "- model: /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "#請更新以下 HF_TOKEN\n",
    "cd LLaMA-Factory\n",
    "CUDA_VISIBLE_DEVICES=0 HF_TOKEN='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' python src/train_web_demo.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CMD LORA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "#請更新以下 HF_TOKEN\n",
    "cd LLaMA-Factory\n",
    "rm -rf saves/LLaMA2-13B/lora/train_2024-02-12-17-13-20\n",
    "CUDA_VISIBLE_DEVICES=1 HF_TOKEN='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' python src/train_bash.py \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-13b-chat-hf \\\n",
    "    --finetuning_type lora \\\n",
    "    --template default \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 30000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-13B/lora/train_2024-02-12-17-13-20 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CMD FULL -> ERROR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "#請更新以下 HF_TOKEN\n",
    "cd LLaMA-Factory\n",
    "rm -rf saves/LLaMA2-7B/full/train_2024-02-12-17-23-41\n",
    "CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf \\\n",
    "    --finetuning_type full \\\n",
    "    --template default \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 30000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B/full/train_2024-02-12-17-23-41 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Method3 ACCELERATE (singularity/notebook)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash \n",
    "## Method4 DEEPSPEED (singularity/notebook)\n",
    "export GPUS_PER_NODE=2\n",
    "export MASTER_ADDR=$(hostname -s)\n",
    "export MASTER_PORT=$(python -c \"import socket; s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(('', 0)); addr = s.getsockname(); s.close(); print(addr[1])\")\n",
    "export DS_CONFIG=\"ds_config_zero3.json\"\n",
    "export MODEL_ID=\"/work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf\"\n",
    "export DATASET=\"alpaca_zh\"\n",
    "export OUTPUT_DIR=\"saves/LLaMA2-7B-Chat/lora/01_sft\"\n",
    "export MAX_SAMPLE=5000\n",
    "\n",
    "## CLEAN CACHE\n",
    "ps -ef |grep  train | awk '{print $2}' | xargs kill -9\n",
    "rm -rf /work/g00cjz00/github/LLaMA-Factory/${OUTPUT_DIR}\n",
    "\n",
    "## RUN (chage kernel to Python3 or your Image kernel)\n",
    "## If you use Image kernel, mark /work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity\n",
    "## If you use python kernel, unmark /work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity\n",
    "\n",
    "#/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity run --nv --cleanenv -B /work -B /work/g00cjz00/libraryFolder/S_work-genai11_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv/local/lib:/home/g00cjz00/.local/lib -B /work/g00cjz00/libraryFolder/S_work-genai11_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv/local/bin:/home/g00cjz00/.local/bin -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin /work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif \\\n",
    "bash -c \"export PATH=\\$PATH:\\$HOME/.local/bin; \\\n",
    "    cd /work/g00cjz00/github/LLaMA-Factory; \\\n",
    "    deepspeed --include=localhost:0,1 --master_port 61000 \\\n",
    "    src/train_bash.py \\\n",
    "    --deepspeed ${DS_CONFIG} \\\n",
    "    --stage sft \\\n",
    "    --do_train \\\n",
    "    --model_name_or_path ${MODEL_ID} \\\n",
    "    --dataset ${DATASET} \\\n",
    "    --template default \\\n",
    "    --finetuning_type full \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir path_to_sft_checkpoint \\\n",
    "    --overwrite_cache \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 1000 \\\n",
    "    --learning_rate 5e-5 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --plot_loss \\\n",
    "    --fp16 \\\n",
    "    --max_samples ${MAX_SAMPLE} \\\n",
    "    --output_dir ${OUTPUT_DIR}\n",
    "\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# reference https://zhuanlan.zhihu.com/p/678211123\n",
    "cd LLaMA-Factory\n",
    "\n",
    "cat << \\EOF >  ds_config1.json\n",
    "{\n",
    "  \"train_batch_size\": \"auto\",\n",
    "  \"train_micro_batch_size_per_gpu\": \"auto\",\n",
    "  \"gradient_accumulation_steps\": \"auto\",\n",
    "  \"gradient_clipping\": \"auto\",\n",
    "  \"zero_allow_untested_optimizer\": true,\n",
    "  \"bf16\": {\n",
    "    \"enabled\": \"auto\"\n",
    "  },\n",
    "  \"fp16\": {\n",
    "    \"enabled\": \"auto\",\n",
    "    \"loss_scale\": 0,\n",
    "    \"initial_scale_power\": 16,\n",
    "    \"loss_scale_window\": 1000,\n",
    "    \"hysteresis\": 2,\n",
    "    \"min_loss_scale\": 1\n",
    "  },\n",
    "  \"zero_optimization\": {\n",
    "    \"stage\": 1\n",
    "  }\n",
    "}\n",
    "EOF\n",
    "\n",
    "\n",
    "cat << \\EOF >  ds_config2.json\n",
    "{\n",
    "  \"train_batch_size\": \"auto\",\n",
    "  \"train_micro_batch_size_per_gpu\": \"auto\",\n",
    "  \"gradient_accumulation_steps\": \"auto\",\n",
    "  \"gradient_clipping\": \"auto\",\n",
    "  \"zero_allow_untested_optimizer\": true,\n",
    "  \"fp16\": {\n",
    "    \"enabled\": \"auto\",\n",
    "    \"loss_scale\": 0,\n",
    "    \"initial_scale_power\": 16,\n",
    "    \"loss_scale_window\": 1000,\n",
    "    \"hysteresis\": 2,\n",
    "    \"min_loss_scale\": 1\n",
    "  },\n",
    "  \"zero_optimization\": {\n",
    "    \"stage\": 2,\n",
    "    \"offload_optimizer\": {\n",
    "         \"device\": \"cpu\",\n",
    "         \"pin_memory\": true\n",
    "     },\n",
    "    \"allgather_partitions\": true,\n",
    "    \"allgather_bucket_size\": 5e8,\n",
    "    \"reduce_scatter\": true,\n",
    "    \"reduce_bucket_size\": 5e8,\n",
    "    \"overlap_comm\": false,\n",
    "    \"contiguous_gradients\": true\n",
    "  }\n",
    "}\n",
    "EOF\n",
    "\n",
    "\n",
    "cat << \\EOF >  ds_config3.json\n",
    "{\n",
    "  \"train_batch_size\": \"auto\",\n",
    "  \"train_micro_batch_size_per_gpu\": \"auto\",\n",
    "  \"gradient_accumulation_steps\": \"auto\",\n",
    "  \"gradient_clipping\": \"auto\",\n",
    "  \"zero_allow_untested_optimizer\": true,\n",
    "  \"fp16\": {\n",
    "    \"enabled\": \"auto\",\n",
    "    \"loss_scale\": 0,\n",
    "    \"initial_scale_power\": 16,\n",
    "    \"loss_scale_window\": 1000,\n",
    "    \"hysteresis\": 2,\n",
    "    \"min_loss_scale\": 1\n",
    "  },\n",
    "    \"zero_optimization\": {\n",
    "        \"stage\": 3,\n",
    "        \"offload_optimizer\": {\n",
    "            \"device\": \"cpu\",\n",
    "            \"pin_memory\": true\n",
    "        },\n",
    "        \"offload_param\": {\n",
    "            \"device\": \"cpu\",\n",
    "            \"pin_memory\": true\n",
    "        },\n",
    "        \"overlap_comm\": true,\n",
    "        \"allgather_partitions\": true,\n",
    "        \"allgather_bucket_size\": 5e10,\n",
    "        \"reduce_scatter\": true,\n",
    "        \"reduce_bucket_size\": 5e10,\n",
    "        \"contiguous_gradients\": true,\n",
    "        \"sub_group_size\": 1e9,\n",
    "        \"reduce_bucket_size\": \"auto\",\n",
    "        \"stage3_prefetch_bucket_size\": \"auto\",\n",
    "        \"stage3_param_persistence_threshold\": \"auto\",\n",
    "        \"stage3_max_live_parameters\": 1e9,\n",
    "        \"stage3_max_reuse_distance\": 1e9,\n",
    "        \"stage3_gather_16bit_weights_on_model_save\": true\n",
    "    }\n",
    "}\n",
    "EOF\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#pip install transformers==4.37.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#rm -rf /home/g00cjz00/.cache/torch_extensions/py310_cu118 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-02-12 23:23:46,405] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 20780\n",
      "[2024-02-12 23:23:53,113] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 20781\n",
      "[2024-02-12 23:23:59,826] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 20782\n",
      "[2024-02-12 23:24:06,413] [INFO] [launch.py:324:sigkill_handler] Main process received SIGINT, exiting\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "#請更新以下 HF_TOKEN\n",
    "cd LLaMA-Factory\n",
    "rm -rf saves/LLaMA2-7B/full/train_2024-02-12-17-23-41\n",
    "#rm -rf /home/g00cjz00/.cache/torch_extensions/py310_cu118 \n",
    "deepspeed --num_gpus 4 --master_port=9901 src/train_bash.py \\\n",
    "    --deepspeed ds_config3.json \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-13b-chat-hf \\\n",
    "    --finetuning_type full \\\n",
    "    --template default \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 2e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 30000 \\\n",
    "    --per_device_train_batch_size 12 \\\n",
    "    --gradient_accumulation_steps 12 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B/full/train_2024-02-12-17-23-41 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "cd LLaMA-Factory\n",
    "rm -rf saves/LLaMA2-7B/full/train_2024-02-12-17-23-41\n",
    "/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv -B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin -B /work/g00cjz00/libraryFolder/S-work-genai_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv/local/lib:/home/g00cjz00/.local/lib -B /work/g00cjz00/libraryFolder/S-work-genai_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv/local/bin:/home/g00cjz00/.local/bin /work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif \\\n",
    "bash -c 'export PATH=$PATH:$HOME/.local/bin; \\\n",
    "deepspeed --num_gpus 4 --master_port=9901 src/train_bash.py \\\n",
    "    --deepspeed ds_config3.json \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-13b-chat-hf \\\n",
    "    --finetuning_type full \\\n",
    "    --template default \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 2e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 30000 \\\n",
    "    --per_device_train_batch_size 12 \\\n",
    "    --gradient_accumulation_steps 12 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B/full/train_2024-02-12-17-23-41 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### slurm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sbatch demo.slurm\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "cat << \\EOF >  demo.slurm\n",
    "#!/work/u00cjz00/binary/bash5.0/bin/bash\n",
    "#SBATCH -A GOV109189                                                    ### project number, Example MST109178\n",
    "#SBATCH -J _t2demo_                                                     ### Job name, Exmaple jupyterlab\n",
    "#SBATCH -p gp4d                                                         ### Partition Name, Example ngs1gpu\n",
    "#SBATCH --nodes=1                                                       ### Nodes, Default 1, node number\n",
    "#SBATCH --ntasks-per-node=1                                             ### Tasks, Default 1, per node tasks\n",
    "#SBATCH -c 16                                                            ### Cores assigned to each task, Example 4\n",
    "#SBATCH --gres=gpu:4                                                    ### GPU number, Example gpu:1\n",
    "##SBATCH --time=0-10:00:00                                                ### Runnung time, days-hours:minutes:seconds or hours:minutes:seconds\n",
    "#SBATCH -o genai_%j.out                                                 ### Log folder, Here %j is job ID\n",
    "#SBATCH -e genai_%j.err                                                 ### Log folder, Here %j is job ID\n",
    "\n",
    "cd /home/g00cjz00/demo2024/llm_course_20231228/LLaMA-Factory\n",
    "rm -rf saves/LLaMA2-7B/full/train_2024-02-12-17-23-41\n",
    "/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv -B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin -B /work/g00cjz00/libraryFolder/S-work-genai_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv/local/lib:/home/g00cjz00/.local/lib -B /work/g00cjz00/libraryFolder/S-work-genai_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv/local/bin:/home/g00cjz00/.local/bin /work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif \\\n",
    "bash -c 'export PATH=$PATH:$HOME/.local/bin; \\\n",
    "deepspeed --num_gpus 4 --master_port=9901 src/train_bash.py \\\n",
    "    --deepspeed ds_config3.json \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-13b-chat-hf \\\n",
    "    --finetuning_type full \\\n",
    "    --template default \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 2e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 30000 \\\n",
    "    --per_device_train_batch_size 12 \\\n",
    "    --gradient_accumulation_steps 12 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B/full/train_2024-02-12-17-23-41 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True'\n",
    "\n",
    "EOF\n",
    "\n",
    "echo sbatch  demo.slurm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## slurm mpi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sbatch demo.slurm\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "cat << \\EOF >  demo.script\n",
    "## Method5 DEEPSPEED + Torch.Distributed.Run (singularity/notebook)\n",
    "export GPUS_PER_NODE=2\n",
    "nodes=( $( scontrol show hostnames $SLURM_JOB_NODELIS ) )\n",
    "nodes_array=($nodes)\n",
    "export MASTER_ADDR=${nodes_array[0]}\n",
    "export MASTER_PORT=29003\n",
    "\n",
    "/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv -B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin -B /work/g00cjz00/libraryFolder/S-work-genai_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv/local/lib:/home/g00cjz00/.local/lib -B /work/g00cjz00/libraryFolder/S-work-genai_pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv/local/bin:/home/g00cjz00/.local/bin /work/u00cjz00/nvidia/pytorch_2.1.2-cuda11.8-cudnn8-devel_opencv.sif \\\n",
    "bash -c \"export PATH=\\$PATH:\\$HOME/.local/bin; \\\n",
    "    cd /work/g00cjz00/github/LLaMA-Factory; \\\n",
    "    python3 -m torch.distributed.run \\\n",
    "    --nproc_per_node ${GPUS_PER_NODE} \\\n",
    "    --nnodes ${SLURM_NNODES}\\\n",
    "    --node_rank ${SLURM_PROCID} \\\n",
    "    --master_addr ${MASTER_ADDR} \\\n",
    "    --master_port ${MASTER_PORT} \\\n",
    "    src/train_bash.py \\\n",
    "    --deepspeed ds_config3.json \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-13b-chat-hf \\\n",
    "    --finetuning_type full \\\n",
    "    --template default \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 2e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 30000 \\\n",
    "    --per_device_train_batch_size 12 \\\n",
    "    --gradient_accumulation_steps 12 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B/full/train_2024-02-12-17-23-41 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True\"\n",
    "\n",
    "EOF\n",
    "\n",
    "\n",
    "cat << \\EOF >  demo.slurm\n",
    "#!/work/u00cjz00/binary/bash5.0/bin/bash\n",
    "#SBATCH -A MST110386                                                    ### project number, Example MST109178\n",
    "#SBATCH -J _t2demo_                                                     ### Job name, Exmaple jupyterlab\n",
    "#SBATCH -p gp4d                                                         ### Partition Name, Example ngs1gpu\n",
    "#SBATCH --nodes=4                                                       ### Nodes, Default 1, node number\n",
    "#SBATCH --ntasks-per-node=1                                             ### Tasks, Default 1, per node tasks\n",
    "#SBATCH -c 8                                                            ### Cores assigned to each task, Example 4\n",
    "#SBATCH --gres=gpu:2                                                    ### GPU number, Example gpu:1\n",
    "##SBATCH --time=0-1:00:00                                                ### Runnung time, days-hours:minutes:seconds or hours:minutes:seconds\n",
    "#SBATCH -o genai_%j.out                                                 ### Log folder, Here %j is job ID\n",
    "#SBATCH -e genai_%j.err                                                 ### Log folder, Here %j is job ID\n",
    "\n",
    "\n",
    "\n",
    "srun --mpi=pmi2 bash demo.script\n",
    "\n",
    "EOF\n",
    "\n",
    "echo sbatch demo.slurm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I11rZlhctzd9"
   },
   "source": [
    "# 上傳輸出的檔案到 HuggingFace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Rte6fcaejOsC"
   },
   "outputs": [],
   "source": [
    "!pip install huggingface_hub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7hJ7hkH21ZLL"
   },
   "source": [
    "## 使用API Token登入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TRbS8HBWTZpu"
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Zj-P4LS21ZLL"
   },
   "source": [
    "## 上傳模型\n",
    "執行本儲存格後填入 API Token(需要有 Write 權限)，然後按下登入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Xd7sv-ZvT4KB"
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import HfApi\n",
    "api = HfApi()\n",
    "\n",
    "model_id = \"username/fine-tune-model\" #@param {type:\"string\"}\n",
    "export_folder_path = \"/content/LLaMA-Factory/export\" #@param {type:\"string\"}\n",
    "api.create_repo(model_id, private=True, exist_ok=True, repo_type=\"model\")\n",
    "api.upload_folder(\n",
    "    folder_path=export_folder_path,\n",
    "    repo_id=model_id,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CMD\n",
    "change kernel to Image ipykernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#請更新以下 HF_TOKEN\n",
    "!cd LLaMA-Factory; CUDA_VISIBLE_DEVICES=0 HF_TOKEN='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' python src/train_bash.py \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf \\\n",
    "    --finetuning_type lora \\\n",
    "    --quantization_bit 4 \\\n",
    "    --template llama2 \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_gpt4_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 1000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-21 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SINGULARITY CMD\n",
    "change kernel to Python ipykernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec \\\n",
    "--nv --cleanenv \\\n",
    "-B /work \\\n",
    "-B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin \\\n",
    "-B /work/g00cjz00/libraryFolder/P-3.10-work-deepspeed2_cuda_11.8.0-cudnn8-runtime-ubuntu22.04/local/lib:/home/g00cjz00/.local/lib \\\n",
    "-B /work/g00cjz00/libraryFolder/P-3.10-work-deepspeed2_cuda_11.8.0-cudnn8-runtime-ubuntu22.04/local/bin:/home/g00cjz00/.local/bin \\\n",
    "/work/u00cjz00/nvidia/cuda_11.8.0-cudnn8-runtime-ubuntu22.04.sif \\\n",
    "bash -c 'export PATH=$PATH:$HOME/.local/bin; cd LLaMA-Factory; \\\n",
    "CUDA_VISIBLE_DEVICES=0 HF_TOKEN=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 \\\n",
    "python src/train_bash.py \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf \\\n",
    "    --finetuning_type lora \\\n",
    "    --quantization_bit 4 \\\n",
    "    --template llama2 \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_gpt4_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 1000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True'\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SINGULARITY CMD DEEPSPEED\n",
    "change kernel to Python ipykernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "export SLURM_NTASKS=2\n",
    "export HOSTFILE=\"./hostfile\"\n",
    "echo \"$(hostname -s) slots=$SLURM_NTASKS\" > $HOSTFILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv -B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin -B /work/g00cjz00/libraryFolder/P-3.10-work-deepspeed4_cuda_11.8.0-cudnn8-devel-ubuntu22.04/local/lib:/home/g00cjz00/.local/lib -B /work/g00cjz00/libraryFolder/P-3.10-work-deepspeed4_cuda_11.8.0-cudnn8-devel-ubuntu22.04/local/bin:/home/g00cjz00/.local/bin /work/u00cjz00/nvidia/cuda_11.8.0-cudnn8-devel-ubuntu22.04.sif \\\n",
    "bash -c 'export PATH=$PATH:$HOME/.local/bin; cd LLaMA-Factory; rm -rf saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25; \\\n",
    "HF_TOKEN=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 \\\n",
    "deepspeed --hostfile=../hostfile src/train_bash.py --deepspeed ../ds_config_zero3.json \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf \\\n",
    "    --finetuning_type lora \\\n",
    "    --template llama2 \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_gpt4_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 1000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#deepspeed --master_port 29500 --num_gpus=2 src/train_bash.py --deepspeed ../ds_config_zero3.json \n",
    "\n",
    "!/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv -B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin -B /work/g00cjz00/libraryFolder/P-3.10-work-deepspeed4_cuda_11.8.0-cudnn8-devel-ubuntu22.04/local/lib:/home/g00cjz00/.local/lib -B /work/g00cjz00/libraryFolder/P-3.10-work-deepspeed4_cuda_11.8.0-cudnn8-devel-ubuntu22.04/local/bin:/home/g00cjz00/.local/bin /work/u00cjz00/nvidia/cuda_11.8.0-cudnn8-devel-ubuntu22.04.sif \\\n",
    "bash -c 'export PATH=$PATH:$HOME/.local/bin; cd LLaMA-Factory; rm -rf saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25; \\\n",
    "HF_TOKEN=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 \\\n",
    "deepspeed --hostfile=../hostfile src/train_bash.py --deepspeed ../ds_config_zero2_v1.json \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf \\\n",
    "    --finetuning_type lora \\\n",
    "    --template llama2 \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_gpt4_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 1000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "!/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec --nv --cleanenv -B /work -B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin -B /work/g00cjz00/libraryFolder/P-3.10-work-deepspeed4_cuda_11.8.0-cudnn8-devel-ubuntu22.04/local/lib:/home/g00cjz00/.local/lib -B /work/g00cjz00/libraryFolder/P-3.10-work-deepspeed4_cuda_11.8.0-cudnn8-devel-ubuntu22.04/local/bin:/home/g00cjz00/.local/bin /work/u00cjz00/nvidia/cuda_11.8.0-cudnn8-devel-ubuntu22.04.sif \\\n",
    "bash -c 'export PATH=$PATH:$HOME/.local/bin; cd LLaMA-Factory; rm -rf saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25; \\\n",
    "deepspeed --master_port 29500 --num_gpus=2 src/train_bash.py --deepspeed ../ds_config_zero2_v1.json \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf \\\n",
    "    --finetuning_type lora \\\n",
    "    --template llama2 \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_gpt4_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 1000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True'\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SLURM SINGULARITY CMD\n",
    "change kernel to Python ipykernel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SLURM method1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat << \\EOF >  demo.slurm\n",
    "#!/work/u00cjz00/binary/bash5.0/bin/bash\n",
    "#SBATCH -A GOV109189                                                    ### project number, Example MST109178\n",
    "#SBATCH -J _t2demo_                                                     ### Job name, Exmaple jupyterlab\n",
    "#SBATCH -p gp4d                                                         ### Partition Name, Example ngs1gpu\n",
    "#SBATCH --nodes=1                                                       ### Nodes, Default 1, node number\n",
    "#SBATCH --ntasks-per-node=1                                             ### Tasks, Default 1, per node tasks\n",
    "#SBATCH -c 4                                                            ### Cores assigned to each task, Example 4\n",
    "#SBATCH --gres=gpu:1                                                    ### GPU number, Example gpu:1\n",
    "#SBATCH --time=0-1:00:00                                                ### Runnung time, days-hours:minutes:seconds or hours:minutes:seconds\n",
    "#SBATCH -o genai_%j.out                                                 ### Log folder, Here %j is job ID\n",
    "#SBATCH -e genai_%j.err                                                 ### Log folder, Here %j is job ID\n",
    "\n",
    "\n",
    "/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec \\\n",
    "--nv --cleanenv \\\n",
    "-B /work \\\n",
    "-B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin \\\n",
    "-B /work/g00cjz00/libraryFolder/P-3.10-work-llama_factory_cuda_11.8.0-cudnn8-runtime-ubuntu22.04/local/lib:/home/g00cjz00/.local/lib \\\n",
    "-B /work/g00cjz00/libraryFolder/P-3.10-work-llama_factory_cuda_11.8.0-cudnn8-runtime-ubuntu22.04/local/bin:/home/g00cjz00/.local/bin \\\n",
    "/work/u00cjz00/nvidia/cuda_11.8.0-cudnn8-runtime-ubuntu22.04.sif \\\n",
    "bash -c 'export PATH=$PATH:$HOME/.local/bin; cd ~/demo2024/llm_course_20231228/LLaMA-Factory; \\\n",
    "CUDA_VISIBLE_DEVICES=0 HF_TOKEN=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 \\\n",
    "python src/train_bash.py \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf \\\n",
    "    --finetuning_type lora \\\n",
    "    --quantization_bit 4 \\\n",
    "    --template llama2 \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_gpt4_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 1000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True'\n",
    "\n",
    "EOF\n",
    "\n",
    "sbatch  demo.slurm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SLURM method2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat << \\EOF >  demo.script\n",
    "/work/opt/ohpc/Taiwania3/libs/singularity/3.10.2/bin/singularity exec \\\n",
    "--nv --cleanenv \\\n",
    "-B /work \\\n",
    "-B /work/u00cjz00/os/ubuntu/bin:/usr/ubuntu_bin \\\n",
    "-B /work/g00cjz00/libraryFolder/P-3.10-work-llama_factory_cuda_11.8.0-cudnn8-runtime-ubuntu22.04/local/lib:/home/g00cjz00/.local/lib \\\n",
    "-B /work/g00cjz00/libraryFolder/P-3.10-work-llama_factory_cuda_11.8.0-cudnn8-runtime-ubuntu22.04/local/bin:/home/g00cjz00/.local/bin \\\n",
    "/work/u00cjz00/nvidia/cuda_11.8.0-cudnn8-runtime-ubuntu22.04.sif \\\n",
    "bash -c 'export PATH=$PATH:$HOME/.local/bin; cd ~/demo2024/llm_course_20231228/LLaMA-Factory; \\\n",
    "CUDA_VISIBLE_DEVICES=0 HF_TOKEN=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 \\\n",
    "python src/train_bash.py \\\n",
    "    --stage sft \\\n",
    "    --do_train True \\\n",
    "    --model_name_or_path /work/u00cjz00/slurm_jobs/github/models/Llama-2-7b-chat-hf \\\n",
    "    --finetuning_type lora \\\n",
    "    --quantization_bit 4 \\\n",
    "    --template llama2 \\\n",
    "    --dataset_dir data \\\n",
    "    --dataset alpaca_gpt4_zh \\\n",
    "    --cutoff_len 1024 \\\n",
    "    --learning_rate 5e-05 \\\n",
    "    --num_train_epochs 3.0 \\\n",
    "    --max_samples 1000 \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --max_grad_norm 1.0 \\\n",
    "    --logging_steps 5 \\\n",
    "    --save_steps 100 \\\n",
    "    --warmup_steps 0 \\\n",
    "    --lora_rank 8 \\\n",
    "    --lora_dropout 0.1 \\\n",
    "    --lora_target q_proj,v_proj \\\n",
    "    --output_dir saves/LLaMA2-7B-Chat/lora/train_2024-02-11-18-12-25 \\\n",
    "    --fp16 True \\\n",
    "    --plot_loss True'\n",
    "EOF\n",
    "\n",
    "\n",
    "cat << \\EOF >  demo.slurm\n",
    "#!/work/u00cjz00/binary/bash5.0/bin/bash\n",
    "#SBATCH -A GOV109189                                                    ### project number, Example MST109178\n",
    "#SBATCH -J _t2demo_                                                     ### Job name, Exmaple jupyterlab\n",
    "#SBATCH -p gp4d                                                         ### Partition Name, Example ngs1gpu\n",
    "#SBATCH --nodes=1                                                       ### Nodes, Default 1, node number\n",
    "#SBATCH --ntasks-per-node=1                                             ### Tasks, Default 1, per node tasks\n",
    "#SBATCH -c 4                                                            ### Cores assigned to each task, Example 4\n",
    "#SBATCH --gres=gpu:1                                                    ### GPU number, Example gpu:1\n",
    "#SBATCH --time=0-1:00:00                                                ### Runnung time, days-hours:minutes:seconds or hours:minutes:seconds\n",
    "#SBATCH -o genai_%j.out                                                 ### Log folder, Here %j is job ID\n",
    "#SBATCH -e genai_%j.err                                                 ### Log folder, Here %j is job ID\n",
    "\n",
    "srun bash demo.script\n",
    "\n",
    "EOF\n",
    "\n",
    "sbatch  demo.slurm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CHECK SLURM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!squeue -u $(whoami)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cat genai_569509.err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!srun --jobid=569509 nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}