From 824061dda1f3f8ce76380532b0d2e13f01b75097 Mon Sep 17 00:00:00 2001 From: raveen-k <128157920+raveen-k@users.noreply.github.com> Date: Tue, 6 Aug 2024 21:52:30 -0700 Subject: [PATCH] Added notebook for llama3 8b training --- .gitignore | 1 + .../sft_ft_cloud_llama3.ipynb | 249 ++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 .gitignore create mode 100644 examples/SFT_finetune_cloud/sft_ft_cloud_llama3.ipynb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..496ee2c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.DS_Store \ No newline at end of file diff --git a/examples/SFT_finetune_cloud/sft_ft_cloud_llama3.ipynb b/examples/SFT_finetune_cloud/sft_ft_cloud_llama3.ipynb new file mode 100644 index 0000000..0af54e8 --- /dev/null +++ b/examples/SFT_finetune_cloud/sft_ft_cloud_llama3.ipynb @@ -0,0 +1,249 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ“ฆ Installation\n", + "To get started with fine-tuning your own LLM models using the Simplifine library, install it directly from the GitHub repository using the following code:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Install the latest Simplifine library from the GitHub repository\n", + "!pip install git+https://github.com/simplifine-llm/Simplifine.git -q\n", + "\n", + "# The 'pip install' command is used to install Python packages.\n", + "# The '-q' option stands for 'quiet', which minimizes the amount of output produced during the installation.\n", + "# 'git+https://github.com/simplifine-llm/Simplifine.git' specifies the URL of the GitHub repository from which to install the package.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿš€ Fine-Tuning LLaMA-3 8B Model\n", + "\n", + "In this section, we will focus on fine-tuning the LLaMA-3 8B model using the Simplifine library. Follow the steps below to set up your environment, initialize WandB, prepare your dataset, and configure the Simplifine client." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/raveenkariyawasam/anaconda3/envs/simplifine/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msimplifine_alpha\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtrain_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Client\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwandb\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/simplifine_alpha/train_utils.py:19\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;124;03m Simplfine is an easy-to-use, open-source library for fine-tuning LLMs models quickly on your own hardware or cloud.\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;124;03m Copyright (C) 2024 Simplifine Corp.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;124;03m along with this program. If not, see .\u001b[39;00m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;124;03m'''\u001b[39;00m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtrain_engine_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m send_train_query, get_company_status, get_job_log, download_directory, stop_job\n\u001b[0;32m---> 19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtrain_engine\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m sftPromptConfig, PromptConfig\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01murl_class\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m url_config\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mzipfile\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/simplifine_alpha/train_engine.py:22\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtqdm\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mauto\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m tqdm\n\u001b[0;32m---> 22\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtransformers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForLanguageModeling\n\u001b[1;32m 23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DataLoader\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtransformers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AutoModelForSequenceClassification\n", + "File \u001b[0;32m:1412\u001b[0m, in \u001b[0;36m_handle_fromlist\u001b[0;34m(module, fromlist, import_, recursive)\u001b[0m\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/transformers/utils/import_utils.py:1593\u001b[0m, in \u001b[0;36m_LazyModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1591\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_module(name)\n\u001b[1;32m 1592\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_class_to_module\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[0;32m-> 1593\u001b[0m module \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_module\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_class_to_module\u001b[49m\u001b[43m[\u001b[49m\u001b[43mname\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1594\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(module, name)\n\u001b[1;32m 1595\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/transformers/utils/import_utils.py:1603\u001b[0m, in \u001b[0;36m_LazyModule._get_module\u001b[0;34m(self, module_name)\u001b[0m\n\u001b[1;32m 1601\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_get_module\u001b[39m(\u001b[38;5;28mself\u001b[39m, module_name: \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m 1602\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1603\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mimportlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimport_module\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m.\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmodule_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;18;43m__name__\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1604\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 1605\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 1606\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to import \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodule_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m because of the following error (look up to see its\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1607\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m traceback):\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1608\u001b[0m ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/importlib/__init__.py:90\u001b[0m, in \u001b[0;36mimport_module\u001b[0;34m(name, package)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m 89\u001b[0m level \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m---> 90\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_bootstrap\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_gcd_import\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m[\u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpackage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/transformers/trainer.py:60\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m __version__\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfiguration_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PretrainedConfig\n\u001b[0;32m---> 60\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_collator\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DataCollator, DataCollatorWithPadding, default_data_collator\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdebug_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DebugOption, DebugUnderflowOverflow\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mfeature_extraction_sequence_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SequenceFeatureExtractor\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/transformers/data/__init__.py:27\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Copyright 2020 The HuggingFace Team. All rights reserved.\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;66;03m# Licensed under the Apache License, Version 2.0 (the \"License\");\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m# See the License for the specific language governing permissions and\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;66;03m# limitations under the License.\u001b[39;00m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata_collator\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m 16\u001b[0m DataCollatorForLanguageModeling,\n\u001b[1;32m 17\u001b[0m DataCollatorForPermutationLanguageModeling,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 25\u001b[0m default_data_collator,\n\u001b[1;32m 26\u001b[0m )\n\u001b[0;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m glue_compute_metrics, xnli_compute_metrics\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprocessors\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m 29\u001b[0m DataProcessor,\n\u001b[1;32m 30\u001b[0m InputExample,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 44\u001b[0m xnli_tasks_num_labels,\n\u001b[1;32m 45\u001b[0m )\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/transformers/data/metrics/__init__.py:19\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m is_sklearn_available, requires_backends\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_sklearn_available():\n\u001b[0;32m---> 19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mstats\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m pearsonr, spearmanr\n\u001b[1;32m 20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msklearn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m f1_score, matthews_corrcoef\n\u001b[1;32m 23\u001b[0m DEPRECATION_WARNING \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 24\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThis metric will be removed from the library soon, metrics should be handled with the ๐Ÿค— Evaluate \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlibrary. You can have a look at this example script for pointers: \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 26\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://github.com/huggingface/transformers/blob/main/examples/pytorch/text-classification/run_glue.py\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 27\u001b[0m )\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/scipy/stats/__init__.py:610\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;124;03m.. _statsrefmanual:\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 605\u001b[0m \n\u001b[1;32m 606\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m \u001b[38;5;66;03m# noqa: E501\u001b[39;00m\n\u001b[1;32m 608\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_warnings_errors\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (ConstantInputWarning, NearConstantInputWarning,\n\u001b[1;32m 609\u001b[0m DegenerateDataWarning, FitError)\n\u001b[0;32m--> 610\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_stats_py\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m 611\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_variation\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m variation\n\u001b[1;32m 612\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdistributions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/scipy/stats/_stats_py.py:49\u001b[0m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;66;03m# Import unused here but needs to stay until end of deprecation periode\u001b[39;00m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;66;03m# See https://github.com/scipy/scipy/issues/15765#issuecomment-1875564522\u001b[39;00m\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m linalg \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[0;32m---> 49\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m distributions\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _mstats_basic \u001b[38;5;28;01mas\u001b[39;00m mstats_basic\n\u001b[1;32m 52\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_stats_mstats_common\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _find_repeats, theilslopes, siegelslopes\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/scipy/stats/distributions.py:10\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# Author: Travis Oliphant 2002-2011 with contributions from\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;66;03m# SciPy Developers 2004-2011\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;66;03m# instead of `git blame -Lxxx,+x`.\u001b[39;00m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_distn_infrastructure\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (rv_discrete, rv_continuous, rv_frozen) \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[0;32m---> 10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _continuous_distns\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _discrete_distns\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_continuous_distns\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m \u001b[38;5;66;03m# noqa: F403\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/scipy/stats/_continuous_distns.py:12\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpolynomial\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Polynomial\n\u001b[0;32m---> 12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01minterpolate\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BSpline\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_lib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdoccer\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (extend_notes_in_docstring,\n\u001b[1;32m 14\u001b[0m replace_notes_in_docstring,\n\u001b[1;32m 15\u001b[0m inherit_docstring_from)\n\u001b[1;32m 16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_lib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_ccallback\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m LowLevelCallable\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/scipy/interpolate/__init__.py:167\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;124;03m========================================\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;124;03mInterpolation (:mod:`scipy.interpolate`)\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[38;5;124;03m(should not be used in new code).\u001b[39;00m\n\u001b[1;32m 166\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 167\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_interpolate\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_fitpack_py\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m 170\u001b[0m \u001b[38;5;66;03m# New interface to fitpack library:\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/scipy/interpolate/_interpolate.py:12\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_lib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_util\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m copy_if_needed\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mspecial\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m comb\n\u001b[0;32m---> 12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _fitpack_py\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_polyint\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _Interpolator1D\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _ppoly\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/scipy/interpolate/_fitpack_py.py:8\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# These are in the API for fitpack even if not used in fitpack.py itself.\u001b[39;00m\n\u001b[0;32m----> 8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_fitpack_impl\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m bisplrep, bisplev, dblint \u001b[38;5;66;03m# noqa: F401\u001b[39;00m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _fitpack_impl \u001b[38;5;28;01mas\u001b[39;00m _impl\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_bsplines\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BSpline\n", + "File \u001b[0;32m~/anaconda3/envs/simplifine/lib/python3.12/site-packages/scipy/interpolate/_fitpack_impl.py:35\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (atleast_1d, array, ones, zeros, sqrt, ravel, transpose,\n\u001b[1;32m 31\u001b[0m empty, iinfo, asarray)\n\u001b[1;32m 33\u001b[0m \u001b[38;5;66;03m# Try to replace _fitpack interface with\u001b[39;00m\n\u001b[1;32m 34\u001b[0m \u001b[38;5;66;03m# f2py-generated version\u001b[39;00m\n\u001b[0;32m---> 35\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _dfitpack \u001b[38;5;28;01mas\u001b[39;00m dfitpack\n\u001b[1;32m 38\u001b[0m dfitpack_int \u001b[38;5;241m=\u001b[39m dfitpack\u001b[38;5;241m.\u001b[39mtypes\u001b[38;5;241m.\u001b[39mintvar\u001b[38;5;241m.\u001b[39mdtype\n\u001b[1;32m 41\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_int_overflow\u001b[39m(x, exception, msg\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "from simplifine_alpha.train_utils import Client\n", + "import wandb\n", + "import os\n", + "\n", + "# Disabling WandB logging. Change this if you'd like to enable it.\n", + "# Note that you will need a WandB token if you enable logging.\n", + "wandb.init(mode='disabled')\n", + "\n", + "# Define your dataset template and response keys.\n", + "# Be sure to adjust the keys, response template, and dataset accordingly.\n", + "template = '''### TITLE: {title}\\n ### ABSTRACT: {abstract}\\n ###EXPLANATION: {explanation}'''\n", + "response_template = '\\n ###EXPLANATION:'\n", + "keys = ['title', 'abstract', 'explanation']\n", + "dataset_name = '' # Provide a Hugging Face dataset name if applicable.\n", + "\n", + "# Set the model name to LLaMA-3 8B. Note that larger models may cause OOM (Out of Memory) errors.\n", + "model_name = 'meta-llama/Meta-Llama-3-8B'\n", + "hf_token = '' # Insert your Hugging Face token here to access the LLaMA-3 model.\n", + "\n", + "from_hf = True # Set to False if using custom data.\n", + "\n", + "# Option to use your own dataset. Change `own_data` to True if you have custom data.\n", + "own_data = False\n", + "if own_data:\n", + " from_hf = False\n", + " data = {} # Insert your custom dataset here.\n", + "\n", + "# Set up the Simplifine client with your API key and GPU type.\n", + "simplifine_api_key = ''\n", + "gpu_type = 'a100' # Options are 'l4' or 'a100'\n", + "\n", + "client = Client(api_key=simplifine_api_key, gpu_type=gpu_type)\n", + "\n", + "# Start the training process for fine-tuning LLaMA-3 8B. Adjust parameters for parallelization if needed.\n", + "client.sft_train_cloud(\n", + " model_name=model_name, \n", + " from_hf=from_hf, \n", + " dataset_name=dataset_name,\n", + " keys=keys,\n", + " template=template, \n", + " job_name='ddp_job',\n", + " response_template=response_template, \n", + " use_zero=True, \n", + " use_ddp=False, \n", + " hf_token=hf_token\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ“Š Checking Job Status\n", + "\n", + "After initiating the fine-tuning process, you might want to check the status of your training jobs. The following code will help you extract and display the statuses of the most recent jobs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Retrieve the status of all jobs from the client.\n", + "status = client.get_all_jobs()\n", + "\n", + "# Display the status of the last 5 jobs.\n", + "for num, i in enumerate(status[-5:]):\n", + " print(f'Number {num} status: {i}\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿ’พ Downloading the Trained Model\n", + "\n", + "Once your fine-tuning job is complete, the next step is to download the trained model. Follow the steps below to create a folder and save the model locally." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'os' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[3], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# creating a folder to store the model\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mos\u001b[49m\u001b[38;5;241m.\u001b[39mmkdir(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msf_trained_model_ZeRO\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# download and save the model to it.\u001b[39;00m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;66;03m# This might take some time, have a sip of that coffee! :)\u001b[39;00m\n\u001b[1;32m 6\u001b[0m client\u001b[38;5;241m.\u001b[39mdownload_model(job_id\u001b[38;5;241m=\u001b[39mjob_id, extract_to\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/content/sf_trained_model_ZeRO\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", + "\u001b[0;31mNameError\u001b[0m: name 'os' is not defined" + ] + } + ], + "source": [ + "job_id = '' # Get your job ID from the list of job statuses above.\n", + "\n", + "# Create a folder to store the trained model.\n", + "os.mkdir('sf_trained_model_ZeRO')\n", + "\n", + "# Download and save the model to the specified folder.\n", + "# This might take some time, so relax and enjoy a cup of coffee! :)\n", + "client.download_model(job_id=job_id, extract_to='/content/sf_trained_model_ZeRO')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ๐Ÿงช Testing Your Fine-Tuned Model\n", + "\n", + "Now that you've downloaded your fine-tuned model, it's time to test it. We'll load the model and tokenizer using the `transformers` library and generate a sample output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoModelForCausalLM, AutoTokenizer\n", + "\n", + "# Define the path where the trained model is stored.\n", + "path = '/content/sf_trained_model_ZeRO'\n", + "\n", + "# Load the fine-tuned model and tokenizer.\n", + "sf_model = AutoModelForCausalLM.from_pretrained(path)\n", + "sf_tokenizer = AutoTokenizer.from_pretrained(path)\n", + "\n", + "# Create an example input for the model.\n", + "input_example = '''### TITLE: title 1\\n ### ABSTRACT: abstract 1\\n ###EXPLANATION: '''\n", + "\n", + "# Tokenize the input example.\n", + "input_example = sf_tokenizer(input_example, return_tensors='pt')\n", + "\n", + "# Generate output from the fine-tuned model.\n", + "output = sf_model.generate(input_example['input_ids'],\n", + " attention_mask=input_example['attention_mask'],\n", + " max_length=30,\n", + " eos_token_id=sf_tokenizer.eos_token_id,\n", + " early_stopping=True,\n", + " pad_token_id=sf_tokenizer.eos_token_id)\n", + "\n", + "# Decode and print the generated output.\n", + "print(sf_tokenizer.decode(output[0]))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "simplifine", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}