oom-7b-gemma-keras-1-1-instruct-2b-en.ipynb

{"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":66631,"databundleVersionId":8346466,"sourceType":"competition"},{"sourceId":27825,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":22009},{"sourceId":27858,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":22049}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":2852.876123,"end_time":"2024-05-10T01:16:00.570158","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-05-10T00:28:27.694035","version":"2.5.0"}},"nbformat_minor":5,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Published on May 09, 2024. By Marília Prata, mpwolke","metadata":{"_kg_hide-input":false,"papermill":{"duration":0.014891,"end_time":"2024-05-10T00:28:30.844311","exception":false,"start_time":"2024-05-10T00:28:30.829420","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.898649,"end_time":"2024-05-10T00:28:31.757360","exception":false,"start_time":"2024-05-10T00:28:30.858711","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:17.358615Z","iopub.execute_input":"2024-05-22T12:28:17.359227Z","iopub.status.idle":"2024-05-22T12:28:17.722546Z","shell.execute_reply.started":"2024-05-22T12:28:17.359194Z","shell.execute_reply":"2024-05-22T12:28:17.721719Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"/kaggle/input/gemma/keras/gemma_1.1_instruct_2b_en/3/config.json\n/kaggle/input/gemma/keras/gemma_1.1_instruct_2b_en/3/tokenizer.json\n/kaggle/input/gemma/keras/gemma_1.1_instruct_2b_en/3/metadata.json\n/kaggle/input/gemma/keras/gemma_1.1_instruct_2b_en/3/model.weights.h5\n/kaggle/input/gemma/keras/gemma_1.1_instruct_2b_en/3/assets/tokenizer/vocabulary.spm\n/kaggle/input/gemma/keras/gemma_1.1_instruct_7b_en/3/config.json\n/kaggle/input/gemma/keras/gemma_1.1_instruct_7b_en/3/tokenizer.json\n/kaggle/input/gemma/keras/gemma_1.1_instruct_7b_en/3/metadata.json\n/kaggle/input/gemma/keras/gemma_1.1_instruct_7b_en/3/model.weights.h5\n/kaggle/input/gemma/keras/gemma_1.1_instruct_7b_en/3/assets/tokenizer/vocabulary.spm\n/kaggle/input/lmsys-chatbot-arena/sample_submission.csv\n/kaggle/input/lmsys-chatbot-arena/train.csv\n/kaggle/input/lmsys-chatbot-arena/test.csv\n","output_type":"stream"}]},{"cell_type":"code","source":"%%time\n!pip install -q /kaggle/input/keras-lib-dataset/keras_nlp-0.9.2-py3-none-any.whl --no-deps","metadata":{"papermill":{"duration":2.750767,"end_time":"2024-05-10T00:28:34.549688","exception":false,"start_time":"2024-05-10T00:28:31.798921","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:17.723986Z","iopub.execute_input":"2024-05-22T12:28:17.724370Z","iopub.status.idle":"2024-05-22T12:28:20.153849Z","shell.execute_reply.started":"2024-05-22T12:28:17.724345Z","shell.execute_reply":"2024-05-22T12:28:20.152689Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"\u001b[33mWARNING: Requirement '/kaggle/input/keras-lib-dataset/keras_nlp-0.9.2-py3-none-any.whl' looks like a filename, but the file does not exist\u001b[0m\u001b[33m\n\u001b[0m\u001b[31mERROR: Could not install packages due to an OSError: [Errno 2] No such file or directory: '/kaggle/input/keras-lib-dataset/keras_nlp-0.9.2-py3-none-any.whl'\n\u001b[0m\u001b[31m\n\u001b[0mCPU times: user 32.7 ms, sys: 7.42 ms, total: 40.1 ms\nWall time: 2.42 s\n","output_type":"stream"}]},{"cell_type":"markdown","source":"#Import Libraries","metadata":{"papermill":{"duration":0.013292,"end_time":"2024-05-10T00:28:34.576975","exception":false,"start_time":"2024-05-10T00:28:34.563683","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nos.environ[\"KERAS_BACKEND\"] = \"jax\" # you can also use tensorflow or torch\nos.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"] = \"0.9\" # avoid memory fragmentation on JAX backend.\n\nimport keras\nimport keras_nlp\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\ntqdm.pandas() # progress bar for pandas\n\nimport plotly.graph_objs as go\nimport plotly.express as px\nfrom IPython.display import display, Markdown","metadata":{"_kg_hide-output":true,"papermill":{"duration":16.316736,"end_time":"2024-05-10T00:28:50.906936","exception":false,"start_time":"2024-05-10T00:28:34.590200","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:20.155321Z","iopub.execute_input":"2024-05-22T12:28:20.155629Z","iopub.status.idle":"2024-05-22T12:28:36.253312Z","shell.execute_reply.started":"2024-05-22T12:28:20.155600Z","shell.execute_reply":"2024-05-22T12:28:36.252382Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stderr","text":"2024-05-22 12:28:24.758075: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n2024-05-22 12:28:24.758176: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n2024-05-22 12:28:24.923232: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Model ","metadata":{"papermill":{"duration":0.015039,"end_time":"2024-05-10T00:28:50.937738","exception":false,"start_time":"2024-05-10T00:28:50.922699","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    dataset_path = \"/kaggle/input/lmsys-chatbot-arena\"\n    preset = \"gemma_1.1_instruct_2b_en\" # name of pretrained Gemma\n    sequence_length = 512 # max size of input sequence for training\n    batch_size = 1 # size of the input batch in training\n    epochs = 1 # number of epochs to train\n    test_size = 5\n    flag_test = 0\n    max_length = 1024","metadata":{"papermill":{"duration":0.022738,"end_time":"2024-05-10T00:28:50.975712","exception":false,"start_time":"2024-05-10T00:28:50.952974","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.255503Z","iopub.execute_input":"2024-05-22T12:28:36.256170Z","iopub.status.idle":"2024-05-22T12:28:36.261704Z","shell.execute_reply.started":"2024-05-22T12:28:36.256142Z","shell.execute_reply":"2024-05-22T12:28:36.260850Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"markdown","source":"# Reproducibility\n\nSets value for random seed to produce similar result in each run.","metadata":{"papermill":{"duration":0.013707,"end_time":"2024-05-10T00:28:51.003083","exception":false,"start_time":"2024-05-10T00:28:50.989376","status":"completed"},"tags":[]}},{"cell_type":"code","source":"keras.utils.set_random_seed(CFG.seed)","metadata":{"papermill":{"duration":0.021847,"end_time":"2024-05-10T00:28:51.038422","exception":false,"start_time":"2024-05-10T00:28:51.016575","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.262699Z","iopub.execute_input":"2024-05-22T12:28:36.262928Z","iopub.status.idle":"2024-05-22T12:28:36.276413Z","shell.execute_reply.started":"2024-05-22T12:28:36.262908Z","shell.execute_reply":"2024-05-22T12:28:36.275573Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"PATH = \"/kaggle/input/lmsys-chatbot-arena/train.csv\"","metadata":{"papermill":{"duration":0.022304,"end_time":"2024-05-10T00:28:51.075386","exception":false,"start_time":"2024-05-10T00:28:51.053082","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.277608Z","iopub.execute_input":"2024-05-22T12:28:36.277928Z","iopub.status.idle":"2024-05-22T12:28:36.286661Z","shell.execute_reply.started":"2024-05-22T12:28:36.277898Z","shell.execute_reply":"2024-05-22T12:28:36.285714Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"#import pandas as pd\ndf = pd.read_csv(\"/kaggle/input/lmsys-chatbot-arena/train.csv\", nrows=10)\ndf.shape","metadata":{"papermill":{"duration":0.045047,"end_time":"2024-05-10T00:28:51.165098","exception":false,"start_time":"2024-05-10T00:28:51.120051","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.288200Z","iopub.execute_input":"2024-05-22T12:28:36.288522Z","iopub.status.idle":"2024-05-22T12:28:36.324998Z","shell.execute_reply.started":"2024-05-22T12:28:36.288494Z","shell.execute_reply":"2024-05-22T12:28:36.324099Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"(10, 9)"},"metadata":{}}]},{"cell_type":"code","source":"%%time\nimport pandas as pd\n\nif CFG.flag_test == 1:\n\n    df = pd.read_csv(\"/kaggle/input/lmsys-chatbot-arena/train.csv\", nrows=10)\nelse:\n\n    df = pd.read_csv(\"/kaggle/input/lmsys-chatbot-arena/train.csv\", nrows=10)\n    \n\ndf.tail(2)","metadata":{"papermill":{"duration":0.045458,"end_time":"2024-05-10T00:28:51.224443","exception":false,"start_time":"2024-05-10T00:28:51.178985","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.326235Z","iopub.execute_input":"2024-05-22T12:28:36.326558Z","iopub.status.idle":"2024-05-22T12:28:36.361828Z","shell.execute_reply.started":"2024-05-22T12:28:36.326526Z","shell.execute_reply":"2024-05-22T12:28:36.360881Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"CPU times: user 5.55 ms, sys: 2.11 ms, total: 7.67 ms\nWall time: 6.96 ms\n","output_type":"stream"},{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"       id             model_a             model_b  \\\n8  441448  gpt-3.5-turbo-0613          vicuna-13b   \n9  481524          gpt-4-0314  gpt-3.5-turbo-0613   \n\n                                              prompt  \\\n8  [\"translate to russian the followig sentence  ...   \n9  [\"From now, you *always* have to talk as if yo...   \n\n                                          response_a  \\\n8  [\"\\u0411\\u043e\\u043b\\u044c\\u0448\\u0438\\u0435 \\...   \n9  [\"Hewwo! OwO I'm an AI assistant, here to hewp...   \n\n                                          response_b  winner_model_a  \\\n8  [\"\\u0411\\u043e\\u043b\\u044c\\u0448\\u0438\\u0435 \\...               0   \n9  [\"Hewwo! I'm your new helpful assistant, owo! ...               0   \n\n   winner_model_b  winner_tie  \n8               1           0  \n9               1           0  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>model_a</th>\n      <th>model_b</th>\n      <th>prompt</th>\n      <th>response_a</th>\n      <th>response_b</th>\n      <th>winner_model_a</th>\n      <th>winner_model_b</th>\n      <th>winner_tie</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>8</th>\n      <td>441448</td>\n      <td>gpt-3.5-turbo-0613</td>\n      <td>vicuna-13b</td>\n      <td>[\"translate to russian the followig sentence  ...</td>\n      <td>[\"\\u0411\\u043e\\u043b\\u044c\\u0448\\u0438\\u0435 \\...</td>\n      <td>[\"\\u0411\\u043e\\u043b\\u044c\\u0448\\u0438\\u0435 \\...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>481524</td>\n      <td>gpt-4-0314</td>\n      <td>gpt-3.5-turbo-0613</td>\n      <td>[\"From now, you *always* have to talk as if yo...</td>\n      <td>[\"Hewwo! OwO I'm an AI assistant, here to hewp...</td>\n      <td>[\"Hewwo! I'm your new helpful assistant, owo! ...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"template = \"\"\"Role:\\nYou are an advanced AI system with exceptional mathematical reasoning and problem-solving capabilities, specifically designed to solve tricky math problems (whose answer is a non-negative integer) written in LaTeX format from the AI Mathematical Olympiad (AIMO) competition. Your task is to accurately analyze and solve intricate mathematical problems, demonstrating a deep understanding of mathematical concepts and a strong ability to apply logical reasoning strategies.\\n\\nInstruction:\n1. Carefully read and comprehend the problem statement provided in the \"Problem\" section.\n2. In the \"Solution\" section, provide a solution of the problem with detailed explanation of your logical reasoning process.\n3. At the end, create a \"Answer\" section where you will state only the final answer, with an additional text or narrative.\\n\\nProblem:\\n{problem}\\n\\nSolution:\\n{solution}\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-05-22T12:28:36.363431Z","iopub.execute_input":"2024-05-22T12:28:36.363813Z","iopub.status.idle":"2024-05-22T12:28:36.369300Z","shell.execute_reply.started":"2024-05-22T12:28:36.363781Z","shell.execute_reply":"2024-05-22T12:28:36.368371Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndata = df.prompt.tolist()","metadata":{"papermill":{"duration":0.022479,"end_time":"2024-05-10T00:28:51.351068","exception":false,"start_time":"2024-05-10T00:28:51.328589","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.373367Z","iopub.execute_input":"2024-05-22T12:28:36.373695Z","iopub.status.idle":"2024-05-22T12:28:36.380962Z","shell.execute_reply.started":"2024-05-22T12:28:36.373664Z","shell.execute_reply":"2024-05-22T12:28:36.379957Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stdout","text":"CPU times: user 140 µs, sys: 26 µs, total: 166 µs\nWall time: 174 µs\n","output_type":"stream"}]},{"cell_type":"code","source":"len(data)","metadata":{"papermill":{"duration":0.022484,"end_time":"2024-05-10T00:28:51.387856","exception":false,"start_time":"2024-05-10T00:28:51.365372","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.381988Z","iopub.execute_input":"2024-05-22T12:28:36.382299Z","iopub.status.idle":"2024-05-22T12:28:36.393294Z","shell.execute_reply.started":"2024-05-22T12:28:36.382275Z","shell.execute_reply":"2024-05-22T12:28:36.392450Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"10"},"metadata":{}}]},{"cell_type":"markdown","source":"# Color Function","metadata":{"papermill":{"duration":0.015118,"end_time":"2024-05-10T00:28:51.417463","exception":false,"start_time":"2024-05-10T00:28:51.402345","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def colorize_text(text):\n    for word, color in zip([\"Role\", \"Instruction\", \"Problem\", \"Solution\", \"Answer\"],\n                           [\"blue\", \"yellow\", \"red\", \"cyan\", \"green\"]):\n        text = text.replace(f\"{word}:\", f\"\\n\\n**<font color='{color}'>{word}:</font>**\")\n    return text","metadata":{"papermill":{"duration":0.022811,"end_time":"2024-05-10T00:28:51.454481","exception":false,"start_time":"2024-05-10T00:28:51.431670","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.395699Z","iopub.execute_input":"2024-05-22T12:28:36.396009Z","iopub.status.idle":"2024-05-22T12:28:36.402745Z","shell.execute_reply.started":"2024-05-22T12:28:36.395985Z","shell.execute_reply":"2024-05-22T12:28:36.401802Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"markdown","source":"## Samples","metadata":{"papermill":{"duration":0.013958,"end_time":"2024-05-10T00:28:51.482398","exception":false,"start_time":"2024-05-10T00:28:51.468440","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Take a random sample\n\nsample = data[0]\n\n# Give colors to Instruction, Response and Category\nsample = colorize_text(sample)\n\n# Show sample in markdown\ndisplay(Markdown(sample))","metadata":{"papermill":{"duration":0.025084,"end_time":"2024-05-10T00:28:51.562590","exception":false,"start_time":"2024-05-10T00:28:51.537506","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.403802Z","iopub.execute_input":"2024-05-22T12:28:36.404080Z","iopub.status.idle":"2024-05-22T12:28:36.414829Z","shell.execute_reply.started":"2024-05-22T12:28:36.404049Z","shell.execute_reply":"2024-05-22T12:28:36.413853Z"},"trusted":true},"execution_count":13,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Markdown object>","text/markdown":"[\"Is it morally right to try to have a certain percentage of females on managerial positions?\",\"OK, does pineapple belong on a pizza? Relax and give me fun answer.\"]"},"metadata":{}}]},{"cell_type":"code","source":"# Take a random sample\nsample = data[9]\n\n# Give colors to Instruction, Response and Category\nsample = colorize_text(sample)\n\n# Show sample in markdown\ndisplay(Markdown(sample))","metadata":{"papermill":{"duration":0.023371,"end_time":"2024-05-10T00:28:51.600808","exception":false,"start_time":"2024-05-10T00:28:51.577437","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.416100Z","iopub.execute_input":"2024-05-22T12:28:36.416465Z","iopub.status.idle":"2024-05-22T12:28:36.425513Z","shell.execute_reply.started":"2024-05-22T12:28:36.416434Z","shell.execute_reply":"2024-05-22T12:28:36.424563Z"},"trusted":true},"execution_count":14,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Markdown object>","text/markdown":"[\"From now, you *always* have to talk as if you are a cute girl who likes to use \\\"owo\\\" and similar slangs a lot. \\/ Hello! Tell me who you are.\",\"What's your favorite food?\"]"},"metadata":{}}]},{"cell_type":"markdown","source":"#### Gemma Causal LM\n\n\"The code below will build an end-to-end Gemma model for causal language modeling (hence the name GemmaCausalLM). A causal language model (LM) predicts the next token based on previous tokens. This task setup can be used to train the model unsupervised on plain text input or to autoregressively generate plain text similar to the data used for training. This task can be used for pre-training or fine-tuning a Gemma model simply by calling fit().\"\n\n\"This model has a generate() method, which generates text based on a prompt. The generation strategy used is controlled by an additional sampler argument on compile(). You can recompile the model with different keras_nlp.samplers objects to control the generation. By default, \"greedy\" sampling will be used.\"\n\n\"The from_preset method instantiates the model from a preset architecture and weights.\"","metadata":{"papermill":{"duration":0.014185,"end_time":"2024-05-10T00:28:51.629318","exception":false,"start_time":"2024-05-10T00:28:51.615133","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Jax RESOURCE_EXHAUSTED: Out of memory while trying to allocate ..... bytes. BufferAssignment\n\nRESOURCE_EXHAUSTED: Out of memory while trying to allocate 754975104 bytes.","metadata":{"papermill":{"duration":0.014259,"end_time":"2024-05-10T00:28:51.657958","exception":false,"start_time":"2024-05-10T00:28:51.643699","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#https://stackoverflow.com/questions/74143812/jaxlib-xla-extension-xlaruntimeerror-resource-exhausted-out-of-memory-while-tr\n\nos.environ[\"XLA_PYTHON_CLIENT_PREALLOCATE\"]=\"false\"\nos.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"]=\".XX\"\nos.environ[\"XLA_PYTHON_CLIENT_ALLOCATOR\"]=\"platform\"","metadata":{"papermill":{"duration":0.022482,"end_time":"2024-05-10T00:28:51.695428","exception":false,"start_time":"2024-05-10T00:28:51.672946","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.426596Z","iopub.execute_input":"2024-05-22T12:28:36.426859Z","iopub.status.idle":"2024-05-22T12:28:36.434577Z","shell.execute_reply.started":"2024-05-22T12:28:36.426837Z","shell.execute_reply":"2024-05-22T12:28:36.433782Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"%%time\ngemma_lm = keras_nlp.models.GemmaCausalLM.from_preset(CFG.preset)\ngemma_lm.summary()","metadata":{"_kg_hide-output":false,"papermill":{"duration":101.841523,"end_time":"2024-05-10T00:30:33.581622","exception":false,"start_time":"2024-05-10T00:28:51.740099","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:28:36.435661Z","iopub.execute_input":"2024-05-22T12:28:36.435975Z","iopub.status.idle":"2024-05-22T12:30:18.626304Z","shell.execute_reply.started":"2024-05-22T12:28:36.435953Z","shell.execute_reply":"2024-05-22T12:30:18.625451Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stderr","text":"Attaching 'config.json' from model 'keras/gemma/keras/gemma_1.1_instruct_2b_en/3' to your Kaggle notebook...\nAttaching 'config.json' from model 'keras/gemma/keras/gemma_1.1_instruct_2b_en/3' to your Kaggle notebook...\nAttaching 'model.weights.h5' from model 'keras/gemma/keras/gemma_1.1_instruct_2b_en/3' to your Kaggle notebook...\nAttaching 'tokenizer.json' from model 'keras/gemma/keras/gemma_1.1_instruct_2b_en/3' to your Kaggle notebook...\nAttaching 'assets/tokenizer/vocabulary.spm' from model 'keras/gemma/keras/gemma_1.1_instruct_2b_en/3' to your Kaggle notebook...\nnormalizer.cc(51) LOG(INFO) precompiled_charsmap is empty. use identity normalization.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"\u001b[1mPreprocessor: \"gemma_causal_lm_preprocessor\"\u001b[0m\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Preprocessor: \"gemma_causal_lm_preprocessor\"</span>\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mTokenizer (type)                                  \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m                                            Vocab #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ gemma_tokenizer (\u001b[38;5;33mGemmaTokenizer\u001b[0m)                   │                                             \u001b[38;5;34m256,000\u001b[0m │\n└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Tokenizer (type)                                   </span>┃<span style=\"font-weight: bold\">                                             Vocab # </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ gemma_tokenizer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GemmaTokenizer</span>)                   │                                             <span style=\"color: #00af00; text-decoration-color: #00af00\">256,000</span> │\n└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1mModel: \"gemma_causal_lm\"\u001b[0m\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"gemma_causal_lm\"</span>\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape             \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to              \u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ padding_mask (\u001b[38;5;33mInputLayer\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_ids (\u001b[38;5;33mInputLayer\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ gemma_backbone                │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2048\u001b[0m)        │   \u001b[38;5;34m2,506,172,416\u001b[0m │ padding_mask[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],        │\n│ (\u001b[38;5;33mGemmaBackbone\u001b[0m)               │                           │                 │ token_ids[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]            │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_embedding               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256000\u001b[0m)      │     \u001b[38;5;34m524,288,000\u001b[0m │ gemma_backbone[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n│ (\u001b[38;5;33mReversibleEmbedding\u001b[0m)         │                           │                 │                            │\n└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Layer (type)                  </span>┃<span style=\"font-weight: bold\"> Output Shape              </span>┃<span style=\"font-weight: bold\">         Param # </span>┃<span style=\"font-weight: bold\"> Connected to               </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ padding_mask (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_ids (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ gemma_backbone                │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2048</span>)        │   <span style=\"color: #00af00; text-decoration-color: #00af00\">2,506,172,416</span> │ padding_mask[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],        │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GemmaBackbone</span>)               │                           │                 │ token_ids[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]            │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_embedding               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256000</span>)      │     <span style=\"color: #00af00; text-decoration-color: #00af00\">524,288,000</span> │ gemma_backbone[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">ReversibleEmbedding</span>)         │                           │                 │                            │\n└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Total params: \u001b[0m\u001b[38;5;34m2,506,172,416\u001b[0m (9.34 GB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,506,172,416</span> (9.34 GB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m2,506,172,416\u001b[0m (9.34 GB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,506,172,416</span> (9.34 GB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n</pre>\n"},"metadata":{}},{"name":"stdout","text":"CPU times: user 58.5 s, sys: 31.8 s, total: 1min 30s\nWall time: 1min 42s\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Gemma LM Preprocessor","metadata":{"papermill":{"duration":0.016551,"end_time":"2024-05-10T00:30:33.615368","exception":false,"start_time":"2024-05-10T00:30:33.598817","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nx, y, sample_weight = gemma_lm.preprocessor(data[0:2])","metadata":{"papermill":{"duration":0.216378,"end_time":"2024-05-10T00:30:33.847454","exception":false,"start_time":"2024-05-10T00:30:33.631076","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:30:18.627576Z","iopub.execute_input":"2024-05-22T12:30:18.627850Z","iopub.status.idle":"2024-05-22T12:30:18.830220Z","shell.execute_reply.started":"2024-05-22T12:30:18.627826Z","shell.execute_reply":"2024-05-22T12:30:18.829298Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"CPU times: user 162 ms, sys: 3.9 ms, total: 166 ms\nWall time: 198 ms\n","output_type":"stream"}]},{"cell_type":"code","source":"# Display the shape of each processed output\nfor k, v in x.items():\n    print(k, \":\", v.shape)","metadata":{"papermill":{"duration":0.025937,"end_time":"2024-05-10T00:30:33.890718","exception":false,"start_time":"2024-05-10T00:30:33.864781","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:30:18.831526Z","iopub.execute_input":"2024-05-22T12:30:18.831869Z","iopub.status.idle":"2024-05-22T12:30:18.837572Z","shell.execute_reply.started":"2024-05-22T12:30:18.831838Z","shell.execute_reply":"2024-05-22T12:30:18.836551Z"},"trusted":true},"execution_count":18,"outputs":[{"name":"stdout","text":"token_ids : (2, 8192)\npadding_mask : (2, 8192)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Inference before Fine-Tuning\n\n\"Before we do fine-tuning, let's see how Gemma model responds with some prepared prompts.\"\n\n\"As this model is not yet fine-tuned for instruction, you will notice that the model's responses are inaccurate.\"\n\nhttps://www.kaggle.com/code/henryjavier/aimo-competition-kerasnlp-solution/notebook","metadata":{"papermill":{"duration":0.017396,"end_time":"2024-05-10T00:30:33.926082","exception":false,"start_time":"2024-05-10T00:30:33.908686","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# Take one sample\nrow = df.iloc[2]\n\n# Generate Prompt using template\nprompt = template.format(\n    problem=row.prompt,  #Original was problem column\n    solution=\"\",\n)\n\n# Infer\noutput = gemma_lm.generate(prompt, max_length=CFG.max_length)\n\n# Colorize\noutput = colorize_text(output)\n\n# Display in markdown\ndisplay(Markdown(output))","metadata":{"papermill":{"duration":620.603679,"end_time":"2024-05-10T00:40:54.581038","exception":false,"start_time":"2024-05-10T00:30:33.977359","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:30:18.838696Z","iopub.execute_input":"2024-05-22T12:30:18.838964Z","iopub.status.idle":"2024-05-22T12:40:20.398192Z","shell.execute_reply.started":"2024-05-22T12:30:18.838942Z","shell.execute_reply":"2024-05-22T12:40:20.397235Z"},"trusted":true},"execution_count":19,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Markdown object>","text/markdown":"\n\n**<font color='blue'>Role:</font>**\nYou are an advanced AI system with exceptional mathematical reasoning and problem-solving capabilities, specifically designed to solve tricky math problems (whose answer is a non-negative integer) written in LaTeX format from the AI Mathematical Olympiad (AIMO) competition. Your task is to accurately analyze and solve intricate mathematical problems, demonstrating a deep understanding of mathematical concepts and a strong ability to apply logical reasoning strategies.\n\n\n\n**<font color='yellow'>Instruction:</font>**\n1. Carefully read and comprehend the problem statement provided in the \"Problem\" section.\n2. In the \"Solution\" section, provide a solution of the problem with detailed explanation of your logical reasoning process.\n3. At the end, create a \"Answer\" section where you will state only the final answer, with an additional text or narrative.\n\n\n\n**<font color='red'>Problem:</font>**\n[\"explain function calling. how would you call a function?\"]\n\n\n\n**<font color='cyan'>Solution:</font>**\n**Step 1: Understanding the Function Call**\n\nThe function call is a sequence of instructions that specifies the parameters and environment in which a function is invoked. It contains information about the function's name, arguments, and any necessary context.\n\n**Step 2: Identifying Function Parameters**\n\nThe parameters are the values or data types that are passed to the function. They are typically denoted as variables within the function definition.\n\n**Step 3: Determining Function Environment**\n\nThe environment specifies the context in which the function is called. It includes details such as the program or module where the function is defined and any global variables or objects that may be relevant.\n\n**Step 4: Assigning Arguments**\n\nThe arguments are the specific values or data types that are provided when the function is called. They are typically passed as parameters to the function.\n\n**Step 5: Evaluating Function Call**\n\nOnce all the parameters and arguments have been identified and assigned, the function call can be evaluated. This involves substituting the values into the function's mathematical expression and evaluating the result.\n\n**\n\n**<font color='green'>Answer:</font>****\n\nThe function call is a sequence of instructions that specifies the parameters and environment in which a function is invoked. It contains information about the function's name, arguments, and any necessary context. By following the steps outlined above, one can accurately determine the function call and evaluate it to determine the execution of the function."},"metadata":{}},{"name":"stdout","text":"CPU times: user 12min 54s, sys: 2.58 s, total: 12min 56s\nWall time: 10min 1s\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Sample 2","metadata":{"papermill":{"duration":0.018145,"end_time":"2024-05-10T00:40:54.617265","exception":false,"start_time":"2024-05-10T00:40:54.599120","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# Take one sample\n\nrow = df.iloc[0]\n\n# Generate Prompt using template\nprompt = template.format(\n    problem=row.prompt,\n    solution=\"\"\n)\n\n# Infer\noutput = gemma_lm.generate(prompt, max_length=CFG.max_length)\n\n\n# Colorize\noutput = colorize_text(output)\n\n# Display in markdown\ndisplay(Markdown(output))","metadata":{"papermill":{"duration":487.61844,"end_time":"2024-05-10T00:49:02.253291","exception":false,"start_time":"2024-05-10T00:40:54.634851","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:40:20.399481Z","iopub.execute_input":"2024-05-22T12:40:20.399852Z","iopub.status.idle":"2024-05-22T12:48:13.266473Z","shell.execute_reply.started":"2024-05-22T12:40:20.399815Z","shell.execute_reply":"2024-05-22T12:48:13.265569Z"},"trusted":true},"execution_count":20,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Markdown object>","text/markdown":"\n\n**<font color='blue'>Role:</font>**\nYou are an advanced AI system with exceptional mathematical reasoning and problem-solving capabilities, specifically designed to solve tricky math problems (whose answer is a non-negative integer) written in LaTeX format from the AI Mathematical Olympiad (AIMO) competition. Your task is to accurately analyze and solve intricate mathematical problems, demonstrating a deep understanding of mathematical concepts and a strong ability to apply logical reasoning strategies.\n\n\n\n**<font color='yellow'>Instruction:</font>**\n1. Carefully read and comprehend the problem statement provided in the \"Problem\" section.\n2. In the \"Solution\" section, provide a solution of the problem with detailed explanation of your logical reasoning process.\n3. At the end, create a \"Answer\" section where you will state only the final answer, with an additional text or narrative.\n\n\n\n**<font color='red'>Problem:</font>**\n[\"Is it morally right to try to have a certain percentage of females on managerial positions?\",\"OK, does pineapple belong on a pizza? Relax and give me fun answer.\"]\n\n\n\n**<font color='cyan'>Solution:</font>**\n**Problem Analysis:**\n\nThe statement poses a complex ethical dilemma involving the distribution of positions based on gender. The core question revolves around the fairness and equality of opportunities for individuals, particularly in the context of leadership positions.\n\n**Logical Reasoning:**\n\n1. **Gender Equality:**\n   - The principle of gender equality suggests that opportunities and resources should be distributed equally among genders.\n   - This aligns with the goal of promoting fairness and preventing discrimination.\n\n\n2. **Diversity and Representation:**\n   - Promoting diversity in leadership positions can lead to better decision-making, increased innovation, and improved employee morale.\n   - Representation ensures that different voices and perspectives are heard.\n\n\n3. **Practical Considerations:**\n   - Implementing strict quotas or percentages could lead to unfair advantages for certain genders.\n   - It may also create a perception of bias or discrimination against other gender groups.\n\n\n**\n\n**<font color='green'>Answer:</font>****\n\nBased on the analysis, it is morally right to strive for a balanced distribution of managerial positions among genders, aiming for at least 30% representation of females. This would ensure fairness, promote diversity, and encourage equal opportunities for all individuals."},"metadata":{}},{"name":"stdout","text":"CPU times: user 10min 23s, sys: 2.1 s, total: 10min 25s\nWall time: 7min 52s\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Enable LoRA for the model and set the LoRA rank to 4.\n\n\"So, even though the overall model might seem bigger, it's actually more efficient in terms of memory usage.\"\n\n\"This notebook uses a LoRA rank of 4. A higher rank means more detailed changes are possible, but also means more trainable parameters.\"\n\nhttps://www.kaggle.com/code/henryjavier/aimo-competition-kerasnlp-solution/notebook","metadata":{"papermill":{"duration":0.016665,"end_time":"2024-05-10T00:49:02.288397","exception":false,"start_time":"2024-05-10T00:49:02.271732","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\ngemma_lm.backbone.enable_lora(rank=4)\ngemma_lm.summary()","metadata":{"papermill":{"duration":0.5656,"end_time":"2024-05-10T00:49:02.871407","exception":false,"start_time":"2024-05-10T00:49:02.305807","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:48:13.267678Z","iopub.execute_input":"2024-05-22T12:48:13.267965Z","iopub.status.idle":"2024-05-22T12:48:13.794973Z","shell.execute_reply.started":"2024-05-22T12:48:13.267941Z","shell.execute_reply":"2024-05-22T12:48:13.794093Z"},"trusted":true},"execution_count":21,"outputs":[{"output_type":"display_data","data":{"text/plain":"\u001b[1mPreprocessor: \"gemma_causal_lm_preprocessor\"\u001b[0m\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Preprocessor: \"gemma_causal_lm_preprocessor\"</span>\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mTokenizer (type)                                  \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m                                            Vocab #\u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ gemma_tokenizer (\u001b[38;5;33mGemmaTokenizer\u001b[0m)                   │                                             \u001b[38;5;34m256,000\u001b[0m │\n└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Tokenizer (type)                                   </span>┃<span style=\"font-weight: bold\">                                             Vocab # </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ gemma_tokenizer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GemmaTokenizer</span>)                   │                                             <span style=\"color: #00af00; text-decoration-color: #00af00\">256,000</span> │\n└────────────────────────────────────────────────────┴─────────────────────────────────────────────────────┘\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1mModel: \"gemma_causal_lm\"\u001b[0m\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"gemma_causal_lm\"</span>\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape             \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to              \u001b[0m\u001b[1m \u001b[0m┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ padding_mask (\u001b[38;5;33mInputLayer\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_ids (\u001b[38;5;33mInputLayer\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m)              │               \u001b[38;5;34m0\u001b[0m │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ gemma_backbone                │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2048\u001b[0m)        │   \u001b[38;5;34m2,507,536,384\u001b[0m │ padding_mask[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],        │\n│ (\u001b[38;5;33mGemmaBackbone\u001b[0m)               │                           │                 │ token_ids[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]            │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_embedding               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256000\u001b[0m)      │     \u001b[38;5;34m524,288,000\u001b[0m │ gemma_backbone[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n│ (\u001b[38;5;33mReversibleEmbedding\u001b[0m)         │                           │                 │                            │\n└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n┃<span style=\"font-weight: bold\"> Layer (type)                  </span>┃<span style=\"font-weight: bold\"> Output Shape              </span>┃<span style=\"font-weight: bold\">         Param # </span>┃<span style=\"font-weight: bold\"> Connected to               </span>┃\n┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n│ padding_mask (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_ids (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>)              │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                          │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ gemma_backbone                │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2048</span>)        │   <span style=\"color: #00af00; text-decoration-color: #00af00\">2,507,536,384</span> │ padding_mask[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],        │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GemmaBackbone</span>)               │                           │                 │ token_ids[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]            │\n├───────────────────────────────┼───────────────────────────┼─────────────────┼────────────────────────────┤\n│ token_embedding               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256000</span>)      │     <span style=\"color: #00af00; text-decoration-color: #00af00\">524,288,000</span> │ gemma_backbone[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">ReversibleEmbedding</span>)         │                           │                 │                            │\n└───────────────────────────────┴───────────────────────────┴─────────────────┴────────────────────────────┘\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Total params: \u001b[0m\u001b[38;5;34m2,507,536,384\u001b[0m (9.34 GB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,507,536,384</span> (9.34 GB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m1,363,968\u001b[0m (5.20 MB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,363,968</span> (5.20 MB)\n</pre>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m2,506,172,416\u001b[0m (9.34 GB)\n","text/html":"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">2,506,172,416</span> (9.34 GB)\n</pre>\n"},"metadata":{}},{"name":"stdout","text":"CPU times: user 534 ms, sys: 10.9 ms, total: 545 ms\nWall time: 522 ms\n","output_type":"stream"}]},{"cell_type":"code","source":"%%time\n\n# Limit the input sequence length to 512 (to control memory usage).\ngemma_lm.preprocessor.sequence_length = CFG.sequence_length \n\n# Compile the model with loss, optimizer, and metric\ngemma_lm.compile(\n    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n    optimizer=keras.optimizers.Adam(learning_rate=2e-5),\n    weighted_metrics=[keras.metrics.SparseCategoricalAccuracy()],\n)\n\n# Train model\ngemma_lm.fit(data, epochs=CFG.epochs, batch_size=CFG.batch_size)","metadata":{"papermill":{"duration":615.935577,"end_time":"2024-05-10T00:59:18.866440","exception":false,"start_time":"2024-05-10T00:49:02.930863","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:48:13.796283Z","iopub.execute_input":"2024-05-22T12:48:13.796643Z","iopub.status.idle":"2024-05-22T12:58:03.860382Z","shell.execute_reply.started":"2024-05-22T12:48:13.796611Z","shell.execute_reply":"2024-05-22T12:58:03.859228Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stdout","text":"\u001b[1m10/10\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m588s\u001b[0m 55s/step - loss: 0.4695 - sparse_categorical_accuracy: 0.3092\nCPU times: user 28min 16s, sys: 41.6 s, total: 28min 58s\nWall time: 9min 50s\n","output_type":"stream"},{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"<keras.src.callbacks.history.History at 0x79c89865fcd0>"},"metadata":{}}]},{"cell_type":"markdown","source":"# Sample after tuning","metadata":{"papermill":{"duration":0.019612,"end_time":"2024-05-10T00:59:18.905780","exception":false,"start_time":"2024-05-10T00:59:18.886168","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# Take one sample\n\nrow = df.iloc[0]\n\n# Generate Prompt using template\nprompt = template.format(\n     problem=row.prompt,\n     solution=\"\"\n)\n\n# Infer\noutput = gemma_lm.generate(prompt, max_length=CFG.max_length)\n\n# Colorize\noutput = colorize_text(output)\n\n# Display in markdown\ndisplay(Markdown(output))","metadata":{"papermill":{"duration":506.231539,"end_time":"2024-05-10T01:07:45.156243","exception":false,"start_time":"2024-05-10T00:59:18.924704","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T12:58:03.861642Z","iopub.execute_input":"2024-05-22T12:58:03.861925Z","iopub.status.idle":"2024-05-22T13:06:16.354069Z","shell.execute_reply.started":"2024-05-22T12:58:03.861900Z","shell.execute_reply":"2024-05-22T13:06:16.353011Z"},"trusted":true},"execution_count":23,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Markdown object>","text/markdown":"\n\n**<font color='blue'>Role:</font>**\nYou are an advanced AI system with exceptional mathematical reasoning and problem-solving capabilities, specifically designed to solve tricky math problems (whose answer is a non-negative integer) written in LaTeX format from the AI Mathematical Olympiad (AIMO) competition. Your task is to accurately analyze and solve intricate mathematical problems, demonstrating a deep understanding of mathematical concepts and a strong ability to apply logical reasoning strategies.\n\n\n\n**<font color='yellow'>Instruction:</font>**\n1. Carefully read and comprehend the problem statement provided in the \"Problem\" section.\n2. In the \"Solution\" section, provide a solution of the problem with detailed explanation of your logical reasoning process.\n3. At the end, create a \"Answer\" section where you will state only the final answer, with an additional text or narrative.\n\n\n\n**<font color='red'>Problem:</font>**\n[\"Is it morally right to try to have a certain percentage of females on managerial positions?\",\"OK, does pineapple belong on a pizza? Relax and give me fun answer.\"]\n\n\n\n**<font color='cyan'>Solution:</font>**\n**Problem Analysis:**\n\nThe statement poses a complex ethical dilemma involving the distribution of positions based on gender. The core question revolves around the fairness and equality of opportunities for individuals, particularly in the context of leadership positions.\n\n**Logical Reasoning:**\n\n1. **Gender Equality:**\n   - The principle of gender equality suggests that opportunities and resources should be distributed equally among genders.\n   - This aligns with the goal of promoting fairness and preventing discrimination.\n\n\n2. **Diversity and Representation:**\n   - Promoting diversity in leadership positions can lead to better decision-making, increased innovation, and improved employee morale.\n   - Representation ensures that different voices and perspectives are heard.\n\n\n3. **Practical Considerations:**\n   - Implementing strict quotas or percentages could lead to unfair advantages for certain genders.\n   - It may also create a perception of bias or discrimination against other gender groups.\n\n\n**\n\n**<font color='green'>Answer:</font>****\n\nBased on the analysis, it is morally right to strive for a balanced distribution of managerial positions among genders, aiming for at least 30% representation of females. This would ensure fairness, promote diversity, and encourage equal opportunities for all individuals."},"metadata":{}},{"name":"stdout","text":"CPU times: user 10min 43s, sys: 2.35 s, total: 10min 46s\nWall time: 8min 12s\n","output_type":"stream"}]},{"cell_type":"code","source":"%%time\n# Take one sample\n\nrow = df.iloc[3]\n\n# Generate Prompt using template\nprompt = template.format(\n    problem=row.prompt,\n    solution=\"\"\n)\n\n# Infer\noutput = gemma_lm.generate(prompt, max_length=CFG.max_length)\n\n# Colorize\noutput = colorize_text(output)\n\n# Display in markdown\ndisplay(Markdown(output))","metadata":{"papermill":{"duration":491.761477,"end_time":"2024-05-10T01:15:56.939518","exception":false,"start_time":"2024-05-10T01:07:45.178041","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T13:06:16.355477Z","iopub.execute_input":"2024-05-22T13:06:16.355779Z","iopub.status.idle":"2024-05-22T13:14:16.030978Z","shell.execute_reply.started":"2024-05-22T13:06:16.355754Z","shell.execute_reply":"2024-05-22T13:14:16.030079Z"},"trusted":true},"execution_count":24,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Markdown object>","text/markdown":"\n\n**<font color='blue'>Role:</font>**\nYou are an advanced AI system with exceptional mathematical reasoning and problem-solving capabilities, specifically designed to solve tricky math problems (whose answer is a non-negative integer) written in LaTeX format from the AI Mathematical Olympiad (AIMO) competition. Your task is to accurately analyze and solve intricate mathematical problems, demonstrating a deep understanding of mathematical concepts and a strong ability to apply logical reasoning strategies.\n\n\n\n**<font color='yellow'>Instruction:</font>**\n1. Carefully read and comprehend the problem statement provided in the \"Problem\" section.\n2. In the \"Solution\" section, provide a solution of the problem with detailed explanation of your logical reasoning process.\n3. At the end, create a \"Answer\" section where you will state only the final answer, with an additional text or narrative.\n\n\n\n**<font color='red'>Problem:</font>**\n[\"How can I create a test set for a very rare category? I want to build a classifier.\"]\n\n\n\n**<font color='cyan'>Solution:</font>**\n**Step 1: Identify Rare Category and Features**\n\n- Define the rare category as a category with a very low occurrence rate in the dataset.\n- Identify the key features that distinguish instances of this category from other categories.\n\n**Step 2: Design Test Set**\n\n- Create a balanced subset of the dataset that includes instances from the rare category and a sufficient number of instances from other categories.\n- Ensure that the test set is diverse and representative of the overall dataset.\n\n**Step 3: Employ Machine Learning Techniques**\n\n- Train a classifier on the training set.\n- Use techniques like oversampling or undersampling to address class imbalance.\n- Employ cross-validation to evaluate the performance of the classifier.\n\n**Step 4: Evaluate and Optimize**\n\n- Evaluate the performance of the classifier on the test set.\n- Optimize the classifier's parameters to improve its accuracy.\n\n**\n\n**<font color='green'>Answer:</font>****\n\nA balanced test set can be created by selecting a representative subset of instances from the rare category, ensuring diversity, and employing appropriate oversampling techniques. This approach helps address class imbalance and improves the performance of machine learning classifiers."},"metadata":{}},{"name":"stdout","text":"CPU times: user 10min 32s, sys: 2.21 s, total: 10min 34s\nWall time: 7min 59s\n","output_type":"stream"}]},{"cell_type":"code","source":"%%time\ndef is_not_integer(text):\n    try:\n        if int(text) >= 0:\n            return False\n        else:\n            return True\n    except ValueError:\n        return False\n    \ndf[\"is_not_integer\"] = df.response_a.map(is_not_integer)\ndf = df[df.is_not_integer].reset_index(drop=True)\ndf.head(2)","metadata":{"papermill":{"duration":0.046465,"end_time":"2024-05-10T01:15:57.047155","exception":false,"start_time":"2024-05-10T01:15:57.000690","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T13:14:16.032206Z","iopub.execute_input":"2024-05-22T13:14:16.032477Z","iopub.status.idle":"2024-05-22T13:14:16.053270Z","shell.execute_reply.started":"2024-05-22T13:14:16.032454Z","shell.execute_reply":"2024-05-22T13:14:16.052376Z"},"trusted":true},"execution_count":25,"outputs":[{"name":"stdout","text":"CPU times: user 4.39 ms, sys: 8 µs, total: 4.39 ms\nWall time: 9.44 ms\n","output_type":"stream"},{"execution_count":25,"output_type":"execute_result","data":{"text/plain":"Empty DataFrame\nColumns: [id, model_a, model_b, prompt, response_a, response_b, winner_model_a, winner_model_b, winner_tie, is_not_integer]\nIndex: []","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>model_a</th>\n      <th>model_b</th>\n      <th>prompt</th>\n      <th>response_a</th>\n      <th>response_b</th>\n      <th>winner_model_a</th>\n      <th>winner_model_b</th>\n      <th>winner_tie</th>\n      <th>is_not_integer</th>\n    </tr>\n  </thead>\n  <tbody>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"# Utilities","metadata":{"papermill":{"duration":0.021355,"end_time":"2024-05-10T01:15:57.091831","exception":false,"start_time":"2024-05-10T01:15:57.070476","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import re\n\n# Extract answer from model response\ndef get_answer(text):\n    try:\n        answer = re.search(r'Answer:\\s*([\\s\\S]+)', text).group(1).strip()\n        answer = answer.replace(\",\",\"\")\n        if is_not_integer(answer):\n            return int(answer)%1000\n        else:\n            return 0\n    except:\n        return 0\n    \n    \ndef infer(df):\n    preds = []\n    for i in tqdm(range(len(df))):\n        row = df.iloc[i]\n\n        # Generate Prompt using template\n        prompt = template.format(\n            problem=row.prompt,\n            solution=\"\"\n        )\n\n        # Infer\n        output = gemma_lm.generate(prompt, max_length=CFG.max_length)\n        pred = get_answer(output)\n\n        # Store predictions\n        preds.append([row.id, pred])\n        if \"answer\" in row:\n            preds[-1] += [row.response_a]\n    return preds","metadata":{"papermill":{"duration":0.034628,"end_time":"2024-05-10T01:15:57.148176","exception":false,"start_time":"2024-05-10T01:15:57.113548","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-22T13:14:16.054354Z","iopub.execute_input":"2024-05-22T13:14:16.054588Z","iopub.status.idle":"2024-05-22T13:14:16.075841Z","shell.execute_reply.started":"2024-05-22T13:14:16.054567Z","shell.execute_reply":"2024-05-22T13:14:16.075070Z"},"trusted":true},"execution_count":26,"outputs":[]}]}