conservision-tpu-michaelye (1).ipynb

{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":7572583,"sourceType":"datasetVersion","datasetId":4408484},{"sourceId":7572838,"sourceType":"datasetVersion","datasetId":4404597},{"sourceId":7602506,"sourceType":"datasetVersion","datasetId":4425871},{"sourceId":7633561,"sourceType":"datasetVersion","datasetId":4448201}],"dockerImageVersionId":30647,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Welcome to my notebook!","metadata":{}},{"cell_type":"markdown","source":"Default code from Kaggle Notebook:","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n   \nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames: \n        pass\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n   ","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:14.994983Z","iopub.execute_input":"2024-02-21T19:07:14.995308Z","iopub.status.idle":"2024-02-21T19:07:20.464351Z","shell.execute_reply.started":"2024-02-21T19:07:14.995266Z","shell.execute_reply":"2024-02-21T19:07:20.463658Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stderr","text":"/tmp/ipykernel_6470/3621782286.py:6: DeprecationWarning: \nPyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0),\n(to allow more performant data types, such as the Arrow string type, and better interoperability with other libraries)\nbut was not found to be installed on your system.\nIf this would cause problems for you,\nplease provide us feedback at https://github.com/pandas-dev/pandas/issues/54466\n        \n  import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Displaying some important libraries:","metadata":{}},{"cell_type":"code","source":"# import tensorflow as tf\n# print(\"Tensorflow:\", tf.__version__)\n\n# import kerastuner as kt\n# print(\"kerastuner:\", kt.__version__)\n\n# import keras_tuner as kt2\n# print(\"keras_tuner:\", kt2.__version__)\n\n# import platform\n# print(\"Python:\", platform.python_version())\n\n# import numpy as np\n# print(\"numpy:\", np.__version__)\n\n# import pandas as pd\n# print(\"pandas:\", pd.__version__)\n\n# import sklearn\n# print(\"sklearn version:\", sklearn.__version__)\n\n# import sklearn\n# print(\"sklearn path:\", sklearn.__path__)\n\n# import matplotlib\n# print(\"matplotlib:\", matplotlib.__version__)\n\n# import seaborn as sns\n# print(\"seaborn:\", sns.__version__)\n\n# # On WSL\n\n# # 2024-01-30 11:17:52.768682: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n# # 2024-01-30 11:17:53.149956: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n# # 2024-01-30 11:17:53.150001: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n# # 2024-01-30 11:17:53.210606: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n# # 2024-01-30 11:17:53.339576: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n# # To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n# # 2024-01-30 11:17:54.568146: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n# # Tensorflow: 2.15.0\n# # /tmp/ipykernel_3814/2917868046.py:4: DeprecationWarning: `import kerastuner` is deprecated, please use `import keras_tuner`.\n# #   import kerastuner as kt\n# # kerastuner: 1.0.5\n# # keras_tuner: 1.3.5\n# # Python: 3.10.12\n# # numpy: 1.24.3\n# # pandas: 2.1.4\n# # sklearn version: 1.2.2\n# # sklearn path: ['/home/michaelye22/.local/lib/python3.10/site-packages/sklearn']\n# # matplotlib: 3.8.2\n# # seaborn: 0.13.0\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:20.465773Z","iopub.execute_input":"2024-02-21T19:07:20.466086Z","iopub.status.idle":"2024-02-21T19:07:23.238343Z","shell.execute_reply.started":"2024-02-21T19:07:20.466060Z","shell.execute_reply":"2024-02-21T19:07:23.237652Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"markdown","source":"## Set Global random seed to make sure we can replicate any model that we create (no randomness)","metadata":{}},{"cell_type":"code","source":"import random\nimport tensorflow as tf\nimport numpy as np\nimport os\n\n\n\nnp.random.seed(42)\nrandom.seed(42)\nnp.random.seed(42)\ntf.random.set_seed(42)\n\nos.environ['TF_DETERMINISTIC_OPS'] = '1'","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:23.239234Z","iopub.execute_input":"2024-02-21T19:07:23.239475Z","iopub.status.idle":"2024-02-21T19:07:26.313719Z","shell.execute_reply.started":"2024-02-21T19:07:23.239450Z","shell.execute_reply":"2024-02-21T19:07:26.312911Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":"## Use the TPU instead of GPU","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    raise BaseException('ERROR: Not connected to a TPU runtime!')\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.TPUStrategy(tpu)  # Updated to the non-experimental version\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:26.314589Z","iopub.execute_input":"2024-02-21T19:07:26.315004Z","iopub.status.idle":"2024-02-21T19:07:33.335070Z","shell.execute_reply.started":"2024-02-21T19:07:26.314976Z","shell.execute_reply":"2024-02-21T19:07:33.334001Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"Running on TPU  \nINFO:tensorflow:Deallocate tpu buffers before initializing tpu system.\nINFO:tensorflow:Initializing the TPU system: local\n","output_type":"stream"},{"name":"stderr","text":"2024-02-21 19:07:28.947152: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.947280: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.947370: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.947453: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.947529: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.947754: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.947846: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.947933: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.948026: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.948117: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.948346: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.948454: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.948553: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.948665: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.948814: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949052: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949146: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949220: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949319: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949411: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949659: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949770: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949859: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.949953: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.950062: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.950360: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.950463: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.950564: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.950666: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.950768: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951037: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951133: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951234: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951322: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951407: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951673: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951784: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951884: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.951983: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:28.952072: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n","output_type":"stream"},{"name":"stdout","text":"INFO:tensorflow:Finished initializing TPU system.\nINFO:tensorflow:Found TPU system:\nINFO:tensorflow:*** Num TPU Cores: 8\nINFO:tensorflow:*** Num TPU Workers: 1\nINFO:tensorflow:*** Num TPU Cores Per Worker: 8\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:0, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:1, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:2, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:3, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:4, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:5, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:6, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU:7, TPU, 0, 0)\nINFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# **Data Preprocessing:**","metadata":{}},{"cell_type":"markdown","source":"### Loading the training and testing data","metadata":{}},{"cell_type":"markdown","source":"train_values are the features (X), and train_labels is the target/label (Y)","metadata":{}},{"cell_type":"code","source":"train_X = pd.read_csv(\"/kaggle/input/conser-vision-data/train_features.csv\")\ntrain_y = pd.read_csv(\"/kaggle/input/conser-vision-data/train_labels.csv\")\n\ntest_values = pd.read_csv(\"/kaggle/input/conser-vision-data/test_features.csv\")\n\n# print(\"train labels:\\n\", train_Y.head())\n\n# print(\"train values:\\n\", train_X.head())\n      \n# print(\"test_values:\\n\", test_values.head())","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:33.337148Z","iopub.execute_input":"2024-02-21T19:07:33.337418Z","iopub.status.idle":"2024-02-21T19:07:37.945212Z","shell.execute_reply.started":"2024-02-21T19:07:33.337389Z","shell.execute_reply":"2024-02-21T19:07:37.944072Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":"### Check to see if there are any missing values in the data. If so, we have to do imputation","metadata":{}},{"cell_type":"code","source":"missing_train_X = train_X.isnull().sum().sum()\nprint(\"Number of missing values in train_X:\", missing_train_X)\n\nmissing_train_y = train_y.isnull().sum().sum()\nprint(\"Number of missing values in train_Y:\", missing_train_y)\n\nmissing_test_values = test_values.isnull().sum().sum()\nprint(\"Number of missing values in test_values:\", missing_test_values )","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:37.946246Z","iopub.execute_input":"2024-02-21T19:07:37.946532Z","iopub.status.idle":"2024-02-21T19:07:38.033391Z","shell.execute_reply.started":"2024-02-21T19:07:37.946503Z","shell.execute_reply":"2024-02-21T19:07:38.032571Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"Number of missing values in train_X: 0\nNumber of missing values in train_Y: 0\nNumber of missing values in test_values: 0\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Since we have 0 missing values in each dataframe, we don't have to do imputation!","metadata":{}},{"cell_type":"markdown","source":"## Stratified train_test split","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\n\n# Load the datasets\n\ntrain_features = train_X\ntrain_labels = train_y\n\n# Merge the datasets on 'id'\nmerged_data = pd.merge(train_features, train_labels, on='id')\n\n# Identifying the animal present in each image and creating a combined category\nanimal_columns = ['antelope_duiker', 'bird', 'blank', 'civet_genet', 'hog', 'leopard', 'monkey_prosimian', 'rodent']\nmerged_data['animal'] = merged_data[animal_columns].idxmax(axis=1)\nmerged_data['site_animal'] = merged_data['site'] + '_' + merged_data['animal']\n\n# Checking the number of instances for each site_animal combination\ncombination_counts = merged_data['site_animal'].value_counts()\nrare_combinations = combination_counts[combination_counts < 5]\n\n# Separating the dataset into common and rare combinations\ncommon_combinations = merged_data[~merged_data['site_animal'].isin(rare_combinations.index)]\nrare_combinations_data = merged_data[merged_data['site_animal'].isin(rare_combinations.index)]\n\n\n\n# Stratified split for common combinations into train and temp sets\ncommon_train_set, common_temp_set = train_test_split(\n    common_combinations, test_size=0.3, stratify=common_combinations['site_animal'], random_state=42)\n\n# Check if each class has at least two instances\nclass_counts = common_temp_set['site_animal'].value_counts()\nsingle_instance_classes = class_counts[class_counts < 2]\n\n# Separate single instance classes\nsingle_instance_data = common_temp_set[common_temp_set['site_animal'].isin(single_instance_classes.index)]\ncommon_temp_set = common_temp_set[~common_temp_set['site_animal'].isin(single_instance_classes.index)]\n\n# Stratified split for common combinations into validation and test sets\ncommon_val_set, common_test_set = train_test_split(\n    common_temp_set, test_size=0.5, stratify=common_temp_set['site_animal'], random_state=42)\n\n# Add single instance classes to the training set\ncommon_train_set = pd.concat([common_train_set, single_instance_data])\n\n\n\n# Randomly splitting rare combinations into train and temp sets\ntotal_samples = rare_combinations_data.shape[0]\ntrain_samples = int(np.round(total_samples * 0.7))\nrare_train_set = rare_combinations_data.sample(n=train_samples, random_state=42)\nrare_temp_set = rare_combinations_data.drop(rare_train_set.index)\n\n# Randomly splitting rare combinations into validation and test sets\ntotal_samples = rare_temp_set.shape[0]\nval_samples = int(np.round(total_samples * 0.5))\nrare_val_set = rare_temp_set.sample(n=val_samples, random_state=42)\nrare_test_set = rare_temp_set.drop(rare_val_set.index)\n\n# Combining the splits into final train, validation and test sets\nfinal_train_set = pd.concat([common_train_set, rare_train_set])\nfinal_val_set = pd.concat([common_val_set, rare_val_set])\nfinal_test_set = pd.concat([common_test_set, rare_test_set])\n\n# Optional: Verifying the final distribution (can be commented out for large datasets)\nfinal_train_distribution = final_train_set['site_animal'].value_counts(normalize=True)\nfinal_val_distribution = final_val_set['site_animal'].value_counts(normalize=True)\nfinal_test_distribution = final_test_set['site_animal'].value_counts(normalize=True)\nfinal_distribution_summary = pd.DataFrame({\n    'Train Distribution': final_train_distribution,\n    'Validation Distribution': final_val_distribution,\n    'Test Distribution': final_test_distribution\n})\nprint(final_distribution_summary.head())","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:38.035029Z","iopub.execute_input":"2024-02-21T19:07:38.035565Z","iopub.status.idle":"2024-02-21T19:07:38.601538Z","shell.execute_reply.started":"2024-02-21T19:07:38.035518Z","shell.execute_reply":"2024-02-21T19:07:38.600651Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"                        Train Distribution  Validation Distribution  \\\nsite_animal                                                           \nS0001_bird                        0.000692                 0.000813   \nS0001_blank                       0.000346                 0.000407   \nS0001_leopard                     0.003112                 0.002847   \nS0001_monkey_prosimian            0.001037                 0.001220   \nS0002_bird                        0.000519                 0.000813   \n\n                        Test Distribution  \nsite_animal                                \nS0001_bird                       0.000407  \nS0001_blank                      0.000407  \nS0001_leopard                    0.003252  \nS0001_monkey_prosimian           0.000813  \nS0002_bird                       0.000407  \n","output_type":"stream"}]},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# from sklearn.model_selection import train_test_split\n\n# # Load the datasets\n\n# train_features = train_X\n# train_labels = train_y\n\n# # Merge the datasets on 'id'\n# merged_data = pd.merge(train_features, train_labels, on='id')\n\n# # Identifying the animal present in each image and creating a combined category\n# animal_columns = ['antelope_duiker', 'bird', 'blank', 'civet_genet', 'hog', 'leopard', 'monkey_prosimian', 'rodent']\n# merged_data['animal'] = merged_data[animal_columns].idxmax(axis=1)\n# merged_data['site_animal'] = merged_data['site'] + '_' + merged_data['animal']\n\n# # Checking the number of instances for each site_animal combination\n# combination_counts = merged_data['site_animal'].value_counts()\n# rare_combinations = combination_counts[combination_counts < 5]\n\n# # Separating the dataset into common and rare combinations\n# common_combinations = merged_data[~merged_data['site_animal'].isin(rare_combinations.index)]\n# rare_combinations_data = merged_data[merged_data['site_animal'].isin(rare_combinations.index)]\n\n# # Stratified split for common combinations into train and temp sets\n# common_train_set, common_temp_set = train_test_split(\n#     common_combinations, test_size=0.3, stratify=common_combinations['site_animal'], random_state=42)\n\n# # Stratified split for common combinations into validation and test sets\n# common_val_set, common_test_set = train_test_split(\n#     common_temp_set, test_size=0.5, stratify=common_temp_set['site_animal'], random_state=42)\n\n# # Randomly splitting rare combinations into train and temp sets\n# total_samples = rare_combinations_data.shape[0]\n# train_samples = int(np.round(total_samples * 0.7))\n# rare_train_set = rare_combinations_data.sample(n=train_samples, random_state=42)\n# rare_temp_set = rare_combinations_data.drop(rare_train_set.index)\n\n# # Randomly splitting rare combinations into validation and test sets\n# total_samples = rare_temp_set.shape[0]\n# val_samples = int(np.round(total_samples * 0.5))\n# rare_val_set = rare_temp_set.sample(n=val_samples, random_state=42)\n# rare_test_set = rare_temp_set.drop(rare_val_set.index)\n\n# # Combining the splits into final train, validation and test sets\n# final_train_set = pd.concat([common_train_set, rare_train_set])\n# final_val_set = pd.concat([common_val_set, rare_val_set])\n# final_test_set = pd.concat([common_test_set, rare_test_set])\n\n# # Optional: Verifying the final distribution (can be commented out for large datasets)\n# final_train_distribution = final_train_set['site_animal'].value_counts(normalize=True)\n# final_val_distribution = final_val_set['site_animal'].value_counts(normalize=True)\n# final_test_distribution = final_test_set['site_animal'].value_counts(normalize=True)\n# final_distribution_summary = pd.DataFrame({\n#     'Train Distribution': final_train_distribution,\n#     'Validation Distribution': final_val_distribution,\n#     'Test Distribution': final_test_distribution\n# })\n# print(final_distribution_summary.head())","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:38.602634Z","iopub.execute_input":"2024-02-21T19:07:38.603197Z","iopub.status.idle":"2024-02-21T19:07:38.608441Z","shell.execute_reply.started":"2024-02-21T19:07:38.603162Z","shell.execute_reply":"2024-02-21T19:07:38.607738Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"# import pandas as pd\n# import numpy as np\n# from sklearn.model_selection import train_test_split\n\n# # Load the datasets\n\n# train_features = train_X\n# train_labels = train_y\n\n# # Merge the datasets on 'id'\n# merged_data = pd.merge(train_features, train_labels, on='id')\n\n# # Identifying the animal present in each image and creating a combined category\n# animal_columns = ['antelope_duiker', 'bird', 'blank', 'civet_genet', 'hog', 'leopard', 'monkey_prosimian', 'rodent']\n# merged_data['animal'] = merged_data[animal_columns].idxmax(axis=1)\n# merged_data['site_animal'] = merged_data['site'] + '_' + merged_data['animal']\n\n# # Checking the number of instances for each site_animal combination\n# combination_counts = merged_data['site_animal'].value_counts()\n# rare_combinations = combination_counts[combination_counts < 5]\n\n# # Separating the dataset into common and rare combinations\n# common_combinations = merged_data[~merged_data['site_animal'].isin(rare_combinations.index)]\n# rare_combinations_data = merged_data[merged_data['site_animal'].isin(rare_combinations.index)]\n\n# # Stratified split for common combinations\n# common_train_set, common_test_set = train_test_split(\n#     common_combinations, test_size=0.25, stratify=common_combinations['site_animal'], random_state=42)\n\n# # Randomly splitting rare combinations\n# total_samples = rare_combinations_data.shape[0]\n# train_samples = int(np.round(total_samples * 0.75))\n# rare_train_set = rare_combinations_data.sample(n=train_samples, random_state=42)\n# rare_test_set = rare_combinations_data.drop(rare_train_set.index)\n\n# # Combining the splits into final train and test sets\n# final_train_set = pd.concat([common_train_set, rare_train_set])\n# final_test_set = pd.concat([common_test_set, rare_test_set])\n\n# # Optional: Verifying the final distribution (can be commented out for large datasets)\n# final_train_distribution = final_train_set['site_animal'].value_counts(normalize=True)\n# final_test_distribution = final_test_set['site_animal'].value_counts(normalize=True)\n# final_distribution_summary = pd.DataFrame({\n#     'Train Distribution': final_train_distribution,\n#     'Test Distribution': final_test_distribution\n# })\n# print(final_distribution_summary.head())\n\n\n\n\n\n# ## Verify the 75/25 train_test split\n\n# # Calculate the number of samples in each set\n# num_train_samples = final_train_set.shape[0]\n# num_test_samples = final_test_set.shape[0]\n# total_samples = num_train_samples + num_test_samples\n\n# # Calculate the proportions\n# train_proportion = num_train_samples / total_samples\n# test_proportion = num_test_samples / total_samples\n\n# # Print out the proportions\n# print(\"Training Set Proportion: {:.2%}\".format(train_proportion))\n# print(\"Test Set Proportion: {:.2%}\".format(test_proportion))\n\n\n\n# ## Making sure that train and test set have a 75/25 split for each site\n\n\n# # Calculate the count of each site in both sets\n# site_counts_train = final_train_set['site'].value_counts()\n# site_counts_test = final_test_set['site'].value_counts()\n\n# # Combine the counts into a single DataFrame for comparison\n# combined_site_counts = pd.DataFrame({'Train Count': site_counts_train, 'Test Count': site_counts_test})\n\n# # Calculate the total counts for each site\n# combined_site_counts['Total Count'] = combined_site_counts['Train Count'] + combined_site_counts['Test Count']\n\n# # Calculate the percentage split for each site\n# combined_site_counts['Train Percentage'] = (combined_site_counts['Train Count'] / combined_site_counts['Total Count']) * 100\n# combined_site_counts['Test Percentage'] = (combined_site_counts['Test Count'] / combined_site_counts['Total Count']) * 100\n\n# # Display the combined counts with percentage split\n# #print(combined_site_counts.head())\n# combined_site_counts.to_csv(\"site_percentage_check.csv\")\n\n# # Check to see if there any rows of data with a train percentage below 70 or above 80 (the ideal is 75)\n# filtered_df = combined_site_counts[(combined_site_counts['Train Percentage'] < 70) | (combined_site_counts['Train Percentage'] > 80)]\n# #print(filtered_df)\n# #print(len(filtered_df)) # There are only 13 sites which have a bad train/test split, but they all side more towards the train set, which is good\n\n\n# #print(len(final_train_set))\n\n\n\n\n\n# ## Making sure the trian and test set have the 75/25 split for each animal \n# # List of label columns\n# label_columns = ['antelope_duiker', 'bird', 'blank', 'civet_genet', 'hog', 'leopard', 'monkey_prosimian', 'rodent']\n\n# # Calculate the count of each label in both sets\n# label_counts_train = final_train_set[label_columns].sum()\n# label_counts_test = final_test_set[label_columns].sum()\n\n# # Combine the counts into a single DataFrame for comparison\n# combined_label_counts = pd.DataFrame({'Train Count': label_counts_train, 'Test Count': label_counts_test})\n\n# # Calculate the total counts for each label\n# combined_label_counts['Total Count'] = combined_label_counts['Train Count'] + combined_label_counts['Test Count']\n\n# # Calculate the percentage split for each label\n# combined_label_counts['Train Percentage'] = (combined_label_counts['Train Count'] / combined_label_counts['Total Count']) * 100\n# combined_label_counts['Test Percentage'] = (combined_label_counts['Test Count'] / combined_label_counts['Total Count']) * 100\n\n# # Display the combined counts with percentage split\n# #print(combined_label_counts)\n\n\n\n\n\n# ## Modifying the data to include only the original columns\n\n# #print(final_train_set)\n\n# # Remove all the new features that I created for the stratified train_test split\n# train_X = final_train_set[['id', 'filepath', 'site']]\n# train_Y = final_train_set[['id','antelope_duiker','bird','blank','civet_genet','hog','leopard','monkey_prosimian','rodent']]\n\n# test_X = final_test_set[['id', 'filepath', 'site']]\n# test_Y = final_test_set[['id','antelope_duiker','bird','blank','civet_genet','hog','leopard','monkey_prosimian','rodent']]\n\n\n# # Make \"id\" the index column\n# train_X.set_index('id', inplace=True) # inplace = True means that it edits the original dataframe, and no new dataframe is created\n# train_Y.set_index('id', inplace=True)\n\n# test_X.set_index('id', inplace=True)\n# test_Y.set_index('id', inplace=True)\n\n\n\n\n# ## Make it so that if there already exists a dataset for train_X, train_Y, and test_X, then we will use those (so that each of my models are trained on the same data, making them deterministic)\n# import os\n\n# if not os.path.exists('train_split_X.csv'):\n#     print(\"create new directory\")\n#     train_X.to_csv('train_split_X.csv')\n#     train_Y.to_csv('train_split_Y.csv')\n#     test_X.to_csv('test_split_X.csv')\n#     test_Y.to_csv('test_split_Y.csv')\n\n# else:\n#     print(\"used old directory\")\n#     train_X = pd.read_csv('train_split_X.csv')\n#     train_Y = pd.read_csv('train_split_Y.csv')\n#     test_X = pd.read_csv('test_split_X.csv')\n#     test_Y = pd.read_csv(\"test_split_Y.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:38.609325Z","iopub.execute_input":"2024-02-21T19:07:38.609570Z","iopub.status.idle":"2024-02-21T19:07:39.801315Z","shell.execute_reply.started":"2024-02-21T19:07:38.609543Z","shell.execute_reply":"2024-02-21T19:07:39.800490Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"markdown","source":"## Verify the 75/25 train_test split","metadata":{}},{"cell_type":"code","source":"# Calculate the number of samples in each set\nnum_train_samples = final_train_set.shape[0]\nnum_val_samples = final_val_set.shape[0]\nnum_test_samples = final_test_set.shape[0]\ntotal_samples = num_train_samples + num_val_samples + num_test_samples\n\n# Calculate the proportions\ntrain_proportion = num_train_samples / total_samples\nval_proportion = num_val_samples / total_samples\ntest_proportion = num_test_samples / total_samples\n\n# Print out the proportions\nprint(\"Training Set Proportion: {:.2%}\".format(train_proportion))\nprint(\"Validation Set Proportion: {:.2%}\".format(val_proportion))\nprint(\"Test Set Proportion: {:.2%}\".format(test_proportion))","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:39.802243Z","iopub.execute_input":"2024-02-21T19:07:39.802485Z","iopub.status.idle":"2024-02-21T19:07:40.013131Z","shell.execute_reply.started":"2024-02-21T19:07:39.802461Z","shell.execute_reply":"2024-02-21T19:07:40.012307Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stdout","text":"Training Set Proportion: 70.17%\nValidation Set Proportion: 14.91%\nTest Set Proportion: 14.92%\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Making sure that train and test set have a 75/25 split for each site","metadata":{}},{"cell_type":"markdown","source":"In other words, for each site, around 75% should be in the training data and around 25% should be in the test data. This ensures niether the train or test set have a unbalanced amount of a certain site, leading to bias and bad predictions","metadata":{}},{"cell_type":"code","source":"# Calculate the count of each site in all sets\nsite_counts_train = final_train_set['site'].value_counts()\nsite_counts_val = final_val_set['site'].value_counts()\nsite_counts_test = final_test_set['site'].value_counts()\n\n# Combine the counts into a single DataFrame for comparison\ncombined_site_counts = pd.DataFrame({'Train Count': site_counts_train, 'Validation Count': site_counts_val, 'Test Count': site_counts_test})\n\n# Calculate the total counts for each site\ncombined_site_counts['Total Count'] = combined_site_counts.sum(axis=1)\n\n# Calculate the percentage split for each site\ncombined_site_counts['Train Percentage'] = (combined_site_counts['Train Count'] / combined_site_counts['Total Count']) * 100\ncombined_site_counts['Validation Percentage'] = (combined_site_counts['Validation Count'] / combined_site_counts['Total Count']) * 100\ncombined_site_counts['Test Percentage'] = (combined_site_counts['Test Count'] / combined_site_counts['Total Count']) * 100\n\n# Display the combined counts with percentage split\ncombined_site_counts.to_csv(\"site_percentage_check.csv\")\n\n# Check to see if there any rows of data with a train percentage below 65 or above 75, validation percentage below 10 or above 20, and test percentage below 10 or above 20\nfiltered_df = combined_site_counts[(combined_site_counts['Train Percentage'] < 65) | (combined_site_counts['Train Percentage'] > 75) | (combined_site_counts['Validation Percentage'] < 10) | (combined_site_counts['Validation Percentage'] > 20) | (combined_site_counts['Test Percentage'] < 10) | (combined_site_counts['Test Percentage'] > 20)]\n\nprint(filtered_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:40.014103Z","iopub.execute_input":"2024-02-21T19:07:40.014383Z","iopub.status.idle":"2024-02-21T19:07:40.238435Z","shell.execute_reply.started":"2024-02-21T19:07:40.014355Z","shell.execute_reply":"2024-02-21T19:07:40.237649Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stdout","text":"       Train Count  Validation Count  Test Count  Total Count  \\\nsite                                                            \nS0007            7               1.0         1.0          9.0   \nS0017           15               2.0         2.0         19.0   \nS0028           22               5.0         2.0         29.0   \nS0046            8               1.0         1.0         10.0   \nS0078            1               1.0         NaN          2.0   \nS0079            2               NaN         NaN          2.0   \nS0092            2               NaN         1.0          3.0   \nS0098            8               2.0         1.0         11.0   \nS0102            1               NaN         NaN          1.0   \nS0106            4               NaN         1.0          5.0   \nS0107            7               NaN         1.0          8.0   \nS0112            4               NaN         NaN          4.0   \nS0115            7               1.0         1.0          9.0   \nS0121           14               5.0         1.0         20.0   \nS0143            2               1.0         NaN          3.0   \nS0146           12               5.0         3.0         20.0   \nS0148            6               1.0         NaN          7.0   \nS0153            6               1.0         2.0          9.0   \nS0157           25               4.0         4.0         33.0   \nS0158           11               NaN         NaN         11.0   \nS0160           17               2.0         3.0         22.0   \nS0171           18               2.0         2.0         22.0   \nS0173           16               4.0         5.0         25.0   \nS0176           20               3.0         3.0         26.0   \nS0177           10               3.0         3.0         16.0   \nS0178            1               1.0         NaN          2.0   \nS0182            6               3.0         1.0         10.0   \nS0190            5               NaN         NaN          5.0   \nS0191            8               2.0         1.0         11.0   \nS0192            6               NaN         NaN          6.0   \nS0196           13               1.0         1.0         15.0   \nS0197           10               4.0         3.0         17.0   \n\n       Train Percentage  Validation Percentage  Test Percentage  \nsite                                                             \nS0007         77.777778              11.111111        11.111111  \nS0017         78.947368              10.526316        10.526316  \nS0028         75.862069              17.241379         6.896552  \nS0046         80.000000              10.000000        10.000000  \nS0078         50.000000              50.000000              NaN  \nS0079        100.000000                    NaN              NaN  \nS0092         66.666667                    NaN        33.333333  \nS0098         72.727273              18.181818         9.090909  \nS0102        100.000000                    NaN              NaN  \nS0106         80.000000                    NaN        20.000000  \nS0107         87.500000                    NaN        12.500000  \nS0112        100.000000                    NaN              NaN  \nS0115         77.777778              11.111111        11.111111  \nS0121         70.000000              25.000000         5.000000  \nS0143         66.666667              33.333333              NaN  \nS0146         60.000000              25.000000        15.000000  \nS0148         85.714286              14.285714              NaN  \nS0153         66.666667              11.111111        22.222222  \nS0157         75.757576              12.121212        12.121212  \nS0158        100.000000                    NaN              NaN  \nS0160         77.272727               9.090909        13.636364  \nS0171         81.818182               9.090909         9.090909  \nS0173         64.000000              16.000000        20.000000  \nS0176         76.923077              11.538462        11.538462  \nS0177         62.500000              18.750000        18.750000  \nS0178         50.000000              50.000000              NaN  \nS0182         60.000000              30.000000        10.000000  \nS0190        100.000000                    NaN              NaN  \nS0191         72.727273              18.181818         9.090909  \nS0192        100.000000                    NaN              NaN  \nS0196         86.666667               6.666667         6.666667  \nS0197         58.823529              23.529412        17.647059  \n","output_type":"stream"}]},{"cell_type":"markdown","source":"There are only 13 sites which have a bad train/test split, but they all side more towards the train set, which is good","metadata":{}},{"cell_type":"markdown","source":"## Making sure the trian and test set have the 75/25 split for each animal ","metadata":{}},{"cell_type":"markdown","source":"In other words, for each animal, around 75% should be in the training data and around 25% should be in the test data. This ensures neither the train or test set have a unbalanced amount of a certain animal, leading to bias and bad predictions","metadata":{}},{"cell_type":"code","source":"# List of label columns\nlabel_columns = ['antelope_duiker', 'bird', 'blank', 'civet_genet', 'hog', 'leopard', 'monkey_prosimian', 'rodent']\n\n# Calculate the count of each label in all sets\nlabel_counts_train = final_train_set[label_columns].sum()\nlabel_counts_val = final_val_set[label_columns].sum()\nlabel_counts_test = final_test_set[label_columns].sum()\n\n# Combine the counts into a single DataFrame for comparison\ncombined_label_counts = pd.DataFrame({'Train Count': label_counts_train, 'Validation Count': label_counts_val, 'Test Count': label_counts_test})\n\n# Calculate the total counts for each label\ncombined_label_counts['Total Count'] = combined_label_counts.sum(axis=1)\n\n# Calculate the percentage split for each label\ncombined_label_counts['Train Percentage'] = (combined_label_counts['Train Count'] / combined_label_counts['Total Count']) * 100\ncombined_label_counts['Validation Percentage'] = (combined_label_counts['Validation Count'] / combined_label_counts['Total Count']) * 100\ncombined_label_counts['Test Percentage'] = (combined_label_counts['Test Count'] / combined_label_counts['Total Count']) * 100\n\n# Display the combined counts with percentage split\ncombined_label_counts.to_csv(\"label_percentage_check.csv\")\n\n# Check to see if there any rows of data with a train percentage below 65 or above 75, validation percentage below 10 or above 20, and test percentage below 10 or above 20\nfiltered_df = combined_label_counts[(combined_label_counts['Train Percentage'] < 65) | (combined_label_counts['Train Percentage'] > 75) | (combined_label_counts['Validation Percentage'] < 10) | (combined_label_counts['Validation Percentage'] > 20) | (combined_label_counts['Test Percentage'] < 10) | (combined_label_counts['Test Percentage'] > 20)]\n\nprint(filtered_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:40.239417Z","iopub.execute_input":"2024-02-21T19:07:40.239707Z","iopub.status.idle":"2024-02-21T19:07:40.430895Z","shell.execute_reply.started":"2024-02-21T19:07:40.239680Z","shell.execute_reply":"2024-02-21T19:07:40.430160Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"Empty DataFrame\nColumns: [Train Count, Validation Count, Test Count, Total Count, Train Percentage, Validation Percentage, Test Percentage]\nIndex: []\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Since each class has around a 75/25 split for train and test split, my data looks good!","metadata":{}},{"cell_type":"markdown","source":"## Modifying the data to include only the original columns","metadata":{}},{"cell_type":"code","source":"# Remove all the new features that I created for the stratified train_test split\ntrain_X = final_train_set[['id', 'filepath', 'site']]\ntrain_Y = final_train_set[['id','antelope_duiker','bird','blank','civet_genet','hog','leopard','monkey_prosimian','rodent']]\n\nvalid_X = final_val_set[['id', 'filepath', 'site']]\nvalid_Y = final_val_set[['id','antelope_duiker','bird','blank','civet_genet','hog','leopard','monkey_prosimian','rodent']]\n\ntest_X = final_test_set[['id', 'filepath', 'site']]\ntest_Y = final_test_set[['id','antelope_duiker','bird','blank','civet_genet','hog','leopard','monkey_prosimian','rodent']]\n\n# Make \"id\" the index column\ntrain_X.set_index('id', inplace=True) # inplace = True means that it edits the original dataframe, and no new dataframe is created\ntrain_Y.set_index('id', inplace=True)\n\nvalid_X.set_index('id', inplace=True)\nvalid_Y.set_index('id', inplace=True)\n\ntest_X.set_index('id', inplace=True)\ntest_Y.set_index('id', inplace=True)\n\nprint(train_X)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:40.431811Z","iopub.execute_input":"2024-02-21T19:07:40.432060Z","iopub.status.idle":"2024-02-21T19:07:40.648534Z","shell.execute_reply.started":"2024-02-21T19:07:40.432034Z","shell.execute_reply":"2024-02-21T19:07:40.647765Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stdout","text":"                             filepath   site\nid                                          \nZJ015450  train_features/ZJ015450.jpg  S0003\nZJ009090  train_features/ZJ009090.jpg  S0150\nZJ007499  train_features/ZJ007499.jpg  S0085\nZJ010855  train_features/ZJ010855.jpg  S0062\nZJ012846  train_features/ZJ012846.jpg  S0002\n...                               ...    ...\nZJ011873  train_features/ZJ011873.jpg  S0134\nZJ008591  train_features/ZJ008591.jpg  S0179\nZJ002311  train_features/ZJ002311.jpg  S0121\nZJ006934  train_features/ZJ006934.jpg  S0050\nZJ014065  train_features/ZJ014065.jpg  S0185\n\n[11569 rows x 2 columns]\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Make it so that if there already exists a dataset for train_X, train_Y, and test_X, then we will use those (so that each of my models are trained on the same data, making them deterministic)","metadata":{}},{"cell_type":"code","source":"import os\n\nif not os.path.exists('/kaggle/input/conser-vision-data/train_split_2_X.csv'):\n    print(\"create new directory\")\n    train_X.to_csv('/kaggle/input/conser-vision-data/train_split_2_X.csv')\n    train_Y.to_csv('/kaggle/input/conser-vision-data/train_split_2_Y.csv')\n    valid_X.to_csv('/kaggle/input/conser-vision-data/valid_split_2_X.csv')\n    valid_Y.to_csv('/kaggle/input/conser-vision-data/valid_split_2_Y.csv')\n    test_X.to_csv('/kaggle/input/conser-vision-data/test_split_2_X.csv')\n    test_Y.to_csv('/kaggle/input/conser-vision-data/test_split_2_Y.csv')\n\nelse:\n    print(\"used old directory\")\n    train_X = pd.read_csv('/kaggle/input/conser-vision-data/train_split_2_X.csv', index_col='id')\n    train_Y = pd.read_csv('/kaggle/input/conser-vision-data/train_split_2_Y.csv', index_col='id')\n    valid_X = pd.read_csv('/kaggle/input/conser-vision-data/valid_split_2_X.csv', index_col='id')\n    valid_Y = pd.read_csv('/kaggle/input/conser-vision-data/valid_split_2_Y.csv', index_col='id')\n    test_X = pd.read_csv('/kaggle/input/conser-vision-data/test_split_2_X.csv', index_col='id')\n    test_Y = pd.read_csv(\"/kaggle/input/conser-vision-data/test_split_2_Y.csv\", index_col='id')","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:40.652001Z","iopub.execute_input":"2024-02-21T19:07:40.652281Z","iopub.status.idle":"2024-02-21T19:07:40.895694Z","shell.execute_reply.started":"2024-02-21T19:07:40.652252Z","shell.execute_reply":"2024-02-21T19:07:40.894837Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stdout","text":"used old directory\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Check to see if each image has the same dimensions since that's important for data preprocessing","metadata":{}},{"cell_type":"code","source":"# from PIL import Image\n# import os\n\n# def check_image_dimensions(directory):\n#     image_sizes = {}\n#     for img_name in os.listdir(directory):\n#         if not img_name.lower().endswith(('.png', '.jpg', '.jpeg', '.tif')): # now also checks for .tif\n#             continue\n#         img_path = os.path.join(directory, img_name)\n#         with Image.open(img_path) as img:\n#             # Get image size\n#             size = img.size\n#             if size in image_sizes:\n#                 image_sizes[size] += 1\n#             else:\n#                 image_sizes[size] = 1\n\n#     for size, count in image_sizes.items():\n#         print(f\"For the {directory} directory, {count} images are of dimension {size}\")\n\n# # Use it on the train and test data only if this code segment was never ran in this coding session:\n\n# # Use it on the train and test data:\n# check_image_dimensions('train_features')\n# check_image_dimensions('test_features')\n\n\n# # # For the train_features directory, different dimensions found: {(160, 120), (960, 515), (640, 335), (960, 540), (640, 360), (360, 215), (160, 95), (360, 240)}\n# # # For the test_features directory, different dimensions found: {(960, 515), (160, 120), (640, 335), (960, 540), (320, 215), (640, 360), (360, 240), (320, 240)}\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:40.896715Z","iopub.execute_input":"2024-02-21T19:07:40.896997Z","iopub.status.idle":"2024-02-21T19:07:41.051485Z","shell.execute_reply.started":"2024-02-21T19:07:40.896968Z","shell.execute_reply":"2024-02-21T19:07:41.050666Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"markdown","source":"## Resizing all the images ","metadata":{}},{"cell_type":"code","source":"# I will actually make the resizing function when I use the ImageDataGenertor. \n# By calling the resize function in the ImageDataGenerator, I won't have to save my images in my local folder and waste space. ","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:41.052383Z","iopub.execute_input":"2024-02-21T19:07:41.052648Z","iopub.status.idle":"2024-02-21T19:07:41.258342Z","shell.execute_reply.started":"2024-02-21T19:07:41.052597Z","shell.execute_reply":"2024-02-21T19:07:41.257567Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"markdown","source":"## Before I use ImageDataGenerator, I have to format my dataframes so that they are able to be read properly by ImageDataGenertor","metadata":{}},{"cell_type":"markdown","source":"This is a multi-class problem so data format has to be modified so that it can be handled by ImageDataGenerator. ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# Merge the two datasets for training set\nmerged_train = pd.merge(train_X, train_Y, on='id')\n\n# Check if 'id' is in columns\nif 'id' in merged_train.columns:\n    # Set \"id\" as the index column\n    merged_train.set_index(\"id\", inplace = True) \nelse:\n    print(\"Column 'id' does not exist in merged_train\")\n\n# Convert multi-label columns into a single column, so that this column tells us what animal type the row is \nmerged_train['labels'] = merged_train.apply(lambda row: ' '.join([col for col in merged_train.columns[2:] if row[col]==1]), axis=1)\n\n\n\n\n# Do the same thing for the validation set\n\n# Merge the two datasets\nmerged_valid = pd.merge(valid_X, valid_Y, on='id')\n\n# Check if 'id' is in columns\nif 'id' in merged_valid.columns:\n    # Set \"id\" as the index column\n    merged_valid.set_index(\"id\", inplace = True) \nelse:\n    print(\"Column 'id' does not exist in merged_train\")\n\n# Convert multi-label columns into a single column, so that this column tells us what animal type the row is \nmerged_valid['labels'] = merged_valid.apply(lambda row: ' '.join([col for col in merged_valid.columns[2:] if row[col]==1]), axis=1)\n\n\n\n\n# Do the same thing for the test set\n\n# Merge the two datasets\nmerged_test = pd.merge(test_X, test_Y, on='id')\n\n# Check if 'id' is in columns\nif 'id' in merged_test.columns:\n    # Set \"id\" as the index column\n    merged_test.set_index(\"id\", inplace = True) \nelse:\n    print(\"Column 'id' does not exist in merged_train\")\n\n\n# Convert multi-label columns into a single column, so that this column tells us what animal type the row is \nmerged_test['labels'] = merged_test.apply(lambda row: ' '.join([col for col in merged_test.columns[2:] if row[col]==1]), axis=1)\n\nprint(merged_test)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:41.259298Z","iopub.execute_input":"2024-02-21T19:07:41.259542Z","iopub.status.idle":"2024-02-21T19:07:41.992941Z","shell.execute_reply.started":"2024-02-21T19:07:41.259516Z","shell.execute_reply":"2024-02-21T19:07:41.992071Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"Column 'id' does not exist in merged_train\nColumn 'id' does not exist in merged_train\nColumn 'id' does not exist in merged_train\n                             filepath   site  antelope_duiker  bird  blank  \\\nid                                                                           \nZJ006365  train_features/ZJ006365.jpg  S0062              0.0   0.0    0.0   \nZJ008401  train_features/ZJ008401.jpg  S0120              0.0   0.0    1.0   \nZJ015840  train_features/ZJ015840.jpg  S0014              0.0   0.0    0.0   \nZJ004201  train_features/ZJ004201.jpg  S0071              0.0   0.0    0.0   \nZJ014267  train_features/ZJ014267.jpg  S0014              1.0   0.0    0.0   \n...                               ...    ...              ...   ...    ...   \nZJ015318  train_features/ZJ015318.jpg  S0134              0.0   0.0    0.0   \nZJ015363  train_features/ZJ015363.jpg  S0176              0.0   0.0    0.0   \nZJ015565  train_features/ZJ015565.jpg  S0124              0.0   1.0    0.0   \nZJ016248  train_features/ZJ016248.jpg  S0092              0.0   0.0    1.0   \nZJ016482  train_features/ZJ016482.jpg  S0146              0.0   0.0    1.0   \n\n          civet_genet  hog  leopard  monkey_prosimian  rodent           labels  \nid                                                                              \nZJ006365          1.0  0.0      0.0               0.0     0.0      civet_genet  \nZJ008401          0.0  0.0      0.0               0.0     0.0            blank  \nZJ015840          0.0  0.0      1.0               0.0     0.0          leopard  \nZJ004201          0.0  0.0      0.0               0.0     1.0           rodent  \nZJ014267          0.0  0.0      0.0               0.0     0.0  antelope_duiker  \n...               ...  ...      ...               ...     ...              ...  \nZJ015318          0.0  0.0      0.0               0.0     1.0           rodent  \nZJ015363          0.0  0.0      1.0               0.0     0.0          leopard  \nZJ015565          0.0  0.0      0.0               0.0     0.0             bird  \nZJ016248          0.0  0.0      0.0               0.0     0.0            blank  \nZJ016482          0.0  0.0      0.0               0.0     0.0            blank  \n\n[2460 rows x 11 columns]\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Change filepaths to account for Kaggle","metadata":{}},{"cell_type":"code","source":"# Base path for Kaggle input directory\nkaggle_input_path = '/kaggle/input/conser-vision-data/'\n\n# Update the 'filepath' column in each dataframe\nmerged_train['filepath'] = kaggle_input_path + 'train_features/' + merged_train['filepath'].astype(str)\nmerged_valid['filepath'] = kaggle_input_path + 'train_features/' + merged_valid['filepath'].astype(str)\nmerged_test['filepath'] = kaggle_input_path + 'train_features/' + merged_test['filepath'].astype(str)\n\ntest_values['filepath'] = kaggle_input_path + 'test_features/' + test_values['filepath'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:41.994046Z","iopub.execute_input":"2024-02-21T19:07:41.994360Z","iopub.status.idle":"2024-02-21T19:07:42.004151Z","shell.execute_reply.started":"2024-02-21T19:07:41.994328Z","shell.execute_reply":"2024-02-21T19:07:42.003345Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"markdown","source":"# Turn train-valid-test sets into TFRecord","metadata":{}},{"cell_type":"code","source":"# Creating a mapping of labels to integers\nunique_labels = merged_train['labels'].unique()\nlabel_to_int = {label: i for i, label in enumerate(unique_labels)}\n\n# Adding an integer label column to each DataFrame\nmerged_train['label_int'] = merged_train['labels'].map(label_to_int)\nmerged_valid['label_int'] = merged_valid['labels'].map(label_to_int)\nmerged_test['label_int'] = merged_test['labels'].map(label_to_int)\n\n\n# Display the label to integer mapping\nfor label, int_value in label_to_int.items():\n    print(f\"Label '{label}' is mapped to integer: {int_value}\")\n\n# Label 'blank' is mapped to integer: 0\n# Label 'monkey_prosimian' is mapped to integer: 1\n# Label 'civet_genet' is mapped to integer: 2\n# Label 'rodent' is mapped to integer: 3\n# Label 'antelope_duiker' is mapped to integer: 4\n# Label 'hog' is mapped to integer: 5\n# Label 'bird' is mapped to integer: 6\n# Label 'leopard' is mapped to integer: 7","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:42.005100Z","iopub.execute_input":"2024-02-21T19:07:42.005361Z","iopub.status.idle":"2024-02-21T19:07:43.753457Z","shell.execute_reply.started":"2024-02-21T19:07:42.005337Z","shell.execute_reply":"2024-02-21T19:07:43.752550Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stdout","text":"Label 'blank' is mapped to integer: 0\nLabel 'monkey_prosimian' is mapped to integer: 1\nLabel 'civet_genet' is mapped to integer: 2\nLabel 'rodent' is mapped to integer: 3\nLabel 'antelope_duiker' is mapped to integer: 4\nLabel 'hog' is mapped to integer: 5\nLabel 'bird' is mapped to integer: 6\nLabel 'leopard' is mapped to integer: 7\n","output_type":"stream"}]},{"cell_type":"code","source":"import tensorflow as tf\nimport io\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image, label):\n    # One-hot encode the label\n    label_one_hot = tf.one_hot(label, depth=8)  # Assuming 8 classes for one hot encoding\n    feature = {\n        'image': _bytes_feature(image),\n        'label': _bytes_feature(label_one_hot.numpy().tobytes())\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()\n\n\n# def serialize_example(image, label):\n#     feature = {\n#         'image': _bytes_feature(image),\n#         'label': _int64_feature(label)\n#     }\n#     example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n#     return example_proto.SerializeToString()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:43.754547Z","iopub.execute_input":"2024-02-21T19:07:43.754884Z","iopub.status.idle":"2024-02-21T19:07:43.910369Z","shell.execute_reply.started":"2024-02-21T19:07:43.754854Z","shell.execute_reply":"2024-02-21T19:07:43.909404Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"# from PIL import Image\n# import pandas as pd\n\n# def create_tfrecord(df, filename):\n#     with tf.io.TFRecordWriter(filename) as writer:\n#         for _, row in df.iterrows():\n#             label = row['label_int']\n#             image_path = row['filepath']\n#             image = Image.open(image_path)\n#             image = image.resize((224, 224))  # Resize if needed\n#             image_bytes = io.BytesIO()\n#             image.save(image_bytes, format='JPEG') # change JPEG to other file types if you want to do tfrecord on other file types\n#             serialized_example = serialize_example(image_bytes.getvalue(), label)\n#             writer.write(serialized_example)\n\n# # Example usage\n# create_tfrecord(merged_train, 'train.tfrecord')\n# create_tfrecord(merged_valid, 'valid.tfrecord')\n# create_tfrecord(merged_test, 'test.tfrecord')\n\n# # 2 min and 46 seconds before I did the one-hot-encoding\n\n# # took 4 min and 14 seconds after I did one-hot-encoding\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:43.911394Z","iopub.execute_input":"2024-02-21T19:07:43.911707Z","iopub.status.idle":"2024-02-21T19:07:44.108430Z","shell.execute_reply.started":"2024-02-21T19:07:43.911676Z","shell.execute_reply":"2024-02-21T19:07:44.107475Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"# def _parse_function(proto):\n#     feature_description = {\n#         'image': tf.io.FixedLenFeature([], tf.string),\n#         'label': tf.io.FixedLenFeature([], tf.int64),\n#     }\n#     parsed_features = tf.io.parse_single_example(proto, feature_description)\n#     image = tf.image.decode_jpeg(parsed_features['image'], channels=3)\n#     image = tf.cast(image, tf.float32) / 255.0  # Normalize to [0, 1]\n#     label = tf.cast(parsed_features['label'], tf.int32)\n#     return image, label\n\ndef _parse_function(proto):\n    feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.string),\n    }\n    parsed_features = tf.io.parse_single_example(proto, feature_description)\n    image = tf.image.decode_jpeg(parsed_features['image'], channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # Normalize to [0, 1]\n\n    # Decode the one-hot encoded label\n    label = tf.io.decode_raw(parsed_features['label'], tf.float32)\n    label = tf.reshape(label, [8])  # Reshape to the number of classes, to adjust for one hot encoding\n    return image, label\n\n\ndef load_dataset(filename, batch_size):\n    dataset = tf.data.TFRecordDataset(filename)\n    dataset = dataset.map(_parse_function)\n    dataset = dataset.batch(batch_size)\n    return dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n\n# Example usage\n#BATCH_SIZE = 128  # Adjust batch size as needed\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\ntrain_dataset = load_dataset('/kaggle/input/tfrecord/train.tfrecord', BATCH_SIZE)\nvalid_dataset = load_dataset('/kaggle/input/tfrecord/valid.tfrecord', BATCH_SIZE)\ntest_dataset = load_dataset('/kaggle/input/tfrecord/test.tfrecord', BATCH_SIZE)\n\n\ntrain_dataset = train_dataset.repeat()  # This repeats the dataset indefinitely. Since we have high epoch in TPU, we need this incase train is too small\n# This code above made my program not work though. Maybe I have to set steps_per_epoch in .fit() in order to make it work","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:44.109344Z","iopub.execute_input":"2024-02-21T19:07:44.109581Z","iopub.status.idle":"2024-02-21T19:07:44.448296Z","shell.execute_reply.started":"2024-02-21T19:07:44.109556Z","shell.execute_reply":"2024-02-21T19:07:44.447518Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"markdown","source":"# Turn test set (actual test set, not from the split) into TFRecord","metadata":{}},{"cell_type":"code","source":"def serialize_test_example(image):\n    feature = {\n        'image': _bytes_feature(image)\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:44.449197Z","iopub.execute_input":"2024-02-21T19:07:44.449431Z","iopub.status.idle":"2024-02-21T19:07:44.533542Z","shell.execute_reply.started":"2024-02-21T19:07:44.449406Z","shell.execute_reply":"2024-02-21T19:07:44.532649Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"# def create_test_tfrecord(df, filename):\n#     with tf.io.TFRecordWriter(filename) as writer:\n#         for _, row in df.iterrows():\n#             image_path = row['filepath']\n#             image = Image.open(image_path)\n#             image = image.resize((224, 224))  # Resize if needed\n#             image_bytes = io.BytesIO()\n#             image.save(image_bytes, format='JPEG') # change JPEG to other file types if you want to do tfrecord on other file types\n#             serialized_example = serialize_test_example(image_bytes.getvalue())\n#             writer.write(serialized_example)\n\n# # Usage\n# create_test_tfrecord(test_values, 'test_values.tfrecord')\n\n# # 46 seconds before one hot encoding\n\n# # 44.5 seconds after doing one hot encoding\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:44.534624Z","iopub.execute_input":"2024-02-21T19:07:44.534892Z","iopub.status.idle":"2024-02-21T19:07:44.747210Z","shell.execute_reply.started":"2024-02-21T19:07:44.534865Z","shell.execute_reply":"2024-02-21T19:07:44.746448Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"def _parse_test_function(proto):\n    feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string)\n    }\n    parsed_features = tf.io.parse_single_example(proto, feature_description)\n    image = tf.image.decode_jpeg(parsed_features['image'], channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # Normalize to [0, 1]\n    return image\n\ndef load_test_dataset(filename, batch_size):\n    dataset = tf.data.TFRecordDataset(filename)\n    dataset = dataset.map(_parse_test_function)\n    dataset = dataset.batch(batch_size)\n    return dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n\n# Example usage for test data\ntest_values_dataset = load_test_dataset('/kaggle/input/tfrecord/test_values.tfrecord', BATCH_SIZE) # the batch size variable is determined from the load_dataset() of the train-valid-test split\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:44.748091Z","iopub.execute_input":"2024-02-21T19:07:44.748329Z","iopub.status.idle":"2024-02-21T19:07:44.969768Z","shell.execute_reply.started":"2024-02-21T19:07:44.748304Z","shell.execute_reply":"2024-02-21T19:07:44.968966Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"markdown","source":"# **Model Development:**","metadata":{}},{"cell_type":"markdown","source":"## Model 1 of ResNet50. Score: 2.0596","metadata":{}},{"cell_type":"code","source":"# from tensorflow.keras.applications import ResNet50\n# from tensorflow.keras.models import Model\n# from tensorflow.keras.layers import Dense, GlobalAveragePooling2D\n\n# # Load pre-trained ResNet50 model without the top layer\n# base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(224, 224, 3))\n\n# # Freeze the layers of the base model\n# for layer in base_model.layers:\n#     layer.trainable = False\n\n# x = base_model.output # This line gets the output of the base_model (ResNet50 without the top layer) and uses it as the starting point for adding new layers.\n# x = GlobalAveragePooling2D()(x)  # Add a global spatial average pooling layer\n# x = Dense(1024, activation='relu')(x)  # Add a fully-connected layer\n# predictions = Dense(8, activation='softmax')(x)  # Add a final output layer\n\n# # This is the model we will train\n# model = Model(inputs=base_model.input, outputs=predictions) # this model links the ResNetModel and the new layers to the prediction layer\n\n\n# # compile model and set the metrics and loss \n# model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', 'categorical_crossentropy'])\n\n\n# # fit model\n# model.fit(train_split_generator, epochs=5, validation_data=test_split_generator)\n\n\n# # create predictions\n# predictions = model.predict(test_original_generator)\n\n# # Create a DataFrame with predictions\n# # Create a DataFrame with predictions\n# prediction_df = pd.DataFrame(predictions, columns=['antelope_duiker', 'bird', 'blank', 'civet_genet', 'hog', 'leopard', 'monkey_prosimian', 'rodent'])\n\n# # Add the 'id' from the original test DataFrame's index\n# prediction_df['id'] = test_values['id'] # id column from the original test set (test_values.csv)\n\n# # Ensure 'id' is the first column\n# prediction_df = prediction_df[['id'] + [col for col in prediction_df.columns if col != 'id']]\n\n# # Save the DataFrame to a CSV file\n# prediction_df.to_csv('submission_ResNet50_2.csv', index=False) # make sure to change file name to represent the model number\n# print(\"Successfully submitted!\")\n\n# # 10 min for 5 epoch of no data augmentation. loss of 1.6907 in training and loss of 2.0124 in reality. shuffle = false for training\n# # 11 min for 5 epoch of light change (20%) and horizontal flip. Loss of 1.7355 in training and loss of 2.0178 in reality. shuffle = false for training\n# # 14 min for 5 epoch of light change (20%) and horizontal flip. Loss of 1.7255 in training and loss of 2.0195 in reality. shuffle = true for training\n# # 13 min for 5 epoch of no data augmentation. Loss of 1.6750 in training and loss of 2.0061 in reality. shuffle = true for training\n\n\n# # 387/387 [==============================] - 154s 390ms/step - loss: 1.9244 - accuracy: 0.2521 - categorical_crossentropy: 1.9244 - val_loss: 1.8092 - val_accuracy: 0.2857 - val_categorical_crossentropy: 1.8092\n# # Epoch 2/5\n# # 387/387 [==============================] - 144s 373ms/step - loss: 1.7773 - accuracy: 0.3056 - categorical_crossentropy: 1.7773 - val_loss: 1.7780 - val_accuracy: 0.2889 - val_categorical_crossentropy: 1.7780\n# # Epoch 3/5\n# # 387/387 [==============================] - 147s 381ms/step - loss: 1.7393 - accuracy: 0.3249 - categorical_crossentropy: 1.7393 - val_loss: 1.6986 - val_accuracy: 0.3362 - val_categorical_crossentropy: 1.6986\n# # Epoch 4/5\n# # 387/387 [==============================] - 153s 397ms/step - loss: 1.7081 - accuracy: 0.3318 - categorical_crossentropy: 1.7081 - val_loss: 1.7598 - val_accuracy: 0.2988 - val_categorical_crossentropy: 1.7598\n# # Epoch 5/5\n# # 387/387 [==============================] - 147s 380ms/step - loss: 1.6750 - accuracy: 0.3436 - categorical_crossentropy: 1.6750 - val_loss: 1.7103 - val_accuracy: 0.3335 - val_categorical_crossentropy: 1.7103\n# # 140/140 [==============================] - 43s 302ms/step\n# # Successfully submitted!","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:44.970959Z","iopub.execute_input":"2024-02-21T19:07:44.971212Z","iopub.status.idle":"2024-02-21T19:07:45.065358Z","shell.execute_reply.started":"2024-02-21T19:07:44.971186Z","shell.execute_reply":"2024-02-21T19:07:45.064548Z"},"trusted":true},"execution_count":26,"outputs":[]},{"cell_type":"markdown","source":"## Optuna tuning","metadata":{}},{"cell_type":"code","source":"# import optuna\n# from tensorflow.keras.applications import ResNet50\n# from tensorflow.keras.layers import Dense, GlobalAveragePooling2D\n# from tensorflow.keras.models import Model\n# from tensorflow.keras.optimizers import Adam\n\n# def objective(trial):\n#     # Load the base pretrained model\n#     base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(224, 224, 3)) # weights = 'imagenet' means that it's using pretrained weights. include_top determines whether or not the fully-connected output layers of the model should be included.\n#     # If include_top=False, the full model is loaded except for the output layers. This is useful if you want to fine-tune the model on a different task. By excluding the top layers, you can add your own output layers that are appropriate for your task.\n    \n#     for layer in base_model.layers:\n#         layer.trainable = False\n    \n#     # Add a new top layer\n#     x = base_model.output\n#     x = GlobalAveragePooling2D()(x)\n#     x = Dense(trial.suggest_int('units', 32, 1024), activation='relu')(x)\n#     predictions = Dense(8, activation='softmax')(x)\n\n#     # This is the model we will train\n#     model = Model(inputs=base_model.input, outputs=predictions)\n\n#     # Compile the model\n#     model.compile(optimizer=Adam(learning_rate=trial.suggest_float('lr', 1e-4, 1e-2, log=True)),\n#                   loss='categorical_crossentropy',\n#                   metrics=['accuracy'])\n\n#     # Train the model\n#     model.fit(train_split_generator, validation_data=valid_split_generator, epochs=5, verbose = 0) # verbose = 0 will not display the progress bar\n\n#     # Evaluate the model\n#     score = model.evaluate(test_split_generator, verbose=0)\n\n#     return score[1]  # Return validation accuracy\n\n# study = optuna.create_study(direction='maximize')\n# study.optimize(objective, n_trials=2)\n\n# # took 18 min with 2 trials and 5 epoch\n\n# # [I 2024-02-02 17:36:02,554] A new study created in memory with name: no-name-8e93bbe0-f12b-4063-a1bd-e96e0f8f6120\n# # /tmp/ipykernel_717/2967665033.py:25: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n# #   model.compile(optimizer=Adam(lr=trial.suggest_loguniform('lr', 1e-4, 1e-2)),\n# # WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n# # Epoch 1/5\n# # 2024-02-02 17:36:06.130938: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:454] Loaded cuDNN version 8902\n# # 2024-02-02 17:36:07.375939: I external/local_xla/xla/service/service.cc:168] XLA service 0x55606bc6a5b0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n# # 2024-02-02 17:36:07.375980: I external/local_xla/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA GeForce RTX 4070 Laptop GPU, Compute Capability 8.9\n# # 2024-02-02 17:36:07.392655: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:269] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n# # WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n# # I0000 00:00:1706913367.505875    1478 device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n# # 362/362 [==============================] - 110s 295ms/step - loss: 1.9394 - accuracy: 0.2361 - val_loss: 1.8956 - val_accuracy: 0.2468\n# # Epoch 2/5\n# # 362/362 [==============================] - 99s 274ms/step - loss: 1.8068 - accuracy: 0.2937 - val_loss: 1.7982 - val_accuracy: 0.3042\n# # Epoch 3/5\n# # 362/362 [==============================] - 106s 292ms/step - loss: 1.7560 - accuracy: 0.3152 - val_loss: 1.7469 - val_accuracy: 0.3164\n# # Epoch 4/5\n# # 362/362 [==============================] - 107s 297ms/step - loss: 1.7170 - accuracy: 0.3262 - val_loss: 1.6725 - val_accuracy: 0.3542\n# # Epoch 5/5\n# # 362/362 [==============================] - 117s 323ms/step - loss: 1.6941 - accuracy: 0.3388 - val_loss: 1.6720 - val_accuracy: 0.3375\n# # [I 2024-02-02 17:45:24,421] Trial 0 finished with value: 0.3341463506221771 and parameters: {'units': 434, 'lr': 0.0024312122287096383}. Best is trial 0 with value: 0.3341463506221771.\n# # WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n# # Epoch 1/5\n# # 362/362 [==============================] - 96s 260ms/step - loss: 1.9440 - accuracy: 0.2457 - val_loss: 1.8002 - val_accuracy: 0.3160\n# # Epoch 2/5\n# # 362/362 [==============================] - 99s 274ms/step - loss: 1.7959 - accuracy: 0.3015 - val_loss: 1.7384 - val_accuracy: 0.3278\n# # Epoch 3/5\n# # 362/362 [==============================] - 102s 282ms/step - loss: 1.7605 - accuracy: 0.3094 - val_loss: 1.7890 - val_accuracy: 0.2981\n# # Epoch 4/5\n# # 362/362 [==============================] - 100s 275ms/step - loss: 1.7198 - accuracy: 0.3239 - val_loss: 1.6666 - val_accuracy: 0.3351\n# # Epoch 5/5\n# # 362/362 [==============================] - 110s 305ms/step - loss: 1.6861 - accuracy: 0.3376 - val_loss: 1.7233 - val_accuracy: 0.3217\n# # [I 2024-02-02 17:54:10,465] Trial 1 finished with value: 0.3178861737251282 and parameters: {'units': 969, 'lr': 0.008059458066605595}. Best is trial 0 with value: 0.3341463506221771.\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:45.066355Z","iopub.execute_input":"2024-02-21T19:07:45.066596Z","iopub.status.idle":"2024-02-21T19:07:45.265964Z","shell.execute_reply.started":"2024-02-21T19:07:45.066570Z","shell.execute_reply":"2024-02-21T19:07:45.265179Z"},"trusted":true},"execution_count":27,"outputs":[]},{"cell_type":"markdown","source":"## Optuna Tuning with mySQL","metadata":{}},{"cell_type":"code","source":"# import optuna\n# from tensorflow.keras.applications import ResNet50\n# from tensorflow.keras.layers import Dense, GlobalAveragePooling2D\n# from tensorflow.keras.models import Model\n# from tensorflow.keras.optimizers import Adam\n\n# # pip install these if you haven't already. these are important for running mySQL with Optuna\n\n# # %pip install mysql-connector-python \n# # %pip install PyMySQL\n\n\n\n# def objective(trial):\n#     # Load the base pretrained model\n#     base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(224, 224, 3)) # weights = 'imagenet' means that it's using pretrained weights. include_top determines whether or not the fully-connected output layers of the model should be included.\n#     # If include_top=False, the full model is loaded except for the output layers. This is useful if you want to fine-tune the model on a different task. By excluding the top layers, you can add your own output layers that are appropriate for your task.\n    \n#     for layer in base_model.layers:\n#         layer.trainable = False\n    \n#     # Add a new top layer\n#     x = base_model.output\n#     x = GlobalAveragePooling2D()(x)\n#     x = Dense(trial.suggest_int('units', 32, 1024), activation='relu')(x)\n#     predictions = Dense(8, activation='softmax')(x)\n\n#     # This is the model we will train\n#     model = Model(inputs=base_model.input, outputs=predictions)\n\n#     # Compile the model\n#     model.compile(optimizer=Adam(learning_rate=trial.suggest_float('lr', 1e-4, 1e-2, log=True)),\n#                   loss='categorical_crossentropy',\n#                   metrics=['accuracy'])\n\n#     # Train the model\n#     model.fit(train_split_generator, validation_data=valid_split_generator, epochs=2, verbose = 0) # verbose = 0 will not display the progress bar\n\n#     # Evaluate the model\n#     score = model.evaluate(test_split_generator, verbose=0)\n\n#     return score[1]  # Return validation accuracy\n\n\n# # Replace the following with your actual username, password, host, and database name. And maybe use \"127.0.0.1\" instead of localhost if it doesn't work\n# # optuna_storage = 'mysql+pymysql://optuna_user:your_password@localhost/database'. And maybe use \"127.0.0.1\" instead of localhost if it doesn't work\n\n# optuna_storage = 'mysql+pymysql://mike:11102004mike@localhost/optuna_first'\n\n# study = optuna.create_study(study_name='example_study', # name of the study\n#                             storage=optuna_storage,  # URL for the mySQL schema\n#                             direction='maximize', # maximize the accuracy\n#                             load_if_exists=True, # makes it so that if the study_name already exists in the schema, then it will append the new trials with the old trials\n#                             )\n\n\n# study.optimize(objective, n_trials=2)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:45.266985Z","iopub.execute_input":"2024-02-21T19:07:45.267335Z","iopub.status.idle":"2024-02-21T19:07:45.479531Z","shell.execute_reply.started":"2024-02-21T19:07:45.267308Z","shell.execute_reply":"2024-02-21T19:07:45.478807Z"},"trusted":true},"execution_count":28,"outputs":[]},{"cell_type":"markdown","source":"### This is how you resume previous Optuna studies (so you can pause your Optuna code and then resume next time, using mySQL)","metadata":{}},{"cell_type":"code","source":"# This is the way I do it:\n\n# optuna_storage = 'mysql+pymysql://mike:11102004mike@localhost/optuna_first'\n\n# study = optuna.create_study( load_if_exists=True, # This makes it so that if the study already exists, it will just resume the study and append it\n#                             )\n\n# study.optimize(objective, n_trials=2)\n\n\n\n\n\n# This is the same thing but with more code (just do the first method instead):\n\n# import optuna\n\n# # Assuming you have already created a study and have stored it in MySQL:\n# optuna_storage = 'mysql+pymysql://username:password@hostname/database_name'\n\n# # To resume a study\n# study = optuna.load_study(study_name='your_study_name', storage=optuna_storage)\n\n# # You can continue the optimization\n# study.optimize(objective, n_trials=additional_trials)\n\n\n\n# Or, you can just do \n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:45.480441Z","iopub.execute_input":"2024-02-21T19:07:45.480710Z","iopub.status.idle":"2024-02-21T19:07:45.695518Z","shell.execute_reply.started":"2024-02-21T19:07:45.480683Z","shell.execute_reply":"2024-02-21T19:07:45.694671Z"},"trusted":true},"execution_count":29,"outputs":[]},{"cell_type":"markdown","source":"## Optuna with mySQL and also make it focus on minimizing log loss instead of maximizing accuracy (since the competition ranks us on log loss, not accuracy)","metadata":{}},{"cell_type":"code","source":"# %pip install optuna\n\n# import optuna\n# from tensorflow.keras.applications import ResNet50\n# from tensorflow.keras.layers import Dense, GlobalAveragePooling2D\n# from tensorflow.keras.models import Model\n# from tensorflow.keras.optimizers import Adam\n# import os\n\n# # pip install these if you haven't already. these are important for running mySQL with Optuna\n\n\n# %pip install mysql-connector-python \n# %pip install PyMySQL\n\n\n\n# def objective(trial):\n    \n#     with strategy.scope(): # This is for making sure you use the TPU since strategey was initialized at the start of the notebook for the TPU\n    \n#         # Load the base pretrained model\n#         base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(224, 224, 3)) # weights = 'imagenet' means that it's using pretrained weights. include_top determines whether or not the fully-connected output layers of the model should be included.\n#         # If include_top=False, the full model is loaded except for the output layers. This is useful if you want to fine-tune the model on a different task. By excluding the top layers, you can add your own output layers that are appropriate for your task.\n\n#         for layer in base_model.layers:\n#             layer.trainable = False\n\n#         # Add a new top layer\n#         x = base_model.output\n#         x = GlobalAveragePooling2D()(x)\n#         x = Dense(trial.suggest_int('units', 32, 1024), activation='relu')(x)\n#         predictions = Dense(8, activation='softmax')(x)\n\n#         # This is the model we will train\n#         model = Model(inputs=base_model.input, outputs=predictions)\n\n#         # Compile the model\n#         model.compile(optimizer=Adam(learning_rate=trial.suggest_float('lr', 1e-4, 1e-2, log=True)),\n#                       loss='categorical_crossentropy',\n#                       metrics=['accuracy'])\n\n        \n        \n#     # Train the model\n#     model.fit(train_dataset, validation_data=valid_dataset, epochs=100, verbose = 1) # verbose = 0 will not display the progress bar\n\n#     # Evaluate the model\n#     score = model.evaluate(test_dataset, verbose=0)\n\n#     return score[0]  # Return validation loss\n\n\n# # Replace the following with your actual username, password, host, and database name. And maybe use \"127.0.0.1\" instead of localhost if it doesn't work\n# # optuna_storage = 'mysql+pymysql://optuna_user:your_password@localhost/database'. And maybe use \"127.0.0.1\" instead of localhost if it doesn't work\n\n# # optuna_storage = 'mysql+pymysql://remote_kaggle:Kaggle123123@50.187.96.101/kaggle'\n\n\n# # Define your Optuna study, using the MySQL connection string\n# optuna_storage = 'mysql+pymysql://<username>:<password>@<host>/<database>?ssl_ca=<path_to_CA_cert>&ssl_verify_cert=true'\n\n# from kaggle_secrets import UserSecretsClient\n# user_secrets = UserSecretsClient()\n# db_password = user_secrets.get_secret(\"DB_PASSWORD\")# This uses the secrets inside of Kaggle so I don't have to explicitly type my password out in code\n\n# # Example with your details (replace '<password>' with your real password and '<database>' with your database name)\n# optuna_storage = f'mysql+pymysql://MichaelAzure:{db_password}@kaggle-third-sql.mysql.database.azure.com/kaggle_first_database?ssl_ca=/kaggle/input/certification&ssl_verify_cert=true'\n\n# study = optuna.create_study(study_name='tpu_1', # name of the study\n#                             storage=optuna_storage,  # URL for the mySQL schema\n#                             direction='minimize', # maximize the log loss\n#                             load_if_exists=True, # makes it so that if the study_name already exists in the schema, then it will append the new trials with the old trials and essentially resume the study. It will also remember the previous trials so it really is resuming the study\n#                             )\n\n# study.optimize(objective, n_trials=100)\n\n# # you can access study trials info \n# # study.trials_dataframe()\n\n\n# # [I 2024-02-04 16:04:32,176] Using an existing study with name 'log_loss_2' instead of creating a new one.\n# # [I 2024-02-04 16:06:35,816] Trial 1 finished with value: 1.8934913873672485 and parameters: {'units': 729, 'lr': 0.0001994746969123514}. Best is trial 1 with value: 1.8934913873672485.\n# # [I 2024-02-04 16:08:29,971] Trial 2 finished with value: 1.863348126411438 and parameters: {'units': 894, 'lr': 0.0033167910282036737}. Best is trial 2 with value: 1.863348126411438.\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:45.696620Z","iopub.execute_input":"2024-02-21T19:07:45.696997Z","iopub.status.idle":"2024-02-21T19:07:45.838453Z","shell.execute_reply.started":"2024-02-21T19:07:45.696967Z","shell.execute_reply":"2024-02-21T19:07:45.837646Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ntrials = pd.read_csv(\"/kaggle/input/trials/trials.csv\")\n#trials.describe()\n\n# Group the DataFrame by 'study_id' and create a dictionary of DataFrames\ngrouped = trials.groupby('study_name')\ndfs = {study_name: group for study_name, group in grouped}\n\n# Sort each DataFrame by the 'value' column in descending order\nsorted_dfs = {study_name: df.sort_values(by='value', ascending=True) for study_name, df in dfs.items()}\n\ntpu_sorted = sorted_dfs[\"tpu_1\"]","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:45.839332Z","iopub.execute_input":"2024-02-21T19:07:45.839591Z","iopub.status.idle":"2024-02-21T19:07:46.083557Z","shell.execute_reply.started":"2024-02-21T19:07:45.839564Z","shell.execute_reply":"2024-02-21T19:07:46.082776Z"},"trusted":true},"execution_count":31,"outputs":[]},{"cell_type":"code","source":"# Assuming `df` is your original DataFrame\n# First, filter out only the relevant columns if there are any extra columns\ndf_filtered = tpu_sorted[['study_id', 'study_name', 'trial_id', 'trial_number', 'param_name', 'param_value', 'value']]\n\n# Pivot the table\ndf_pivoted = df_filtered.pivot_table(index=['study_id', 'study_name', 'trial_id', 'trial_number', 'value'], \n                                     columns='param_name', \n                                     values='param_value',\n                                     aggfunc='first').reset_index()\n\n# Renaming the columns for easier access\ndf_pivoted.columns = ['study_id', 'study_name', 'trial_id', 'trial_number', 'value', 'lr', 'units']\n\n# Now `df_pivoted` will have one row per trial_number with `lr`, `units`, and `value` as columns\n\n# Sort the pivoted table by value in ascending order (lowest values on top)\ntpu_final = df_pivoted.sort_values('value', ascending=True)\n\ntpu_final.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:46.084718Z","iopub.execute_input":"2024-02-21T19:07:46.085018Z","iopub.status.idle":"2024-02-21T19:07:46.335669Z","shell.execute_reply.started":"2024-02-21T19:07:46.084987Z","shell.execute_reply":"2024-02-21T19:07:46.334838Z"},"trusted":true},"execution_count":32,"outputs":[{"execution_count":32,"output_type":"execute_result","data":{"text/plain":"     study_id study_name  trial_id  trial_number     value        lr  units\n97         77      tpu_1       820           116  1.529258  0.000570  783.0\n174        77      tpu_1      1003           203  1.532762  0.000541  730.0\n146        77      tpu_1       943           173  1.533543  0.000547  830.0\n131        77      tpu_1       907           156  1.534082  0.000562  779.0\n145        77      tpu_1       940           172  1.535002  0.000557  821.0","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>study_id</th>\n      <th>study_name</th>\n      <th>trial_id</th>\n      <th>trial_number</th>\n      <th>value</th>\n      <th>lr</th>\n      <th>units</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>97</th>\n      <td>77</td>\n      <td>tpu_1</td>\n      <td>820</td>\n      <td>116</td>\n      <td>1.529258</td>\n      <td>0.000570</td>\n      <td>783.0</td>\n    </tr>\n    <tr>\n      <th>174</th>\n      <td>77</td>\n      <td>tpu_1</td>\n      <td>1003</td>\n      <td>203</td>\n      <td>1.532762</td>\n      <td>0.000541</td>\n      <td>730.0</td>\n    </tr>\n    <tr>\n      <th>146</th>\n      <td>77</td>\n      <td>tpu_1</td>\n      <td>943</td>\n      <td>173</td>\n      <td>1.533543</td>\n      <td>0.000547</td>\n      <td>830.0</td>\n    </tr>\n    <tr>\n      <th>131</th>\n      <td>77</td>\n      <td>tpu_1</td>\n      <td>907</td>\n      <td>156</td>\n      <td>1.534082</td>\n      <td>0.000562</td>\n      <td>779.0</td>\n    </tr>\n    <tr>\n      <th>145</th>\n      <td>77</td>\n      <td>tpu_1</td>\n      <td>940</td>\n      <td>172</td>\n      <td>1.535002</td>\n      <td>0.000557</td>\n      <td>821.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"from tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nimport os\n\n# pip install these if you haven't already. these are important for running mySQL with Optuna\n\n# Combine valid_dataset and test_dataset\ncombined_validation_dataset = valid_dataset.concatenate(test_dataset)\n\nfor i in range(1):\n\n    units = int(tpu_final.iloc[i][\"units\"])\n    lr = tpu_final.iloc[i][\"lr\"]\n    \n    print(f\"Model {i} with units: {units} and lr: {lr}\")\n\n\n\n    with strategy.scope():\n        \n        # Load the base pretrained model\n        base_model = ResNet50(weights='imagenet', include_top=False, input_shape=(224, 224, 3)) # weights = 'imagenet' means that it's using pretrained weights. include_top determines whether or not the fully-connected output layers of the model should be included.\n        # If include_top=False, the full model is loaded except for the output layers. This is useful if you want to fine-tune the model on a different task. By excluding the top layers, you can add your own output layers that are appropriate for your task.\n\n        for layer in base_model.layers:\n            layer.trainable = False\n\n        # Add a new top layer\n        x = base_model.output\n        x = GlobalAveragePooling2D()(x)\n        x = Dense(units, activation='relu')(x)\n        predictions = Dense(8, activation='softmax')(x)\n\n        # This is the model we will train\n        model = Model(inputs=base_model.input, outputs=predictions)\n\n        # Compile the model\n        model.compile(optimizer=Adam(learning_rate=lr),\n                      loss='categorical_crossentropy',\n                      metrics=['accuracy'])\n\n    \n        steps_per_epoch = (11569 // BATCH_SIZE) \n        \n        print(\"Steps_per_epoch:\", steps_per_epoch)\n        \n        # Train the model\n        model.fit(train_dataset, steps_per_epoch = steps_per_epoch, validation_data=valid_dataset, epochs=3, verbose = 1) # verbose = 0 will not display the progress bar\n    \n    \n    score = model.evaluate(test_dataset, verbose=1)\n    \n    print(score)\n    \n#     total_prediction_samples = 4464\n#     prediction_steps = -(-total_prediction_samples // BATCH_SIZE)  # Ceiling division\n#     predictions = model.predict(test_values_dataset, steps=prediction_steps, verbose=1)\n\n    #num_of_batches = 4464//BATCH_SIZE\n    num_of_batches = 2460//BATCH_SIZE\n    \n    predictions = model.predict(test_dataset, steps = num_of_batches, verbose = 1)\n#     for batch in test_dataset.take(1):  # Taking one batch from the dataset\n#         predictions = model.predict_on_batch(batch[0])  # batch[0] contains the images\n#         print(predictions.shape)\n\n        \n    \n#     for x in test_values_dataset.take(1):\n#         predictions = model.predict(x, verbose=1)\n#         print(predictions)\n\n    print(predictions)\n    \n#     # create predictions\n#     predictions = model.predict(test_values_dataset, verbose = 1)\n\n#     # Assuming 'test_values' DataFrame has an 'id' column and is in the same order as 'test_values_dataset'\n#     prediction_df = pd.DataFrame(predictions, columns=['blank', 'monkey_prosimian', 'civet_genet', 'rodent', 'antelope_duiker', 'hog', 'bird', 'leopard']) # make sure the column order matches the order of the tfrecord\n\n#     # Insert the 'id' column\n#     prediction_df.insert(0, 'id', test_values['id'].values)\n\n#     # Ensure 'id' is the first column\n#     prediction_df = prediction_df[['id'] + [col for col in prediction_df.columns if col != 'id']]\n\n#     # Define the new order of the columns\n#     new_order = ['id', 'antelope_duiker', 'bird', 'blank', 'civet_genet', 'hog', 'leopard', 'monkey_prosimian', 'rodent'] \n\n#     # Reorder the columns\n#     prediction_df = prediction_df.reindex(columns=new_order) # reorder the columns since that's how the drivendata competition wants it\n\n#     # Save the DataFrame to a CSV file\n#     prediction_df.to_csv('submission_ResNet50_test.csv', index=False) # make sure to change file name to represent the model number\n#     print(\"Successfully submitted!\")\n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:07:46.336773Z","iopub.execute_input":"2024-02-21T19:07:46.337035Z","iopub.status.idle":"2024-02-21T19:08:49.532298Z","shell.execute_reply.started":"2024-02-21T19:07:46.337009Z","shell.execute_reply":"2024-02-21T19:08:49.531047Z"},"trusted":true},"execution_count":33,"outputs":[{"name":"stdout","text":"Model 0 with units: 783 and lr: 0.0005701665095518\n","output_type":"stream"},{"name":"stderr","text":"WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\nI0000 00:00:1708542472.588247    6470 device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n","output_type":"stream"},{"name":"stdout","text":"Steps_per_epoch: 90\nEpoch 1/3\n","output_type":"stream"},{"name":"stderr","text":"2024-02-21 19:07:53.213224: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:53.235941: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:53.257185: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:53.278750: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:53.300442: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:53.323105: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:53.345230: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:07:53.367232: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:01.093589: E tensorflow/core/grappler/optimizers/meta_optimizer.cc:961] model_pruner failed: INVALID_ARGUMENT: Graph does not contain terminal node StatefulPartitionedCall.\n2024-02-21 19:08:01.863702: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:01.864899: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:01.869324: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:01.870983: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:01.970130: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:01.975600: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:01.999987: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.000422: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.019808: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.022048: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.026677: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.030807: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.035130: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.036397: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.040949: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:02.042118: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n","output_type":"stream"},{"name":"stdout","text":"\u001b[1m 3/90\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m6s\u001b[0m 79ms/step - accuracy: 0.7882 - loss: 17.8223","output_type":"stream"},{"name":"stderr","text":"2024-02-21 19:08:10.097211: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:10.097406: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:10.102816: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:10.108756: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:10.113718: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:10.114457: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:10.114866: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n2024-02-21 19:08:10.115229: E external/local_xla/xla/stream_executor/stream_executor_internal.h:177] SetPriority unimplemented for this stream.\n","output_type":"stream"},{"name":"stdout","text":"\u001b[1m90/90\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m33s\u001b[0m 183ms/step - accuracy: 1.2794 - loss: 17.1248 - val_accuracy: 0.2406 - val_loss: 1.9483\nEpoch 2/3\n\u001b[1m 3/90\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m4s\u001b[0m 51ms/step - accuracy: 1.2986 - loss: 16.3469","output_type":"stream"},{"name":"stderr","text":"/usr/local/lib/python3.10/contextlib.py:153: UserWarning: Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches. You may need to use the `.repeat()` function when building your dataset.\n  self.gen.throw(typ, value, traceback)\n","output_type":"stream"},{"name":"stdout","text":"\u001b[1m90/90\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 61ms/step - accuracy: 1.7177 - loss: 15.9263 - val_accuracy: 0.2812 - val_loss: 1.9919\nEpoch 3/3\n\u001b[1m90/90\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 65ms/step - accuracy: 1.8674 - loss: 15.5918 - val_accuracy: 0.1875 - val_loss: 1.8999\n\u001b[1m20/20\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 42ms/step - accuracy: 1.9479 - loss: nan\n[1.9006685018539429, 0.22499999403953552]\n()\n1023.9999\n","output_type":"stream"}]},{"cell_type":"code","source":"print(predictions.shape)\nprint(num_of_batches)","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:08:49.534075Z","iopub.execute_input":"2024-02-21T19:08:49.534369Z","iopub.status.idle":"2024-02-21T19:08:49.538809Z","shell.execute_reply.started":"2024-02-21T19:08:49.534338Z","shell.execute_reply":"2024-02-21T19:08:49.538010Z"},"trusted":true},"execution_count":34,"outputs":[{"name":"stdout","text":"()\n19\n","output_type":"stream"}]},{"cell_type":"code","source":"for batch in test_dataset.take(1):\n    print(batch[0].shape)  # Assuming batch[0] contains the images","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:08:49.539782Z","iopub.execute_input":"2024-02-21T19:08:49.540055Z","iopub.status.idle":"2024-02-21T19:08:49.642822Z","shell.execute_reply.started":"2024-02-21T19:08:49.540027Z","shell.execute_reply":"2024-02-21T19:08:49.641845Z"},"trusted":true},"execution_count":35,"outputs":[{"name":"stdout","text":"(128, 224, 224, 3)\n","output_type":"stream"}]},{"cell_type":"code","source":"# # Example to inspect the first batch of the dataset when it only contains inputs\n# step = 0\n# for x in test_values_dataset.take(200):\n#     print(\"Shape of x:\", x.shape)\n#     print(\"Type of x:\", x.dtype)\n#     step += 1\n    \n# print(step)\n    \n# print(test_values_dataset.take(1))","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:08:49.644185Z","iopub.execute_input":"2024-02-21T19:08:49.644478Z","iopub.status.idle":"2024-02-21T19:08:49.648382Z","shell.execute_reply.started":"2024-02-21T19:08:49.644449Z","shell.execute_reply":"2024-02-21T19:08:49.647634Z"},"trusted":true},"execution_count":36,"outputs":[]},{"cell_type":"code","source":"# # Example to inspect the first batch of the dataset when it contains both inputs and labels\n# step = 0\n# for features, labels in train_dataset.take(20000):\n# #     print(\"Shape of features:\", features.shape)\n# #     print(\"Type of features:\", features.dtype)\n# #     #If you also want to print the shape and type of labels\n# #     print(\"Shape of labels:\", labels.shape)\n# #     print(\"Type of labels:\", labels.dtype)\n#     step += 1\n    \n# print(step)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:08:49.649232Z","iopub.execute_input":"2024-02-21T19:08:49.649482Z","iopub.status.idle":"2024-02-21T19:08:49.662983Z","shell.execute_reply.started":"2024-02-21T19:08:49.649455Z","shell.execute_reply":"2024-02-21T19:08:49.662230Z"},"trusted":true},"execution_count":37,"outputs":[]},{"cell_type":"code","source":"# def count_dataset_items(tfrecord_file):\n#     count = 0\n#     for _ in tf.data.TFRecordDataset(tfrecord_file):\n#         count += 1\n#     return count\n\n# # Example usage\n# train_dataset_count = count_dataset_items('/kaggle/input/tfrecord/train.tfrecord')\n# print(\"Number of training examples:\", train_dataset_count)\n\n# # Example usage\n# train_dataset_count = count_dataset_items('/kaggle/input/tfrecord/valid.tfrecord')\n# print(\"Number of training examples:\", train_dataset_count)\n\n# # Example usage\n# train_dataset_count = count_dataset_items('/kaggle/input/tfrecord/test.tfrecord')\n# print(\"Number of training examples:\", train_dataset_count)\n\n# # Example usage\n# train_dataset_count = count_dataset_items('/kaggle/input/tfrecord/test_values.tfrecord')\n# print(\"Number of training examples:\", train_dataset_count)\n\n# print(BATCH_SIZE)\n\n# # Number of training examples: 11569\n# # Number of training examples: 2459\n# # Number of training examples: 2460\n# # Number of training examples: 4464","metadata":{"execution":{"iopub.status.busy":"2024-02-21T19:08:49.663808Z","iopub.execute_input":"2024-02-21T19:08:49.664083Z","iopub.status.idle":"2024-02-21T19:08:49.682813Z","shell.execute_reply.started":"2024-02-21T19:08:49.664047Z","shell.execute_reply":"2024-02-21T19:08:49.682061Z"},"trusted":true},"execution_count":38,"outputs":[]}]}