Molecular_GAN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "37706c9c-1ef4-4051-b17e-e54aa0664180",
   "metadata": {},
   "source": [
    "# Generative Adversarial Network for Molecular Graph images\n",
    "\n",
    "This block\n",
    "\n",
    "- imports all modules/libraries needed\n",
    "- contains a function for a learning rate scheduler (if needed)\n",
    "- reads in SMILES strings and creates a list\n",
    "- featurizes via the SmilesTo Image featurizer from Deepchem (https://github.com/deepchem/deepchem)\n",
    "- Four channel image used for featurization\n",
    "  * Atomic numbers + bonds\n",
    "  * partial charges\n",
    "  * valence\n",
    "  * hybridization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "979d0bc1-0585-4b38-903d-b64edabe0891",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\rdkitenv\\lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\rdkitenv\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py:588: calling function (from tensorflow.python.eager.polymorphic_function.polymorphic_function) with experimental_relax_shapes is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "experimental_relax_shapes is deprecated, use reduce_retracing instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipped loading modules with pytorch-geometric dependency, missing a dependency. No module named 'torch_geometric'\n",
      "Skipped loading modules with pytorch-geometric dependency, missing a dependency. cannot import name 'DMPNN' from 'deepchem.models.torch_models' (C:\\ProgramData\\Anaconda3\\envs\\rdkitenv\\lib\\site-packages\\deepchem\\models\\torch_models\\__init__.py)\n",
      "Skipped loading some Jax models, missing a dependency. No module named 'jax'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features and datapoints:  (6731, 80, 80, 4)\n",
      "featurization done with:  SmilesToImage\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd \n",
    "import deepchem as dc\n",
    "import time\n",
    "from rdkit import Chem\n",
    "from functools import partial\n",
    "import matplotlib.pyplot as plt\n",
    "from rdkit.Chem import AllChem, Draw\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def exp_decay(lr0,s):\n",
    "    def exp_decay_fcn(epoch):\n",
    "        return lr0*0.1**(epoch/s)\n",
    "    return exp_decay_fcn\n",
    "\n",
    "#df = pd.read_csv(\"392-azo.csv\")\n",
    "#df = pd.read_csv(\"212-azo.csv\")\n",
    "#df = pd.read_csv(\"604-azo.csv\")\n",
    "#df = pd.read_csv(\"621-azo.csv\")\n",
    "df = pd.read_csv(\"6731-azo.csv\")\n",
    "\n",
    "Xa = [\"\"]*len(df)\n",
    "w = [\"\"]*len(df)\n",
    "i=0\n",
    "for name in df[\"SMILES\"]:\n",
    "    Xa[i]=name\n",
    "    w[i]=1.0\n",
    "    i += 1\n",
    "Xa = list(Xa)\n",
    "w=list(w)\n",
    "\n",
    "#===========================================================================================\n",
    "#featurizer\n",
    "\n",
    "featurizer=dc.feat.SmilesToImage(img_spec=\"engd\")\n",
    "featname=\"SmilesToImage\"\n",
    "\n",
    "f = featurizer.featurize(Xa)\n",
    "print(\"Number of features and datapoints: \",f.shape)\n",
    "\n",
    "#===========================================================================================\n",
    "print(\"featurization done with: \",featname)\n",
    "\n",
    "# set npp for how many points in the dataset, len(df) for whole dataset\n",
    "nnp=len(df)\n",
    "\n",
    "#set y to lambdamax\n",
    "y = df[\"Lmax\"]\n",
    "\n",
    "#set dataset and split\n",
    "\n",
    "dataset = dc.data.NumpyDataset(X=f[:nnp,:,:,:], y=y[:nnp], ids=Xa[:nnp], w=w[:nnp])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0dc88c3-17a9-48af-8032-4b0cf8d34951",
   "metadata": {},
   "source": [
    "## Define the Discriminator and Generator\n",
    "\n",
    "- architecture adapted from https://github.com/davidADSP/Generative_Deep_Learning_2nd_Edition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0a3fa376-fceb-4b59-8ef1-0ae20a1718c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generator and Discriminator built.\n"
     ]
    }
   ],
   "source": [
    "embsize=128\n",
    "dropout_rate = 0.45\n",
    "\n",
    "#=======================================================================================================\n",
    "\n",
    "discriminator_input = tf.keras.layers.Input(shape=[80,80,4])\n",
    "x = tf.keras.layers.Conv2D(filters=64,kernel_size=(4,4), strides=2,padding=\"same\", use_bias = False)(discriminator_input)\n",
    "x = tf.keras.layers.BatchNormalization()(x)\n",
    "x = tf.keras.layers.LeakyReLU(0.2)(x)\n",
    "x = tf.keras.layers.Dropout(dropout_rate)(x)\n",
    "x = tf.keras.layers.Conv2D(filters=128,kernel_size=(4,4), strides=2,padding=\"same\", use_bias = False)(x)\n",
    "x = tf.keras.layers.BatchNormalization(momentum=0.9)(x)\n",
    "x = tf.keras.layers.LeakyReLU(0.2)(x)\n",
    "x = tf.keras.layers.Dropout(dropout_rate)(x)\n",
    "x = tf.keras.layers.Conv2D(filters=256,kernel_size=(4,4), strides=2,padding=\"same\", use_bias = False)(x)\n",
    "x = tf.keras.layers.BatchNormalization(momentum=0.9)(x)\n",
    "x = tf.keras.layers.LeakyReLU(0.2)(x)\n",
    "x = tf.keras.layers.Dropout(dropout_rate)(x)\n",
    "x = tf.keras.layers.Conv2D(filters=512,kernel_size=(4,4), strides=2,padding=\"same\", use_bias = False)(x)\n",
    "x = tf.keras.layers.BatchNormalization(momentum=0.9)(x)\n",
    "x = tf.keras.layers.LeakyReLU(0.2)(x)\n",
    "x = tf.keras.layers.Dropout(dropout_rate)(x)\n",
    "x = tf.keras.layers.Conv2D(filters=1,kernel_size=(5,5), strides=1,padding=\"valid\", use_bias = False,\n",
    "                          activation=\"sigmoid\")(x)\n",
    "discriminator_output = tf.keras.layers.Flatten()(x)\n",
    "\n",
    "discriminator = tf.keras.models.Model(discriminator_input,discriminator_output)\n",
    "#=======================================================================================================\n",
    "\n",
    "generator_input = tf.keras.layers.Input(shape=(embsize,))\n",
    "x = tf.keras.layers.Reshape((1,1,embsize))(generator_input)\n",
    "x = tf.keras.layers.Conv2DTranspose(filters=512,kernel_size=(5,5),strides=1, padding=\"valid\", use_bias = False)(x)\n",
    "x = tf.keras.layers.BatchNormalization(momentum=0.9)(x)\n",
    "x = tf.keras.layers.LeakyReLU(0.2)(x)\n",
    "x = tf.keras.layers.Conv2DTranspose(filters=256,kernel_size=(4,4),strides=2, padding=\"same\", use_bias = False)(x)\n",
    "x = tf.keras.layers.BatchNormalization(momentum=0.9)(x)\n",
    "x = tf.keras.layers.LeakyReLU(0.2)(x)\n",
    "x = tf.keras.layers.Conv2DTranspose(filters=128,kernel_size=(4,4),strides=2, padding=\"same\", use_bias = False)(x)\n",
    "x = tf.keras.layers.BatchNormalization(momentum=0.9)(x)\n",
    "x = tf.keras.layers.LeakyReLU(0.2)(x)\n",
    "x = tf.keras.layers.Conv2DTranspose(filters=64,kernel_size=(4,4),strides=2, padding=\"same\", use_bias = False)(x)\n",
    "x = tf.keras.layers.BatchNormalization(momentum=0.9)(x)\n",
    "x = tf.keras.layers.LeakyReLU(0.2)(x)\n",
    "generator_output = tf.keras.layers.Conv2DTranspose(filters=4,kernel_size=(4,4),strides=2, padding=\"same\", \n",
    "                                                   activation = \"sigmoid\", use_bias = False)(x)  #using activation = \"sigmoid\" here leads to blue backgrounds\n",
    "\n",
    "generator = tf.keras.models.Model(generator_input,generator_output)\n",
    "\n",
    "print(\"Generator and Discriminator built.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7aae5b43-eed5-4e85-b0c0-c2693688b2c5",
   "metadata": {},
   "source": [
    "## Training for the GAN\n",
    "\n",
    "- training loop adapted from https://github.com/davidADSP/Generative_Deep_Learning_2nd_Edition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b4be5e51-b8d8-4d00-bc94-252fbe1b4bcb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "211/211 [==============================] - 106s 483ms/step - d_loss: 1.9860 - g_loss: 0.0859\n",
      "Epoch 2/2\n",
      "211/211 [==============================] - 112s 533ms/step - d_loss: 2.6968 - g_loss: 0.0561\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x239087452a0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class GAN(tf.keras.models.Model):\n",
    "    def __init__(self,discriminator,generator,latent_dim):\n",
    "        super(GAN, self).__init__()\n",
    "        self.discriminator = discriminator\n",
    "        self.generator = generator\n",
    "        self.latent_dim = latent_dim\n",
    "\n",
    "    def compile(self, d_optimizer, g_optimizer):\n",
    "        super(GAN,self).compile()\n",
    "        self.loss_fn = tf.keras.losses.BinaryCrossentropy()\n",
    "        self.d_optimizer = d_optimizer\n",
    "        self.g_optimizer = g_optimizer\n",
    "        self.d_loss_metric = tf.keras.metrics.Mean(name=\"d_loss\")\n",
    "        self.g_loss_metric = tf.keras.metrics.Mean(name=\"g_loss\")\n",
    "\n",
    "    @property\n",
    "    def metrics(self):\n",
    "        return [ self.d_loss_metric, self.g_loss_metric]\n",
    "\n",
    "    def train_step(self, real_images):\n",
    "        batch_size = tf.shape(real_images)[0]\n",
    "        random_latent_vectors = tf.random.normal(shape=(batch_size,self.latent_dim))\n",
    "\n",
    "        with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n",
    "            generated_images = self.generator(random_latent_vectors, training = True)\n",
    "            real_predictions = self.discriminator(real_images, training = True)\n",
    "            fake_predictions = self.discriminator(generated_images, training = True)\n",
    "\n",
    "            real_labels = tf.ones_like(real_predictions)\n",
    "            real_noisy_labels = real_labels + 0.1*tf.random.uniform(tf.shape(real_predictions))\n",
    "            fake_labels = tf.zeros_like(fake_predictions)\n",
    "            fake_noisy_labels = fake_labels + 0.1*tf.random.uniform(tf.shape(fake_predictions))\n",
    "\n",
    "            d_real_loss = self.loss_fn(real_noisy_labels, real_predictions)\n",
    "            d_fake_loss = self.loss_fn(fake_noisy_labels, fake_predictions)\n",
    "            d_loss = (d_real_loss + d_fake_loss) / 2.0\n",
    "            g_loss = self.loss_fn(real_labels, fake_predictions)\n",
    "\n",
    "        gradients_of_discriminator = disc_tape.gradient(d_loss, self.discriminator.trainable_variables)\n",
    "        gradients_of_generator = gen_tape.gradient(g_loss, self.generator.trainable_variables)\n",
    "\n",
    "        self.d_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))\n",
    "        self.g_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))\n",
    "\n",
    "        self.d_loss_metric.update_state(d_loss)\n",
    "        self.g_loss_metric.update_state(g_loss)\n",
    "\n",
    "        return {m.name: m.result() for m in self.metrics}\n",
    "\n",
    "dcgan = GAN(discriminator = discriminator, generator = generator, latent_dim = embsize)\n",
    "\n",
    "dcgan.compile(d_optimizer = tf.keras.optimizers.SGD(learning_rate = 0.0001),\n",
    "             g_optimizer = tf.keras.optimizers.Adam(learning_rate = 0.001, beta_1 = 0.5, beta_2 = 0.999))\n",
    "dcgan.fit(dataset.X, epochs = 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40fa585e-c87d-4f2d-ad70-31c6fab66be1",
   "metadata": {},
   "source": [
    "## Plot generated images\n",
    "\n",
    "- use the generator to make a couple of images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "67b11efd-d5fa-4ebf-b03b-94ce46795f2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 194ms/step\n",
      "(2, 80, 80, 4)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9wAAAHuCAYAAABtfQSSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8gElEQVR4nO3de3RV5Z0//k8E5BarCEpAFLAoaAEthbaCVQEVEKuCTrGOjlovIAo6DipQLYj1UtRRhpGfVEdpXaNgrQsVB7xQ6ppCHeQL4wUsHS9YFfDCrYhFuezfH13J8ngOkJA8SQiv11r8kc959t7P3jk7J2+ecz4pyrIsCwAAAKBK7VPTEwAAAIC6SOAGAACABARuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABARuAAAASGC3Avdrr70Wl1xySXzzm9+Mxo0bR+PGjeOII46IoUOHxqJFi6p6jjVqwYIFMX78+Fi/fn2V7/uiiy6Kdu3a7XLcSSedFJ07d67y4++p+vbtG8OGDavQNr///e+jqKgonnjiiUSzql4rVqyIoqKimDZtWqX3tXHjxrj++uvj1FNPjYMOOiiKiopi/Pjxldrn+eefH0VFRXH66afn1NetWxcHHHBAzJw5s1L7BwCAPUGFA/fUqVPjO9/5TvzP//xPXH311TFr1qx49tln45prromlS5dGjx494u23304x1xqxYMGCuPnmm5MEbiruqaeeivnz58dNN91U01OpM9asWRO//OUv44svvoizzjqr0vt79tlnY+bMmfGNb3wj77FmzZrFP//zP8d1110XX375ZaWPBQAAtVmFAvf8+fNj+PDhMWDAgFi8eHGMHDky+vbtG3369Ikrr7wy/vCHP8Tjjz8ejRs3TjXfSvv8889regpUwm233RaDBg2KQw45pKanUme0bds21q1bFy+99FLcfvvtldrXhg0bYujQoXHLLbdEs2bNCo4ZNmxYrFixos682wAAAHakQoH7tttui3r16sXUqVNj3333LTjmH/7hH6J169Y5tUWLFsUZZ5wRBx54YDRq1Ci+/e1vx+OPP54zZtq0aVFUVBTz5s2LK664Ilq0aBHNmzePwYMHx8qVK/OOM2PGjDjuuOOiadOmUVxcHP369YslS5bkjLnooouiuLg4Xn/99Tj11FNjv/32i759+0ZExAsvvBBnnnlmtGnTJho1ahQdOnSIoUOHxqefflq2/fjx4+O6666LiIj27dtHUVFRFBUVxe9///sKzaP0/Dp27BgNGzaMo446Kn7961/v5ErvWlFRUVx11VXx8MMPR8eOHaNx48bRvXv3ePnllyPLsrjzzjujffv2UVxcHH369Im33norZ/vynH+pp556Krp27RoNGzaMww8/PCZNmhTjx4+PoqKinHFZlsWUKVPi2GOPjcaNG0ezZs3inHPOiXfeeSdn3JIlS+L000+Pgw8+OBo2bBitW7eOgQMHxgcffLDTc16yZEksXLgwLrjggrzHPvzww7j88svj0EMPjX333Tdat24d55xzTnz00Uc547Zs2RI//elPo3Xr1vGNb3wjTj755Fi+fPluXZvSa7B06dL48Y9/HPvvv3+0bNkyfvKTn8SGDRtyxpZ+vx555JE46qijokmTJnHMMcfErFmz8s7l//7v/+K8884ruz5HHXVU3HfffTu9NhERn3zySdk1aNiwYRx00EHRq1evePHFF3e6Xenzuir8y7/8S7Rq1SpGjhy5wzEtW7aMU045Je6///4qOSYAANRW9cs7cNu2bTFv3rzo3r17tGrVqtwHmDdvXvTv3z++973vxf333x/7779/TJ8+PYYMGRKff/55XHTRRTnjL7300hg4cGA8+uij8f7778d1110X559/fvzud78rG3PbbbfFjTfeGBdffHHceOON8eWXX8add94ZP/jBD2LhwoVx9NFHl4398ssv44wzzoihQ4fG6NGjY+vWrRER8fbbb8dxxx0Xl156aey///6xYsWK+Nd//dc4/vjj4/XXX48GDRrEpZdeGmvXro3JkyfHk08+WXbepfsv7zymTZsWF198cZx55plx9913x4YNG2L8+PHxxRdfxD777H7fulmzZsWSJUvijjvuiKKiorjhhhti4MCBceGFF8Y777wT//7v/x4bNmyIa6+9Ns4+++z43//937JgVZ7zj4iYM2dODB48OE444YSYMWNGbN26Ne666668IBsRMXTo0Jg2bVqMHDkyfvGLX8TatWtjwoQJ0bNnz3j11VejZcuWsWnTpjjllFOiffv2cd9990XLli1j9erVMW/evNi4ceMuz7devXpxwgkn5NQ//PDD6NGjR2zZsiXGjh0bXbt2jTVr1sRzzz0X69ati5YtW5aNHTt2bPTq1SsefPDB+Otf/xo33HBD/PCHP4w333wz6tWrV6FrU+rss8+OIUOGxCWXXBKvv/56jBkzJiIiHnrooZxxzz77bLzyyisxYcKEKC4ujokTJ8agQYNi+fLlcfjhh0dExLJly6Jnz55x2GGHxd133x0lJSXx3HPPxciRI+PTTz+NcePG7fD6XHDBBbF48eK49dZb48gjj4z169fH4sWLY82aNTu9rlXlxRdfjF//+tfxyiuvlF3LHTnppJNizJgxsX79+jjggAOqZX4AAFDtsnJavXp1FhHZueeem/fY1q1bsy1btpT92759e9ljnTp1yr797W9nW7Zsydnm9NNPz1q1apVt27Yty7Ise/jhh7OIyIYPH54zbuLEiVlEZKtWrcqyLMv+8pe/ZPXr189GjBiRM27jxo1ZSUlJ9qMf/aisduGFF2YRkT300EM7Pbft27dnW7Zsyd57770sIrKnnnqq7LE777wzi4js3XffzdmmvPPYtm1b1rp166xbt24512XFihVZgwYNsrZt2+50blmWZSeeeGL2rW99K6cWEVlJSUn22WefldVmzpyZRUR27LHH5hzr3nvvzSIie+211yp8/j169MgOPfTQ7Isvvsg5x+bNm2dfffr88Y9/zCIiu/vuu3P2/f7772eNGzfOrr/++izLsmzRokVZRGQzZ87c5Xl/3YABA7JOnTrl1X/yk59kDRo0yJYtW7bDbefNm5dFRHbaaafl1B9//PEsIrI//vGPBbfb2bUZN25cFhHZxIkTc7YZPnx41qhRo5zvQURkLVu2zP7617+W1VavXp3ts88+2e23315W69evX9amTZtsw4YNOfu86qqrskaNGmVr167NsizL3n333SwisocffrhsTHFxcXbNNdfs8BqUxyeffJJFRDZu3LgKbbdx48asXbt22ZgxY8pqbdu2zQYOHFhw/AsvvJBFRDZ79uzKTBcAAGq1KvmzYN/5zneiQYMGZf/uvvvuiIh466234k9/+lP84z/+Y0REbN26tezfaaedFqtWrcp7O+8ZZ5yR83XXrl0jIuK9996LiIjnnnsutm7dGv/0T/+Us79GjRrFiSeemPN271Jnn312Xu3jjz+OYcOGxaGHHhr169ePBg0aRNu2bSMi4s0339zlOZd3HsuXL4+VK1fGeeedl/O23bZt20bPnj13eZyd6d27dzRt2rTs66OOOioiIgYMGJBzrNJ66TWMKN/5b9q0KRYtWhRnnXVWzkcIiouL44c//GHOXGbNmhVFRUVx/vnn51yPkpKSOOaYY8quR4cOHaJZs2Zxww03xP333x/Lli0r9/muXLkyDj744Lz67Nmzo3fv3mXnuTO7en5FVPy5UWifmzdvjo8//jin3rt379hvv/3Kvm7ZsmUcfPDBZcfevHlzzJ07NwYNGhRNmjTJu182b94cL7/88g7P7bvf/W5MmzYtfv7zn8fLL78cW7Zs2eX1qIjt27fnzGnbtm1lj40ePToaNGgQP/vZz8q1r9Lv44cfflilcwQAgNqk3G8pb9GiRTRu3DgnmJR69NFH4/PPP49Vq1blhI/Stx2PGjUqRo0aVXC/X/9cbPPmzXO+btiwYURE/O1vf8vZZ48ePQru7+tv0W7SpElet+Tt27fHqaeeGitXroybbropunTpEk2bNo3t27fH97///bJj7Ux551H6dt6SkpK8MSUlJbFixYpdHmtHDjzwwJyvS0PxjuqbN2+OiPKf/7p16yLLspy3ZJf6eu2jjz7a4diIKHvL9P777x8vvfRS3HrrrTF27NhYt25dtGrVKi677LK48cYb896u/VV/+9vfCu7/k08+iTZt2uxwu6/a1fNrd54bu9rnjsaVji0dt2bNmti6dWtMnjw5Jk+eXHD+hT5jX2rGjBnx85//PB588MG46aabori4OAYNGhQTJ04s+PyrqAkTJsTNN99c9nXbtm1jxYoVsXDhwpgyZUo8+eSTsXnz5pzn2datW2P9+vXRuHHjsusSEdGoUaOIyL9GAABQl5Q7cNerVy/69OkTzz//fKxatSrnc9yln1X+enhs0aJFRESMGTMmBg8eXHC/HTt2rNCES/f5xBNPlK067kyhZlBvvPFGvPrqqzFt2rS48MILy+pfbyxWFfMoDVmrV6/Oe6xQrTqU9/ybNWsWRUVFBT+v/fW5t2jRIoqKiuK///u/c4JVqa/WunTpEtOnT48sy+K1116LadOmxYQJE6Jx48YxevToHc67RYsWsXbt2rz6QQcdtMuGa+VVFc+N3dWsWbOoV69eXHDBBXHllVcWHNO+ffsdbt+iRYu499574957742//OUv8fTTT8fo0aPj448/jjlz5lR6fpdffnnO39Uu/Z4uW7YssiyLQYMG5W3z/vvvR7NmzeKee+6Ja665pqxe+n0svY8AAKAuKnfgjvh7cJ49e3YMGzYsnnjiiZ2uRkb8PUwfccQR8eqrr8Ztt91WqYmW6tevX9SvXz/efvvtgm8VL4/SEP71YDh16tS8sTtarSzvPDp27BitWrWKxx57LK699tqyY7/33nuxYMGCvI7u1aG859+0adPo3r17zJw5M+66666ylfLPPvssr7v26aefHnfccUd8+OGH8aMf/ajc8zjmmGPinnvuiWnTpsXixYt3Or5Tp04xc+bMvPqAAQPikUceieXLl1f4P3AKzSmifM+NqtakSZPo3bt3LFmyJLp27brDvwRQHocddlhcddVVMXfu3Jg/f36VzK9169YFn6/9+/ePefPm5dXPPffcaN++fdx+++3RoUOHnMdKO9d/tcEhAADUNRUK3L169Yr77rsvRowYEd26dYvLL788vvWtb8U+++wTq1atit/+9rcRETlv4Z46dWoMGDAg+vXrFxdddFEccsghsXbt2njzzTdj8eLF8Zvf/KZCE27Xrl1MmDAhfvrTn8Y777wT/fv3j2bNmsVHH30UCxcujKZNm+a87bWQTp06xTe/+c0YPXp0ZFkWBx54YDzzzDPxwgsv5I3t0qVLRERMmjQpLrzwwmjQoEF07Nix3PPYZ5994pZbbolLL700Bg0aFJdddlmsX78+xo8fXyVv890dFTn/CRMmxMCBA6Nfv35x9dVXx7Zt2+LOO++M4uLinNXmXr16xeWXXx4XX3xxLFq0KE444YRo2rRprFq1Kv7whz9Ely5d4oorrohZs2bFlClT4qyzzorDDz88siyLJ598MtavXx+nnHLKTud90kknxUMPPRR//vOf48gjj8yZ4+zZs+OEE06IsWPHRpcuXWL9+vUxZ86cuPbaa6NTp05Jrk0KkyZNiuOPPz5+8IMfxBVXXBHt2rWLjRs3xltvvRXPPPNMTrf+r9qwYUP07t07zjvvvOjUqVPst99+8corr5R1md+V2bNnx6ZNm8o6xS9btqzs72Sfdtpp0aRJkx1uW1JSUvC53KhRo2jevHmcdNJJeY+9/PLL0bx587L7CwAA6qIKBe6IiGHDhsVxxx0XkyZNinvuuSdWrlwZRUVF0aZNm+jZs2fMnTs3+vTpUza+d+/esXDhwrj11lvjmmuuiXXr1kXz5s3j6KOPLvdK6NeNGTMmjj766Jg0aVI89thj8cUXX0RJSUn06NEjhg0btsvtGzRoEM8880xcffXVMXTo0Khfv36cfPLJ8eKLL8Zhhx2WM7b0zxf96le/igceeCC2b98e8+bNK6uXZx6XXHJJRET84he/iMGDB0e7du1i7Nix8dJLLxVs8pZaRc6/f//+8dvf/jZ+9rOfxZAhQ6KkpCSGDx8eK1eujEceeSRn7NSpU+P73/9+TJ06NaZMmRLbt2+P1q1bR69eveK73/1uREQcccQRccABB8TEiRNj5cqVse+++0bHjh3z3sJdyJlnnhnFxcXx1FNPlf199IiIQw45JBYuXBjjxo2LO+64I9asWRMHHXRQHH/88XmfZ6/Ka5PC0UcfHYsXL45bbrklbrzxxvj444/jgAMOiCOOOCJOO+20HW7XqFGj+N73vhePPPJIrFixIrZs2RKHHXZY3HDDDXH99dfv8rhXXHFFTn+G3/zmN2X/Gfbuu+9Gu3btKn1upbIsi6effjqvkSAAANQ1RVmWZTU9CfYsW7ZsiWOPPTYOOeSQeP7556v12CNGjIi5c+fG0qVLhbU91Ny5c+PUU0+NpUuXVujdBwAAsKcRuNmlSy65JE455ZRo1apVrF69Ou6///546aWX4vnnn4+TTz65Wufy0UcfxZFHHhn/8R//Eeecc061Hpuq0bt37+jQoUM88MADNT0VAABIqsJvKWfvs3Hjxhg1alR88skn0aBBg+jWrVv813/9V7WH7Yi//zmy//zP/4x169ZV+7GpvHXr1sWJJ54Yw4cPr+mpAABAcla4AQAAIIF9anoCAAAAUBcJ3AAAAJCAwA0AAAAJCNwAAACQQJ3oUt7tlt/V9BTqpJ6dW+XVFryxqgZmQlVZfFOfmp7CDrmPoXxq830c4V6G8nIvQ92wq3vZCjcAAAAkIHADAABAAgI3AAAAJFAnPsNNGj6vDQAAsPuscAMAAEACAjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAICNwAAACQgcAMAAEAC9Wt6AlS/np1b5dUWvLGqBmZS+7g2AABAVbHCDQAAAAkI3AAAAJCAwA0AAAAJCNwAAACQgKZpe6HKNAFb1HRcXq37ppsrM52Caqp5mQZpAABAVbHCDQAAAAkI3AAAAJCAwA0AAAAJCNwAAACQgMANAAAACehSToWk6EheSE11C6+p7uhAWu5tAKAmWOEGAACABARuAAAASEDgBgAAgAQEbgAAAEhA0zR26OSPPsur3XHZhrzar748Ma82ec5bSeaUmiZKUDe5twGAmmCFGwAAABIQuAEAACABgRsAAAASELgBAAAgAU3T2KEXWxbn1bo/nV+LqFyDtJ6dW+XVqrrBUV05BgCkNH1Im7zauTM+qIGZwN6t23uP5tUWtz2vUvsc0b9DXm1PbXS8J7HCDQAAAAkI3AAAAJCAwA0AAAAJCNwAAACQgKZpVNqMc7vm1YZMf63c29eV5mUapEFa1wx5J69274zDa2AmUHftjQ3SND2lNqpsg7RC/t8Hm6p8n+yaFW4AAABIQOAGAACABARuAAAASEDgBgAAgAQ0TaPSKtIgrTpUR6MTDVag+mmQBqTg9Zu9hed6zbDCDQAAAAkI3AAAAJCAwA0AAAAJCNwAAACQgKZpJDF9SJuC9XNnfFDNM0lD0wkAAGBXrHADAABAAgI3AAAAJCBwAwAAQAICNwAAACSgaRpJ1JXmaADs2Xp2blWucZphApCCFW4AAABIQOAGAACABARuAAAASEDgBgAAgAQ0TaPGTR/SJq+m6RoAVUEzNABqkhVuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAENE2r4y5rsTyv1rdv33JtW12NyzRIA+qKnp1b5dU07YLaodD9GVH4HnUvszN39cqvtWmjCXAKdeFetMINAAAACQjcAAAAkIDADQAAAAkI3AAAAJCApml13AOfdsyvVaKBw/QhVd8Qom+f5/Jqc3/Xr1L7BKgJe1ojF9ibVOT+dC+zM6PmF6pqkJZCXbgXrXADAABAAgI3AAAAJCBwAwAAQAICNwAAACSgaRoVUtkGaYVokAYA1W/GuV3zalm2Nq+W4rW/PHp2bpVXqwsNlEjL84baxgo3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACmqbthfr2eS6vpnEZAOxdhkx/raansFMaXbE7quN5U6gx27dW/z6v9sCnHZPPhdrPCjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAKapu2FaluDtEKNJ2pTo5RFTcfl1bpvurkGZkJ1qu3PS6B83MtAVSv0M2RBaJBGYVa4AQAAIAGBGwAAABIQuAEAACABgRsAAAAS0DStBlV1I5cR/Tvk1SbPeWu391ddaqp5TaHrVUj3ORqk7Y0KPS/Le8/uqfci1EV7YoM0P0MA6g4r3AAAAJCAwA0AAAAJCNwAAACQgMANAAAACQjcAAAAkIAu5TWoqjun1qYOptOHtClYP3fGB9U8kx2rTdeLPUN579n/98GmxDOpHnvCfQx1kdenitHVnb2F1+U9kxVuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAENE0jiQf/7U+FH2hZnFcq1ABC8wf2ZFXdELGm7I33oeZLsOdxj7K32Btfl+sCK9wAAACQgMANAAAACQjcAAAAkIDADQAAAAlomkYSLxZojrYjH3ygAQTURtcMeadg/d4Zh1fzTKqP5kvs6SrbiPSyFsvzag982rFScwLYm1nhBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABIQNM0kujZuVXB+oI3VuXVRs1PPRvgqxY1HZdX677p5rzakQccsYM9bKviGQG7MqJ/h7xaoSZ/FWmQVogGaQBVywo3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACmqaRRKHmaBERd/XKr2maBtWrUIO0QoZPrTvN0Qo1ctzRzymojQo1SKus6UPalGtcZRuxAezNrHADAABAAgI3AAAAJCBwAwAAQAICNwAAACSgadoeoC41+9EgDagJe+rPTEipUDO0+37crcDI2tM0rby/E9Wl352APZsVbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABDRNq2XK2+RjREnTvNrk1ZvyatOHtMmr/dvSbeU6Bn+3qPOTebXubwyugZkAtYmmTFTUiP4d8mqT57xVAzPZsSsfW1zTU9ip8t5j7kV25c5ebfNq181/rwZmQl1nhRsAAAASELgBAAAgAYEbAAAAEhC4AQAAIAFN06pJeZvrFKrd1St/f6Pm5zdIK+TcGR+Ua9zeqFBDufkbGuXVus/RIA3IpykTFVXbGqTVdtXVmFADxL1TeRukXTPknbzavTMOr+rpUIdZ4QYAAIAEBG4AAABIQOAGAACABARuAAAASEDTtGpSmeYbo+ZX4UT2YNXR1ERDGwCofoUamVZX41cN0tiZut4grXBz5t3f32UtlufVHvi04+7vsA6wwg0AAAAJCNwAAACQgMANAAAACQjcAAAAkICmaewxKtPUZNEZH+bVRi7Nb8IGAFS/6mqQBuSq6ubMe3uDtEKscAMAAEACAjcAAAAkIHADAABAAgI3AAAAJKBpGpV2WYvlebXKNkyYPqRNXm3u3LnlOs6Mc7vm1Ua+kd8g7Tuf/jWvtqC8EwSAPcRdvQrXCzVLKvT6q6EZ1A49Oxdu+FuosXC39x7Nq11//fV5taq+v/0MyWeFGwAAABIQuAEAACABgRsAAAASELgBAAAgAU3T9kKFGi4UarZQyH0/7pZXu/KxSk8pT+HmCuVrxDbpjU/yaoXOT4M0APYGhZqj7ci/Ld2WbiJQh1Xm9+vyqsj+Frc9L69WHc3LKnOM6riGNcEKNwAAACQgcAMAAEACAjcAAAAkIHADAABAApqm7YUq03zgyscWl2vcXb0K10et/P/yi+9esdvzmTK0Xl5t+NQ9v7kCANSEutCgCGpCee+dFI3BCjc1Lt/v7LVJXf35Y4UbAAAAEhC4AQAAIAGBGwAAABIQuAEAACABgRsAAAAS0KWcJEbNL1yfMvSqvNrwqdt2+ziV2RYAAKpTZTpx9+3zXMH6lY/t9i5jRP8OebXJc97a/R2Sxwo3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACmqZRrTQ5AwCAnevZuVVebe7v+lX5cTRIS88KNwAAACQgcAMAAEACAjcAAAAkIHADAABAApqmAQB1VqHGQwveWFUDMwEoPz+n6g4r3AAAAJCAwA0AAAAJCNwAAACQgMANAAAACWiathda1PnJvNrIuDKvVpPNGkb075BXmzznrRqYCQB7Mo2HAP7O79c1wwo3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACmqbVcT07t8qrdX9jcIGRtaupTFU3cCh0HTTSAQBgb6FBWs2wwg0AAAAJCNwAAACQgMANAAAACQjcAAAAkICmaXWcxmB/5zoAAFBKQ12qixVuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEirIsy2p6EgAAAFDXWOEGAACABARuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABARuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABARuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABARuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABARuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEditwv/baa3HJJZfEN7/5zWjcuHE0btw4jjjiiBg6dGgsWrSoqudYoxYsWBDjx4+P9evXV/m+L7roomjXrt0ux5100knRuXPnKj/+nqpv374xbNiwCm3z+9//PoqKiuKJJ55INKvqtWLFiigqKopp06ZVel8bN26M66+/Pk499dQ46KCDoqioKMaPH1+pfZ5//vlRVFQUp59+ek593bp1ccABB8TMmTMrtX8AANgTVDhwT506Nb7zne/E//zP/8TVV18ds2bNimeffTauueaaWLp0afTo0SPefvvtFHOtEQsWLIibb745SeCm4p566qmYP39+3HTTTTU9lTpjzZo18ctf/jK++OKLOOussyq9v2effTZmzpwZ3/jGN/Iea9asWfzzP/9zXHfddfHll19W+lgAAFCbVShwz58/P4YPHx4DBgyIxYsXx8iRI6Nv377Rp0+fuPLKK+MPf/hDPP7449G4ceNU8620zz//vKanQCXcdtttMWjQoDjkkENqeip1Rtu2bWPdunXx0ksvxe23316pfW3YsCGGDh0at9xySzRr1qzgmGHDhsWKFSvqzLsNAABgRyoUuG+77baoV69eTJ06Nfbdd9+CY/7hH/4hWrdunVNbtGhRnHHGGXHggQdGo0aN4tvf/nY8/vjjOWOmTZsWRUVFMW/evLjiiiuiRYsW0bx58xg8eHCsXLky7zgzZsyI4447Lpo2bRrFxcXRr1+/WLJkSc6Yiy66KIqLi+P111+PU089Nfbbb7/o27dvRES88MILceaZZ0abNm2iUaNG0aFDhxg6dGh8+umnZduPHz8+rrvuuoiIaN++fRQVFUVRUVH8/ve/r9A8Ss+vY8eO0bBhwzjqqKPi17/+9U6u9K4VFRXFVVddFQ8//HB07NgxGjduHN27d4+XX345siyLO++8M9q3bx/FxcXRp0+feOutt3K2L8/5l3rqqaeia9eu0bBhwzj88MNj0qRJMX78+CgqKsoZl2VZTJkyJY499tho3LhxNGvWLM4555x45513csYtWbIkTj/99Dj44IOjYcOG0bp16xg4cGB88MEHOz3nJUuWxMKFC+OCCy7Ie+zDDz+Myy+/PA499NDYd999o3Xr1nHOOefERx99lDNuy5Yt8dOf/jRat24d3/jGN+Lkk0+O5cuX79a1Kb0GS5cujR//+Mex//77R8uWLeMnP/lJbNiwIWds6ffrkUceiaOOOiqaNGkSxxxzTMyaNSvvXP7v//4vzjvvvLLrc9RRR8V9992302sTEfHJJ5+UXYOGDRvGQQcdFL169YoXX3xxp9uVPq+rwr/8y79Eq1atYuTIkTsc07JlyzjllFPi/vvvr5JjAgBAbVW/vAO3bdsW8+bNi+7du0erVq3KfYB58+ZF//7943vf+17cf//9sf/++8f06dNjyJAh8fnnn8dFF12UM/7SSy+NgQMHxqOPPhrvv/9+XHfddXH++efH7373u7Ixt912W9x4441x8cUXx4033hhffvll3HnnnfGDH/wgFi5cGEcffXTZ2C+//DLOOOOMGDp0aIwePTq2bt0aERFvv/12HHfccXHppZfG/vvvHytWrIh//dd/jeOPPz5ef/31aNCgQVx66aWxdu3amDx5cjz55JNl5126//LOY9q0aXHxxRfHmWeeGXfffXds2LAhxo8fH1988UXss8/u962bNWtWLFmyJO64444oKiqKG264IQYOHBgXXnhhvPPOO/Hv//7vsWHDhrj22mvj7LPPjv/93/8tC1blOf+IiDlz5sTgwYPjhBNOiBkzZsTWrVvjrrvuyguyERFDhw6NadOmxciRI+MXv/hFrF27NiZMmBA9e/aMV199NVq2bBmbNm2KU045Jdq3bx/33XdftGzZMlavXh3z5s2LjRs37vJ869WrFyeccEJO/cMPP4wePXrEli1bYuzYsdG1a9dYs2ZNPPfcc7Fu3bpo2bJl2dixY8dGr1694sEHH4y//vWvccMNN8QPf/jDePPNN6NevXoVujalzj777BgyZEhccskl8frrr8eYMWMiIuKhhx7KGffss8/GK6+8EhMmTIji4uKYOHFiDBo0KJYvXx6HH354REQsW7YsevbsGYcddljcfffdUVJSEs8991yMHDkyPv300xg3btwOr88FF1wQixcvjltvvTWOPPLIWL9+fSxevDjWrFmz0+taVV588cX49a9/Ha+88krZtdyRk046KcaMGRPr16+PAw44oFrmBwAA1S4rp9WrV2cRkZ177rl5j23dujXbsmVL2b/t27eXPdapU6fs29/+drZly5acbU4//fSsVatW2bZt27Isy7KHH344i4hs+PDhOeMmTpyYRUS2atWqLMuy7C9/+UtWv379bMSIETnjNm7cmJWUlGQ/+tGPymoXXnhhFhHZQw89tNNz2759e7Zly5bsvffeyyIie+qpp8oeu/POO7OIyN59992cbco7j23btmWtW7fOunXrlnNdVqxYkTVo0CBr27btTueWZVl24oknZt/61rdyahGRlZSUZJ999llZbebMmVlEZMcee2zOse69994sIrLXXnutwuffo0eP7NBDD82++OKLnHNs3rx59tWnzx//+McsIrK77747Z9/vv/9+1rhx4+z666/PsizLFi1alEVENnPmzF2e99cNGDAg69SpU179Jz/5SdagQYNs2bJlO9x23rx5WURkp512Wk798ccfzyIi++Mf/1hwu51dm3HjxmURkU2cODFnm+HDh2eNGjXK+R5ERNayZcvsr3/9a1lt9erV2T777JPdfvvtZbV+/fplbdq0yTZs2JCzz6uuuipr1KhRtnbt2izLsuzdd9/NIiJ7+OGHy8YUFxdn11xzzQ6vQXl88sknWURk48aNq9B2GzduzNq1a5eNGTOmrNa2bdts4MCBBce/8MILWURks2fPrsx0AQCgVquSPwv2ne98Jxo0aFD27+67746IiLfeeiv+9Kc/xT/+4z9GRMTWrVvL/p122mmxatWqvLfznnHGGTlfd+3aNSIi3nvvvYiIeO6552Lr1q3xT//0Tzn7a9SoUZx44ok5b/cudfbZZ+fVPv744xg2bFgceuihUb9+/WjQoEG0bds2IiLefPPNXZ5zeeexfPnyWLlyZZx33nk5b9tt27Zt9OzZc5fH2ZnevXtH06ZNy74+6qijIiJiwIABOccqrZdew4jynf+mTZti0aJFcdZZZ+V8hKC4uDh++MMf5sxl1qxZUVRUFOeff37O9SgpKYljjjmm7Hp06NAhmjVrFjfccEPcf//9sWzZsnKf78qVK+Pggw/Oq8+ePTt69+5ddp47s6vnV0TFnxuF9rl58+b4+OOPc+q9e/eO/fbbr+zrli1bxsEHH1x27M2bN8fcuXNj0KBB0aRJk7z7ZfPmzfHyyy/v8Ny++93vxrRp0+LnP/95vPzyy7Fly5ZdXo+K2L59e86ctm3bVvbY6NGjo0GDBvGzn/2sXPsq/T5++OGHVTpHAACoTcr9lvIWLVpE48aNc4JJqUcffTQ+//zzWLVqVU74KH3b8ahRo2LUqFEF9/v1z8U2b9485+uGDRtGRMTf/va3nH326NGj4P6+/hbtJk2a5HVL3r59e5x66qmxcuXKuOmmm6JLly7RtGnT2L59e3z/+98vO9bOlHcepW/nLSkpyRtTUlISK1as2OWxduTAAw/M+bo0FO+ovnnz5ogo//mvW7cusizLeUt2qa/XPvroox2OjYiyt0zvv//+8dJLL8Wtt94aY8eOjXXr1kWrVq3isssuixtvvDHv7dpf9be//a3g/j/55JNo06bNDrf7ql09v3bnubGrfe5oXOnY0nFr1qyJrVu3xuTJk2Py5MkF51/oM/alZsyYET//+c/jwQcfjJtuuimKi4tj0KBBMXHixILPv4qaMGFC3HzzzWVft23bNlasWBELFy6MKVOmxJNPPhmbN2/OeZ5t3bo11q9fH40bNy67LhERjRo1ioj8awQAAHVJuQN3vXr1ok+fPvH888/HqlWrcj7HXfpZ5a+HxxYtWkRExJgxY2Lw4MEF99uxY8cKTbh0n0888UTZquPOFGoG9cYbb8Srr74a06ZNiwsvvLCs/vXGYlUxj9KQtXr16rzHCtWqQ3nPv1mzZlFUVFTw89pfn3uLFi2iqKgo/vu//zsnWJX6aq1Lly4xffr0yLIsXnvttZg2bVpMmDAhGjduHKNHj97hvFu0aBFr167Nqx900EG7bLhWXlXx3NhdzZo1i3r16sUFF1wQV155ZcEx7du33+H2LVq0iHvvvTfuvffe+Mtf/hJPP/10jB49Oj7++OOYM2dOped3+eWX5/xd7dLv6bJlyyLLshg0aFDeNu+//340a9Ys7rnnnrjmmmvK6qXfx9L7CAAA6qJyB+6Ivwfn2bNnx7Bhw+KJJ57Y6WpkxN/D9BFHHBGvvvpq3HbbbZWaaKl+/fpF/fr14+233y74VvHyKA3hXw+GU6dOzRu7o9XK8s6jY8eO0apVq3jsscfi2muvLTv2e++9FwsWLMjr6F4dynv+TZs2je7du8fMmTPjrrvuKlsp/+yzz/K6a59++ulxxx13xIcffhg/+tGPyj2PY445Ju65556YNm1aLF68eKfjO3XqFDNnzsyrDxgwIB555JFYvnx5hf8Dp9CcIsr33KhqTZo0id69e8eSJUuia9euO/xLAOVx2GGHxVVXXRVz586N+fPnV8n8WrduXfD52r9//5g3b15e/dxzz4327dvH7bffHh06dMh5rLRz/VcbHAIAQF1TocDdq1evuO+++2LEiBHRrVu3uPzyy+Nb3/pW7LPPPrFq1ar47W9/GxGR8xbuqVOnxoABA6Jfv35x0UUXxSGHHBJr166NN998MxYvXhy/+c1vKjThdu3axYQJE+KnP/1pvPPOO9G/f/9o1qxZfPTRR7Fw4cJo2rRpztteC+nUqVN885vfjNGjR0eWZXHggQfGM888Ey+88ELe2C5dukRExKRJk+LCCy+MBg0aRMeOHcs9j3322SduueWWuPTSS2PQoEFx2WWXxfr162P8+PFV8jbf3VGR858wYUIMHDgw+vXrF1dffXVs27Yt7rzzziguLs5Zbe7Vq1dcfvnlcfHFF8eiRYvihBNOiKZNm8aqVaviD3/4Q3Tp0iWuuOKKmDVrVkyZMiXOOuusOPzwwyPLsnjyySdj/fr1ccopp+x03ieddFI89NBD8ec//zmOPPLInDnOnj07TjjhhBg7dmx06dIl1q9fH3PmzIlrr702OnXqlOTapDBp0qQ4/vjj4wc/+EFcccUV0a5du9i4cWO89dZb8cwzz+R06/+qDRs2RO/eveO8886LTp06xX777RevvPJKWZf5XZk9e3Zs2rSprFP8smXLyv5O9mmnnRZNmjTZ4bYlJSUFn8uNGjWK5s2bx0knnZT32MsvvxzNmzcvu78AAKAuqlDgjogYNmxYHHfccTFp0qS45557YuXKlVFUVBRt2rSJnj17xty5c6NPnz5l43v37h0LFy6MW2+9Na655ppYt25dNG/ePI4++uhyr4R+3ZgxY+Loo4+OSZMmxWOPPRZffPFFlJSURI8ePWLYsGG73L5BgwbxzDPPxNVXXx1Dhw6N+vXrx8knnxwvvvhiHHbYYTljS/980a9+9at44IEHYvv27TFv3ryyennmcckll0RExC9+8YsYPHhwtGvXLsaOHRsvvfRSwSZvqVXk/Pv37x+//e1v42c/+1kMGTIkSkpKYvjw4bFy5cp45JFHcsZOnTo1vv/978fUqVNjypQpsX379mjdunX06tUrvvvd70ZExBFHHBEHHHBATJw4MVauXBn77rtvdOzYMe8t3IWceeaZUVxcHE899VTZ30ePiDjkkENi4cKFMW7cuLjjjjtizZo1cdBBB8Xxxx+f93n2qrw2KRx99NGxePHiuOWWW+LGG2+Mjz/+OA444IA44ogj4rTTTtvhdo0aNYrvfe978cgjj8SKFStiy5Ytcdhhh8UNN9wQ119//S6Pe8UVV+T0Z/jNb35T9p9h7777brRr167S51Yqy7J4+umn8xoJAgBAXVOUZVlW05Ngz7Jly5Y49thj45BDDonnn3++Wo89YsSImDt3bixdulRY20PNnTs3Tj311Fi6dGmF3n0AAAB7GoGbXbrkkkvilFNOiVatWsXq1avj/vvvj5deeimef/75OPnkk6t1Lh999FEceeSR8R//8R9xzjnnVOuxqRq9e/eODh06xAMPPFDTUwEAgKQq/JZy9j4bN26MUaNGxSeffBINGjSIbt26xX/9139Ve9iO+PufI/vP//zPWLduXbUfm8pbt25dnHjiiTF8+PCangoAACRnhRsAAAAS2KemJwAAAAB1kcANAAAACQjcAAAAkIDADQAAAAnUiS7l3W75XU1PoU7q2blVXm3BG6tqYCZUlcU39anpKeyQ+xjKpzbfxxHuZSgv9zLUDbu6l61wAwAAQAICNwAAACQgcAMAAEACdeIz3KTh89oAAAC7zwo3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAL1a3oC1A49O7fKqy14Y1UNzAQAAKBusMINAAAACQjcAAAAkIDADQAAAAkI3AAAAJCApml7oUWdn8yrdX9jcLm2HdG/Q15t8py3KjWf2tSwrTbNBQB2JsVrMrDn8bOgdrPCDQAAAAkI3AAAAJCAwA0AAAAJCNwAAACQgKZpe6HyNkgrZPOiZwtUO+7+ZKJwU7IpQ+vl1YZP3Vap4+zuXACgpj37+ll5tYExs9rnAdQ+GqTVbla4AQAAIAGBGwAAABIQuAEAACABgRsAAAASELgBAAAgAV3KqZAHPq1cR/Ly+vP6/ytQPTz5cXt2bpVX07kcgJrW8vvH5Bc3Vf88gMpZdMaHebXuTx9SAzOhuljhBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABIQNM0dmj6kDblGvfBBx/k1UbNL/9xCjUqu3dG+bevShqkAVAbdd90c01PAagCGqTtfaxwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkoGkaO3TujPxmaCloVAYAtcOzr5+VVxvYZWa1zwP2doWaF1f2d/Nu7z2aV1vc9rxK7ZNds8INAAAACQjcAAAAkIDADQAAAAkI3AAAAJCApmlUWoqmDpVxV6/82qj51T8PoGq5tyE9DdKgdkjxu/R55+U3SFvsdTQ5K9wAAACQgMANAAAACQjcAAAAkIDADQAAAAlomkal1WSDtEI0UYK6yb0NALvP62jNsMINAAAACQjcAAAAkIDADQAAAAkI3AAAAJCApmkkMX1Im4L12tZgDQAAIBUr3AAAAJCAwA0AAAAJCNwAAACQgMANAAAACWiaRhKaowFQW/Xs3CqvtuCNVTUwEwDqOivcAAAAkIDADQAAAAkI3AAAAJCAwA0AAAAJaJpGjZtxbte82pDpr9XATADYG2iQBkB1scINAAAACQjcAAAAkIDADQAAAAkI3AAAAJCApml7oelD2uTVPvjgg7zaqPnVMRsN0oC6o2fnVnk1Dbqgdih0f0a4R6kad/XKr1XX79J1WV14XbXCDQAAAAkI3AAAAJCAwA0AAAAJCNwAAACQgKZpe6FzZ+Q3SCuvQg3XKrM/gLpkT2vkAnsT9ycpaZCWRl24b61wAwAAQAICNwAAACQgcAMAAEACAjcAAAAkoGkaFaJBGgDUXT07t8qr1YWmRew9PIepbaxwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkoGkaewUNNABg17w2sqfzHKa2scINAAAACQjcAAAAkIDADQAAAAkI3AAAAJCApmnUuEVNx+XVum+6uUqPUZkGGvf9uFte7crHFldmOuwBNNqDusG9DEBNssINAAAACQjcAAAAkIDADQAAAAkI3AAAAJCApmk1SCOXv6vqBmnlVagZWruxh+XVBsbMapgNtU2he7G89+yI/h3yapPnvFU1EwMqZG98XQWg9rDCDQAAAAkI3AAAAJCAwA0AAAAJCNwAAACQgMANAAAACehSXoPqcufU6UPaFKyfO+ODap7Jjl352OL8YpeZ1T4P9hzlvWf/3webEs+keuwJ9zGQ69nXz8qrDazjr23+6kvdUuj7GeF7GuF1eU9lhRsAAAASELgBAAAgAYEbAAAAEhC4AQAAIAFN06hxhRpA1KbmD5qxUFF15flRm+7D6uJ+Z09X1xukFeIerVt8P3fswX/7U+EHWhZX70SoECvcAAAAkIDADQAAAAkI3AAAAJCAwA0AAAAJaJpGhUwZWi+vduD6/CZDdanZkuYd7K1O/uizgvUX63BzFvc7e7rKNiKt7Y1MYW8xoqRpXm1yRbbv3yF/+zlvVWJGtUehBqcRtfc13Ao3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACmqZRIcOnbitQrVwzlYkTJ+YX255XqX2mVqhZQ21t1ABfV96mSJeO7FRw+xc1UIJqd1ev/Nqo+fm1yjY40yANaofJqzdVbvs60iCtkD3td24r3AAAAJCAwA0AAAAJCNwAAACQgMANAAAACWiaRrUq3KypdjdIK2RPa9YAX9Vh6b8VqA7Oq2ieBLVHoQZplXVZi+V5tb59++bV/CwA2H1WuAEAACABgRsAAAASELgBAAAgAYEbAAAAEtA0bQ/Qs3OrvNqe2rRL4xWoed3fyG+QBux9Hvi0Y15t6KxLCoy8Of1kyqm8vxPVpd+dgD2bFW4AAABIQOAGAACABARuAAAASEDgBgAAgAQ0TatlytvkY8a5XfNqQ6a/llebPqRNXu2DD/Ibl42aX94Z7n00XgEK8bOBiirvc6bbe4/m1Ra3PS/JnL6u+6ba0yCtkPLeY+5FduWyFsvzaoUaCUJlWeEGAACABARuAAAASEDgBgAAgAQEbgAAAEhA07RqUt5GKYVqd/XK31+hBmmFnDsjv0Eaf7eo6bi82sj2v8yrabwCFOJnAxVV3udMdTVIq+00JiSl8jZI69vnubza3N/1q+rp1Gl7+71shRsAAAASELgBAAAgAYEbAAAAEhC4AQAAIAFN06pJeRsDFGoqMGp+1TYV2NsbF5Qa/c51ebUFm/a+6wAANW3K0Hp5teFTvSZT8+p6g7QR/Tvk1SbPeWu39/fs62fl1QbGzN3eX11ghRsAAAASELgBAAAgAYEbAAAAEhC4AQAAIAFN02qZ6mhetjc2SLvvx93yalc+trgGZgIAfN3wqdtqegqwV6pMg7RCBnaZWaX7qwuscAMAAEACAjcAAAAkIHADAABAAgI3AAAAJKBpGnsFDdIAYNd6dm6VV9sbm61CXVQdTYRH9O+QV6vqxmx7GivcAAAAkIDADQAAAAkI3AAAAJCAwA0AAAAJaJq2F9IQBQCq357w+lvb5gN7ij3h/q6OJsKVaZC2J1zD3WGFGwAAABIQuAEAACABgRsAAAASELgBAAAgAU3T9kJ1ofkAAOxpvP5C3eX+rry6eg2tcAMAAEACAjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkIHADAABAAgI3AAAAJCBwAwAAQAICNwAAACQgcAMAAEACAjcAAAAkUL+mJ0D169m5VV5twRuramAmAAAAdZcVbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABDRN2wtpkAYAAJCeFW4AAABIQOAGAACABARuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABARuAAAASKAoy7KspicBAAAAdY0VbgAAAEhA4AYAAIAEBG4AAABIQOAGAACABARuAAAASEDgBgAAgAQEbgAAAEhA4AYAAIAEBG4AAABI4P8H0KvkQG0Cv+4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1200x600 with 8 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#load a model\n",
    "#decoder = tf.keras.models.load_model(\"6731_30March_test_decoder\")\n",
    "\n",
    "z_gen = np.random.normal(size=(2,embsize))\n",
    "\n",
    "recons_embed = generator.predict(z_gen)\n",
    "print(recons_embed.shape)\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from scipy import ndimage\n",
    "\n",
    "cmap=\"tab20c\"\n",
    "\n",
    "def new_images(image,channel):\n",
    "    recons = image[:,:,channel] #last number cycles through channels 0-3\n",
    "    rot = ndimage.rotate(recons, 0)\n",
    "    rot  = rot[20:60, 20:60]\n",
    "    return rot\n",
    "\n",
    "newimages=[]\n",
    "newimages2=[]\n",
    "for i in range(4):\n",
    "    new1 = new_images(image=recons_embed[0],channel=i)\n",
    "    newimages.append(new1)\n",
    "    new1 = new_images(image=recons_embed[1],channel=i)\n",
    "    newimages2.append(new1)\n",
    "    \n",
    "#can insert scaling code here\n",
    "\n",
    "fig, arrax = plt.subplots(2, 4, figsize=(12,6))\n",
    "for i in range(4):\n",
    "    arrax[0,i].imshow(newimages[i],cmap=cmap)\n",
    "    arrax[1,i].imshow(newimages2[i],cmap=cmap)\n",
    "    arrax[0,i].axis(\"off\")\n",
    "    arrax[1,i].axis(\"off\")\n",
    "arrax[0,0].set_title(\"Generated Images (channels 1-4)\")\n",
    "arrax[1,0].set_title(\"Generated Images (channels 1-4)\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2a718f7-d9a1-4df2-bda2-b8c810428f2e",
   "metadata": {},
   "source": [
    "## Save GAN components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7b7d6ca5-7484-4f78-a6c0-0f3ebd834859",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: 6731_1April_test_discriminator\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: 6731_1April_test_discriminator\\assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Discriminator model saved.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: 6731_1April_test_generator\\assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: 6731_1April_test_generator\\assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generator model saved.\n"
     ]
    }
   ],
   "source": [
    "discriminator.save(\"6731_1April_test_discriminator\",save_format=\"tf\")\n",
    "print(\"Discriminator model saved.\")\n",
    "\n",
    "generator.save(\"6731_1April_test_generator\",save_format=\"tf\")\n",
    "print(\"Generator model saved.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96afa96f-6948-4c9a-add7-225ff943c148",
   "metadata": {},
   "source": [
    "## Load and restart training on the GAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8c5f08b6-39cd-4671-a73f-fb30ac584c73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 51/70\n",
      "211/211 [==============================] - 99s 443ms/step - d_loss: 0.7496 - g_loss: 0.6111\n",
      "Epoch 52/70\n",
      "211/211 [==============================] - 102s 486ms/step - d_loss: 0.7420 - g_loss: 0.6152\n",
      "Epoch 53/70\n",
      "211/211 [==============================] - 105s 497ms/step - d_loss: 0.7731 - g_loss: 0.5896\n",
      "Epoch 54/70\n",
      "211/211 [==============================] - 105s 496ms/step - d_loss: 0.7361 - g_loss: 0.6198\n",
      "Epoch 55/70\n",
      "211/211 [==============================] - 107s 506ms/step - d_loss: 0.7069 - g_loss: 0.6440\n",
      "Epoch 56/70\n",
      "211/211 [==============================] - 110s 523ms/step - d_loss: 0.7528 - g_loss: 0.6073\n",
      "Epoch 57/70\n",
      "211/211 [==============================] - 109s 518ms/step - d_loss: 0.7331 - g_loss: 0.6220\n",
      "Epoch 58/70\n",
      "211/211 [==============================] - 108s 514ms/step - d_loss: 0.7425 - g_loss: 0.6158\n",
      "Epoch 59/70\n",
      "211/211 [==============================] - 109s 518ms/step - d_loss: 0.7489 - g_loss: 0.6038\n",
      "Epoch 60/70\n",
      "211/211 [==============================] - 109s 517ms/step - d_loss: 0.7244 - g_loss: 0.6246\n",
      "Epoch 61/70\n",
      "211/211 [==============================] - 110s 522ms/step - d_loss: 0.7324 - g_loss: 0.6045\n",
      "Epoch 62/70\n",
      "211/211 [==============================] - 112s 529ms/step - d_loss: 0.7451 - g_loss: 0.6080\n",
      "Epoch 63/70\n",
      "211/211 [==============================] - 110s 521ms/step - d_loss: 0.7291 - g_loss: 0.6175\n",
      "Epoch 64/70\n",
      "211/211 [==============================] - 110s 520ms/step - d_loss: 0.7454 - g_loss: 0.6061\n",
      "Epoch 65/70\n",
      "211/211 [==============================] - 114s 538ms/step - d_loss: 0.7429 - g_loss: 0.6080\n",
      "Epoch 66/70\n",
      "211/211 [==============================] - 117s 556ms/step - d_loss: 0.7357 - g_loss: 0.6070\n",
      "Epoch 67/70\n",
      "211/211 [==============================] - 118s 558ms/step - d_loss: 0.7432 - g_loss: 0.6074\n",
      "Epoch 68/70\n",
      "211/211 [==============================] - 114s 542ms/step - d_loss: 0.7394 - g_loss: 0.6152\n",
      "Epoch 69/70\n",
      "211/211 [==============================] - 113s 536ms/step - d_loss: 0.7143 - g_loss: 0.6256\n",
      "Epoch 70/70\n",
      "211/211 [==============================] - 120s 567ms/step - d_loss: 0.7114 - g_loss: 0.6183\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x2390e835990>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#autoencoder = tf.keras.models.load_model(\"6731_30March_test\")\n",
    "epcs = 50\n",
    "\n",
    "dcgan.compile(d_optimizer = tf.keras.optimizers.SGD(learning_rate = 0.0001),\n",
    "             g_optimizer = tf.keras.optimizers.Adam(learning_rate = 0.001, beta_1 = 0.5, beta_2 = 0.999))\n",
    "\n",
    "dcgan.fit(dataset.X, epochs = epcs+20, initial_epoch = epcs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5e80d75-0a3a-4ddf-adc3-f8f1e94088c6",
   "metadata": {},
   "source": [
    "## Code for scaling images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68654c36-66de-49fe-a5dc-b54b51b01c13",
   "metadata": {},
   "outputs": [],
   "source": [
    "#scaling for images\n",
    "biggest, smallest = 0.0, 6000.0\n",
    "tmp=np.empty(2)\n",
    "for i in range(4):\n",
    "    tmp[0] = newimages[i].max()\n",
    "    tmp[1] = oldimages[i].max()\n",
    "    tmps = tmp.max()\n",
    "    if tmps > biggest:\n",
    "        biggest = tmps\n",
    "    tmp[0] = newimages[i].min()\n",
    "    tmp[1] = oldimages[i].min()\n",
    "    tmps = tmp.min()\n",
    "    if tmps < smallest:\n",
    "        smallest = tmps\n",
    "scaled_newimages = ((newimages - smallest) * (1/(biggest-smallest)) * 255).astype('uint8')\n",
    "scaled_oldimages = ((oldimages - smallest) * (1/(biggest-smallest)) * 255).astype('uint8')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}