CS6910_Assignment1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": " CS6910_Assignment1",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2174ec287cf94766bdb9a31f09518eae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "VBoxView",
            "_dom_classes": [],
            "_model_name": "VBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6940ffb31b3b4b9f8f62238e7d15df72",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7237b35791904be7b4c395147ab0756f",
              "IPY_MODEL_d0ba154ada2140099d67ae877344092a"
            ]
          }
        },
        "6940ffb31b3b4b9f8f62238e7d15df72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7237b35791904be7b4c395147ab0756f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "LabelView",
            "style": "IPY_MODEL_c70e5b0abd514e9b966dec3f68e13781",
            "_dom_classes": [],
            "description": "",
            "_model_name": "LabelModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0.01MB of 0.01MB uploaded (0.00MB deduped)\r",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aaba43fc77644fec9fc9e6a0900c91dc"
          }
        },
        "d0ba154ada2140099d67ae877344092a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a63045e0e64643299d8dcb23deed14e8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f547356504af48eb887c8e3ca132a183"
          }
        },
        "c70e5b0abd514e9b966dec3f68e13781": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aaba43fc77644fec9fc9e6a0900c91dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a63045e0e64643299d8dcb23deed14e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f547356504af48eb887c8e3ca132a183": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0aa640b5c9e843b6b86370443e7de989": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "VBoxView",
            "_dom_classes": [],
            "_model_name": "VBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_510566ce597a42539c099fc4e2d80644",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7702c6812b6b4d9285fd00d4104430be",
              "IPY_MODEL_d96c0570414545c3b812a0d90301edbc"
            ]
          }
        },
        "510566ce597a42539c099fc4e2d80644": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7702c6812b6b4d9285fd00d4104430be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "LabelView",
            "style": "IPY_MODEL_84c5bac8fe254dcb95d5526b50c496a1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "LabelModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0.01MB of 0.01MB uploaded (0.00MB deduped)\r",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2c84c4f31d184131ae17610f9d75e2e9"
          }
        },
        "d96c0570414545c3b812a0d90301edbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6c2458ee2e334fefb79d0d7ede73c0f5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c6a586dcfa984e658e48dd7811d9a4ad"
          }
        },
        "84c5bac8fe254dcb95d5526b50c496a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2c84c4f31d184131ae17610f9d75e2e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c2458ee2e334fefb79d0d7ede73c0f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c6a586dcfa984e658e48dd7811d9a4ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0804e5724c844c199b71f01b926033f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "VBoxView",
            "_dom_classes": [],
            "_model_name": "VBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_112fa9071e7f4643839af5675dde4b29",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7afce843532642d0bdc48b72d5286e15",
              "IPY_MODEL_5e22b33609344bd6ad42cbf6895d5f63"
            ]
          }
        },
        "112fa9071e7f4643839af5675dde4b29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7afce843532642d0bdc48b72d5286e15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "LabelView",
            "style": "IPY_MODEL_e37519bd05ea426391da7710a6f4e8fd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "LabelModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0.01MB of 0.01MB uploaded (0.00MB deduped)\r",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dbe32aecff434ec8905e72c57888aa75"
          }
        },
        "5e22b33609344bd6ad42cbf6895d5f63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e4767a5976e04fde957e72e41419b62b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_86a1b56c04c14c72b642fdbc8b0e54c9"
          }
        },
        "e37519bd05ea426391da7710a6f4e8fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dbe32aecff434ec8905e72c57888aa75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e4767a5976e04fde957e72e41419b62b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "86a1b56c04c14c72b642fdbc8b0e54c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/safikhanSoofiyani/CS6910-Assignment1/blob/main/CS6910_Assignment1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1> Feed Forward Neural Network</h1>\n",
        "\n",
        "Work Done by:<br>\n",
        "<ul> \n",
        "<li>Mohammed Safi Ur Rahman Khan - CS21M035 </li>\n",
        "<li>Vamsi Sai Krishna Malineni  - OE20S302 </li>"
      ],
      "metadata": {
        "id": "R5q7bg9s5L8S"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The goal of this work is two fold: <br>\n",
        "(i) implement and use gradient descent (and its variants) with backpropagation for a classification task <br>\n",
        "(ii) Getting familiar with wandb which is a cool tool for running and keeping track of a large number of experiments"
      ],
      "metadata": {
        "id": "nGbZ6RDKJuHV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this project we have implemented a feedforward neural network and written the backpropagation code for training the network. We have used numpy for all matrix/vector operations. We have not used any automatic differentiation packages.<br> <br>\n",
        "This network is trained and tested using the Fashion-MNIST dataset. Specifically, given an input image (28 x 28 = 784 pixels) from the Fashion-MNIST dataset, the network is trained to classify the image into 1 of 10 classes."
      ],
      "metadata": {
        "id": "Y3rdmr2YJ-Eb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The numbering of various sections of this document are done in accordance with the instructions and question numbers from the below document:<br> \n",
        "https://wandb.ai/miteshk/assignments/reports/Assignment-1--VmlldzozNjk4NDE?accessToken=r7ndsh8lf4wlxyjln7phvvfb8ftvc0n4lyn4tiowdg06hhzpzfzki4jrm28wqh44"
      ],
      "metadata": {
        "id": "B5NOodH1L9zO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Declaration\n",
        "\n",
        "For the purpose of this project, we have used help from various online resources for help in understanding some concepts, intuition about the code flow, Debugging the various errors, etc.<br> <br>\n",
        "Wherever possible we have credited all the resources in their respective sections. <br> <br>\n",
        "Some of the frequently used resources include:<br>\n",
        "<ul> <li> StackOverflow (and its variants)</li>\n",
        "<li> Towards Data Science Blogs</li>\n",
        "<li> machinelearningmastery.com </li>\n",
        "<li> Medium Blogs </li>\n",
        "<li> Kaggle </li>\n",
        "<li> Lecture slides from Prof. Mitesh Khapra's  CS6910-Fundamentals of Deep Learning course.</li>\n",
        "</ul>\n",
        "<br> <br>\n",
        "\n",
        "We have tried to include as many resources that we have used, but in all possibiliy we may have missed many.<br>\n",
        "We profusely apologize to any other authors for using your resouces and content but failing to appropriately credit you.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "avy-vAbnq-Ag"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3>Importing necessary libraries.</h3>"
      ],
      "metadata": {
        "id": "s_fr1Bcc6eOG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ahmgbKIg2f0o"
      },
      "outputs": [],
      "source": [
        "import keras\n",
        "from keras.datasets import fashion_mnist\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import copy \n",
        "from tqdm import tqdm\n",
        "import random\n",
        "import tensorflow as tf\n",
        "import seaborn as sn"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3>Installing and importing wandb</h3>"
      ],
      "metadata": {
        "id": "OG-qZnHL7Gyb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Wandb is used to keep track of various experiments performed and for efficient logging while doing hyperparameter tuning. <br>\n",
        "The report for this project is also created using wandb"
      ],
      "metadata": {
        "id": "MQuSOYQGR0bD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install wandb -qqq\n",
        "import wandb"
      ],
      "metadata": {
        "id": "VVG9Nfzc7DLP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h2>1. About the Dataset</h2>"
      ],
      "metadata": {
        "id": "2qdoUbpBQ-_A"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Fashion-MNIST is a dataset of Zalando's article images—consisting of a training set of 60,000 examples and a test set of 10,000 examples. Each example is a 28x28 grayscale image, associated with a label from 10 classes.<br>\n",
        "<br>\n",
        "Each image is 28 pixels in height and 28 pixels in width, for a total of 784 pixels in total. Each pixel has a single pixel-value associated with it, indicating the lightness or darkness of that pixel, with higher numbers meaning darker. This pixel-value is an integer between 0 and 255. <br><br>\n",
        "\n",
        "<b>Labels:</b>\n",
        "\n",
        "Each training and test example is assigned to one of the following labels:\n",
        "<ul>\n",
        "<li> T-shirt/top\n",
        "<li> Trouser\n",
        "<li> Pullover\n",
        "<li> Dress\n",
        "<li> Coat\n",
        "<li> Sandal\n",
        "<li> Shirt\n",
        "<li> Sneaker\n",
        "<li> Bag\n",
        "<li> Ankle boot\n",
        "</ul>"
      ],
      "metadata": {
        "id": "UkQpSIXORJo2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Source: https://www.kaggle.com/zalando-research/fashionmnist"
      ],
      "metadata": {
        "id": "ouh4YpcaT9Nn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3>Preparing dataset</h3>"
      ],
      "metadata": {
        "id": "GU1iINihIbYJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "#sklearn library is used only for train test validation split\n",
        "\n",
        "\n",
        "def prepare_data():\n",
        "\n",
        "  '''This function is used to load the data, define the class labels, performing\n",
        "      the train-test-validation split, normalizing the data, flattening each data\n",
        "      point, converting the class labels to one hot encoded vector.\n",
        "\n",
        "      It return all the split data sets '''\n",
        "\n",
        "\n",
        "  # Loading data from online source\n",
        "  (train_x,train_y),(test_x,test_y)=fashion_mnist.load_data()\n",
        "\n",
        "  # Defining labels for data\n",
        "  num_classes = 10\n",
        "  labels=['T-shirt/top','Trouser','Pullover','Dress','Coat','Sandal','Shirt','Sneaker','Bag','Ankle boot']\n",
        "\n",
        "  print(\"Number of data points in train data (initially) - \", len(train_x))\n",
        "  print(\"Number of data points in test data (initially) - \", len(test_x))\n",
        "\n",
        "\n",
        "  #performing the train-validation split\n",
        "  train_x, val_x, train_y, val_y = train_test_split(train_x, train_y, test_size=0.1, random_state=40)\n",
        "  \n",
        "\n",
        "  print(\"Shape of each image - 28x28\" )\n",
        "  image_shape=train_x.shape[1]*train_x.shape[2]\n",
        "  print(\"shape of each image (1D) - \",image_shape)\n",
        "  \n",
        "  #storing the number of points in each set\n",
        "  train_image_count=len(train_x)\n",
        "  val_image_count = len(val_x)\n",
        "  test_image_count=len(test_x)\n",
        "  \n",
        "  # Creating a matrix of image data \n",
        "  # each image is represented as a row by flattening the matrix: converting (60000,28,28) tensor to (60000,784) matrix\n",
        "  X_train=np.zeros((train_image_count,image_shape))\n",
        "  X_val=np.zeros((val_image_count,image_shape))\n",
        "  X_test=np.zeros((test_image_count,image_shape))\n",
        "  \n",
        "  # converting the images into grayscale by normalizing\n",
        "  for i in range(train_image_count):\n",
        "    X_train[i]=(copy.deepcopy(train_x[i].flatten()))/255.0 \n",
        "  for i in range(val_image_count):\n",
        "    X_val[i]=(copy.deepcopy(val_x[i].flatten()))/255.0\n",
        "  for i in range(test_image_count):\n",
        "    X_test[i]=(copy.deepcopy(test_x[i].flatten()))/255.0\n",
        "  \n",
        "\n",
        "\n",
        "  #One hot encoding the label vectors to represent a probability distribution\n",
        "  y_train = np.zeros((train_y.size, 10))\n",
        "  y_train[np.arange(train_y.size), train_y] = 1\n",
        "\n",
        "  y_val = np.zeros((val_y.size, 10))\n",
        "  y_val[np.arange(val_y.size), val_y] = 1\n",
        "\n",
        "  y_test = np.zeros((test_y.size, 10))\n",
        "  y_test[np.arange(test_y.size), test_y] = 1\n",
        "\n",
        "  \n",
        "\n",
        "  #returning all the datasets along with the labels\n",
        "  return X_train,X_val,X_test,y_train,y_val,y_test,labels\n",
        "  "
      ],
      "metadata": {
        "id": "uIanRhNjIZiu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3>Plotting images locally</h3>"
      ],
      "metadata": {
        "id": "kM6jDLswWRYP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_locally():\n",
        "\n",
        "  '''This function is used to plot a single image of each class label on the \n",
        "  local machine'''\n",
        "\n",
        "  #getting all the datasets\n",
        "  xtrain, xval, xtest, ytrain, yval, ytest, labels = prepare_data()\n",
        "\n",
        "  # Creating training dataset\n",
        "  train=np.asarray(list(zip(xtrain,ytrain)))\n",
        "\n",
        "  # plotting a single image from each class\n",
        "  sample_images=[]\n",
        "  wandb_arr=[]\n",
        "  i=1\n",
        "  plt.suptitle(\"Plotting image of each class from Fashion MNIST Dataset\")\n",
        "\n",
        "  while(len(sample_images)!=10):\n",
        "    n=random.randrange(0,len(train))\n",
        "    lab_index=np.asarray(np.nonzero(train[n][1]))[0][0]\n",
        "    \n",
        "    if(lab_index not in sample_images):\n",
        "      plt.subplot(3,5,i)\n",
        "      sample_images.append(lab_index)\n",
        "      plt.title(labels[lab_index])\n",
        "      plt.axis(False)\n",
        "      plt.imshow(train[n][0].reshape((28,28)))\n",
        "      i=i+1"
      ],
      "metadata": {
        "id": "VmT6fEAtWQzV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plot_locally()"
      ],
      "metadata": {
        "id": "wTFiRD0Zcu2Y",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 512
        },
        "outputId": "30051634-3a87-49ea-c5e8-caa966b27252"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n",
            "32768/29515 [=================================] - 0s 0us/step\n",
            "40960/29515 [=========================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n",
            "26427392/26421880 [==============================] - 0s 0us/step\n",
            "26435584/26421880 [==============================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n",
            "16384/5148 [===============================================================================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n",
            "4423680/4422102 [==============================] - 0s 0us/step\n",
            "4431872/4422102 [==============================] - 0s 0us/step\n",
            "Number of data points in train data (initially) -  60000\n",
            "Number of data points in test data (initially) -  10000\n",
            "Shape of each image - 28x28\n",
            "shape of each image (1D) -  784\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
            "  after removing the cwd from sys.path.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWYAAAC0CAYAAACnp4aGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9eZxlyVXf+T0Rd3l7brV3dVWrVd2t1o4EEpssBmRAAgYGfzDGYJDBDIIxg4edYRkMGIMHD3jGBrzACAGSkDHIwLCIrcGSQAJLINGtpbul6q6u7uqqyqrMfPm2e2/EmT8iXubLrMysrCWzs8X7fT7v8967a9xzI06cOHHOL0RVmWKKKaaY4uDAPNMFmGKKKaaYYiOminmKKaaY4oBhqpinmGKKKQ4Ypop5iimmmOKAYaqYp5hiiikOGKaKeYopppjigOGWFLOIPCAi/+R2FUZEflZEvv92XW/iuqdEZFVE7O2+9n5CRH5ERC6LyIV9vOdZEXnNQbnODdxv32W1VxARFZEz2+z7ShF5x36XaYq9xXUVc2xQg6jYnhaRN4pI60ZuIiJ3xcqVTGx7vYi8c/I4VX2Dqv7wjVx7N1DVx1W1parudl97vyAip4BvA56vqsee6fIcZDzTshKRHxSRMraZ8ec79+JeqvrLqvq5t/u6IvJZsc3++qbtL4nbH5jYpiLyQRExE9t+RETeGH9vaP8iclJE/kvsOJdF5G+iPnjVhLx68ZxJGZ7aopxj/dQVkSURebeIvGGyLNd5zmt0017gRu+zW4v5i1S1BbwM+GTg+262gFPcNE4Bi6p68ZkuyLMAO8pqrxthxK9EY2D8+Vf7cM/bjUvAp4nIwsS2rwE+usWxJ4B/sMvr/iJwDjgNLAD/CHhaVf/bWF7AC+KxsxMyfHyb632Rqrbj9X4M+C7g53ZZlgOJG3JlqOp54HeAF27eJyJGRL5PRB4TkYsi8iYRmYm7/zR+L8We79OAnyW89FURWYrXeKOI/Ej8/Vki8oSIfFu83lMi8o8n7rcgIr8pIisi8hexh95ggU8cu7nHfiAe/+54/9+M1/vlievdNXH+vxGRc3HffxeRV03sq4vIL4jIVRH5kIh8p4g8MbH/RLQOLonIx0Xkf91OviIyE+V2Kcrx+6JcXwP8PnAilveN25z/hSLyVxOWw4sn9n23iDwaLYuHROR/2nTu18fyj/e/bGL3S0XkA9G6+RURqe3wDDtdZ3zMK0Tkz2I5nxKRfysiWdwnIvKT8Z2vSLDEXhj3vS5esysi50Xk27e49jWymnj/XycijwN/tFN9nTj+H8f3flWCFfYpUQ5LIvJvt5PBDrLZ9h2IyBkR+ZMo48si8iubTn+NiDwc7/3vRETieRtGniLy6bH+LsfvT5/Y94CI/LCIvCuW4R0icmiHIhfA24kKV4Ir8MuBX97i2H8F/HPZXaf3KcAbVbWnqpWqvl9Vf2cX5+0IVV1W1d+IZfyaiXrzBSLy/lifzonID06cdo1uEpHnisgfichifBe/LCKz4xNE5Lti/euKyEdE5HPidjPxjhdF5G0iMr/dfa73MDt+gLPAa+LvO4EHgR+O/x8A/kn8/bXAI8DdQAv4NeAX4767AAWSieu+Hnjnpnu9EfiR+PuzgAr4ISAFXgf0gbm4/63x0wCeT+iB37nNM2y4fyz3I8BzgRngIYIV8BogAd4E/L8T538VoWdPCEPkC0At7vsx4E+AOeAk8AHgibjPAP8d+AEgi7L5GPB525TzTcB/BdqxzB8Fvm5CHk/s8J4+CbgIvBKwBMvmLJDH/V9GsGoMoeL2gOMT+84TGowAZ4DTE+//vfHceeBDwBu2KcP1rjOuRy8HPjXK8654zX8W931elNlsvMb9E+V8CnhV/D0HvGybcmyQ1cT7fxPQBOrsrr7+LFADPhcYEpTUEeCOKOtXb3P/HwR+aRv5bPcO3gJ8b9xXAz5z4jwFfivK5BTBkv38ze0ovp+rBAs0Ab4i/l+YqPePAvdGGTwA/NhOMgQ+HXhP3PY64PeAfwI8sKl898T3NtYHP0JQvpPyHLe/PwDeRVD4p3bTZnejnzZtfxz4xolneVGU7YuBp4Ev2UE3nQH+LpADhwlK9afivvsIuubExPnPjb+/Bfhzgh7IgX8PvOVGnmetDLtUzKvAEvAY8NNAfQvF/IfAN02cdx9Qst74bkYxDzadc5HQoG289n0T+35k8/Wuo5i/d2L/vwZ+Z+L/FwF/tYNMrgIvib83KFpCpR0r5lcCj28693uYUPoT2y3BQnn+xLZvIDYArq+Yf4bYYU5s+wjbK4+/Ar44/v494Ft2eP9fNfH/XwE/u82x17vONQ0o7vtnwK/H359N6JA+FTBbNLZvADrXqbMbZDXx/u+e2Lab+nrHxP5F4Msn/v8XYmeyxf1/ML7LpYnPieu8gzcB/wE4ucVxykZF/Tbguze3I4JCfu+mc/8MeP1Evf++iX3fBPzu9WQIPBzl81bgK9laMZ8hKO7HCEbITop5jmDQPAi4KIdP2anN7vCut6xXBAX5vduc81PAT+72PsCXAO+Pv88Q9NBrgHTTcR8CPmfi//Et6tSuFPNuXRlfoqqzqnpaVb9JVQdbHHMivpQxHosFOrrLe2yFRVWtJv73CdbN4XjtcxP7Jn/vBk9P/B5s8X9tglNEvj0Oz5cluF1mgPEQ8MQO5ThNGFIvjT/A/87WMjlEGBlsluEdu3ye08C3bbrXnbF8iMhXy7qbY4ngjho/w50ES2o7TEY2jN/BVrjedYhluVdEfktELojICvCj47Ko6h8B/xb4d8BFEfkPItKJp/49YuOPw/6dh4PXYvLd7Ka+7rqObIG3xTYz/jx5nXfwnYQRwntF5EER+dpN19vNO9j8TOPnmqxDu32Xk/hF4J8C/wPw69sdpKq/TbCyv2Gni6nqVVX9blV9AUHefwW8feyeuU24A7gCICKvFJE/luAiXAbewLrcr4GIHBWRt0Z3xQrwS6zXz0cIhsQPEurnW0XkRDz1NPDrE+/3Q4SO54Z14O2MY34yFmyMUwRXxNOEnmIzttq2W1yK1z45se3OW7jetpDgT/5O4O8T3CizwDKhEUEYXm9XjnPAxzc10Laqvm6LW10m9K6bZXh+l0U9B/yLTfdqqOpbROQ08B8JjWshPsPfTDzDOYJb51ax2+v8DPBh4B5V7RA6q7VGqar/t6q+nOCiuhf4jrj9L1T1iwnuhLcTLMcbwWSd26m+3nZc7x2o6gVV/XpVPUFQbD8t24TI7YDNzwQ3Voe2wy8SrOvfVtX+dY79XsL7bOzmwqp6GfgJ1l1ltwwR+RSCYh773t8M/AZwp6rOEFxU4/q2lR760bj9RbF+fhUb6+ebVfUzCbJW4MfjrnPAaze1wZqGubkb0ne3UzG/BfjfROQ5EsLpfpQwM10RFKkn+PPGeBo4KXHS50agIezt14AfFJGGiDwP+OpbfoKt0SY02EtAIiI/AHQm9r8N+B4RmROROwgNb4z3At04WVAXESsiL4wVZ6tnehvwL0SkHRvytxJ6693gPwJviNaBiEgzTnq0CX5Vjc+AhEnUyQnc/wR8u4i8PJ57Jt7/RrHb67SBFWA1vrtvHO+QMMH2ShFJCT7YIeBFJJMQszujqmU8399EGcfYqb7uBXZ8ByLyZSIy7uCvxmNv9Pl+G7hXRP6hiCQi8uWEzu23bqXgqvpx4NUEpXu9Yx8gdDhfs90xIvLjsR0ksX5+I/CIqi7eSjlFpCMiX0hwufySqn4w7moDV1R1KCKvAP7hxGlb6aY2wX27HNv0d0zc4z4R+WwRyQl1c8D6e/pZQvs9HY89LCJfvMN9tsXtVMw/T+hZ/xT4eCz0NwPEXvZfAO+KZv6nAn9E8DFdEJHLN3G/f0pwKVyI930LMLrVh9gCvwf8LsHv+RjhuSaHxD9EGL59nDCp8avjckRl+4XAS+P+ywTlNcPW+GaCMvoYobd/M0Gu14Wq/iXw9QQ3wFXCxNbr476HCH70PyN0iC8iTL6Mz/3PhPfzZqBLsEZv2Hq5get8O6FxdAkdymQEQiduu0qQ9yLwf8Z9/wg4G4eXbyD4O28W29bXvcD13gFhwvQ9IrJKsO6+RVU/doP3WCTUt28jyO07gS+MVumtlv+dqvrkLg//PnauPw2CS2SJUNdPA//jLRTvN0WkS2iX3wv8X8A/ntj/TcAPxWN+gImR1ja66Z8TQoOXgf+PYASOkRP845cJuucIYd4I4N8Q3t074r3+nDDPtN19toVEJ/WzHiLy48AxVd22p96ncnwj8A9U9dXPZDmmmGKKZy+etVwZIvI8EXlxHDK/Avg6dpiY2MNyHBeRz5AQw3gfwVrZ93JMMcUUnzjYjwyovUKb4L44QRga/mtCDPB+IyPEKz6HMDR7KyGkcIopppjipvAJ48qYYooppvhEwbPWlTHFFFNM8YmKqWKeYooppjhgOBCKWXbg6pVABfiR/S7TJxpkB07fTcftCw3i7YZsQSO7af/viMgzGrHzbMBObfHZis11Y7dt4ZnErRLlT3KlelnnbV4VkVuJMV2DBirA+65Tji0rk4h8hYi8+SArGxH5TAlMcMsickUC89c1CShTBNysvFT1tar6Cztcd0fF/kxgWjeuhdwGfvhnA25JMesE3yyBYOaLJrZtRQ14W7ELRfsFhGyoAwkJHBC/Bfw/hID8OwjB7XuRKPOsx17J64B22M/aurEP8nxW8cPfjDz2zZUhIockENcsxd7/v8nGVQa25PyVyMs8cZ2zElKcPwD0ROQtBD6A35SJlSLitf8uIWtvK87V3fDx/s8i8qQEzuBruH9vA+4FUNW3qKpT1YGqvkNVPyDX54Q9K4FcaUueZBH5jljuJ2UTIY7szE97kLGtvMYHiMhPSOBP/riIvHZi+wMSl0GL1vG7JPA+LxIyD6/hB3+GsVPdeL2IvHOHZ50RkZ+L7/+8BO5xG/ftWK8mISL3x2t/Rfy/E9/35na5552dTvDDy7UrJK29750g23Og5/E5J9PmD0dr/Uj8v3fy2A0F3W4+7EDrGPf/S0LlT+PnVayH651lG85frqVwPEtgo7qTdfrRa+5NoI38M92G2o/d8fG+hcBx8CJCrvu2z3eTMusQUmd/AXgtkWta1+kFt+SE3YXMPp8Q2/3CWP43x+c5MyHTXfPTHpTPdeT1egIJ1NcTKFS/kUDqM65jD7BOUft6Av/JNxNi+etsQUP7LH7WXyfE1jcJKcPvBb7hBurVawjW6OOElG64Pt/3WTa1yz2Sy1mu5Yf/xc11dov3/c6JfZNtYScO9J8nEIONz/tfiDSpey2PPRHYNvt/KArgzDbnbsn5y9aK+Wuvd2/gh4Hvj7/v2uLF7YaP93mbyvRze1DR7ifwUD9BUBa/ARzd4rg1TthdyOznmSBAJ1hfa5Vxi2vfED/tM/nZTl6x8T0ycVwjPsex+P8BNjbUzTzZr+cAKeabfda4fzSpDAiE+X+8zT22qlf/PN7zsya278j3vVW73COZnOVafvj7t2jfm9/3NYqZ63OgvwZ4dGLfu4Cv3g957IkrQ9ZXpV6VQMoCgYjmEQLBx8dE5Ls3nXYjPLG74V5+HTv7l3fDx3tu0/4T3Gao6odU9fWqepJg4Z4Afkp24ISdwHYy28wRvYGjV26Qn/YgYTt5xd0XJo4b01NuV49ulL9733GTz3qaMCJ9StZ5gf89wXLekWt4Am8A3q2BKW6M0+zA9x2xXzLdwA9PYHi7GVyPA/2PgUZsL3cRyMjGdAt7Ko89Ucy6vir1eGIQVe2q6rep6t0EJqlvlbhW1s3cYqf/InKMsHrA+7Y5HnbHx3vnpv27Zde6KajqhwkW0gu5DifsdfAU15Z9Ejvx0z5rsEleN3z6df4fKNzAs54jWMyHdJ0TuKOBlB52V6/eAJwSkZ/cdN0t+b4ni3lzT3fL6MXvSQ7o3ayOviMHuq5T8X5F/PyWqnbjcXsqj/2c/PtCCfy8QqDTc9wan+4knmYjz+lrCb6gsWC24kLdDR/v90vge34BgUZw8wKZtwQJREzfJpGHV0TuJFSAP2cHTthd4G3A60Xk+SLSAP6PTft34qc9sLiOvG4VN80Pvhe42WdV1aeAdwD/WgI/sYkTfmO2w93Uqy5hnuLviMiPxW078X0/o1DVSwRl+lUSOM+/ll0s2KC740B/M2F9xq+Mv8fYU3nsZ4LJPQS+4lUCJ+1Pq+of36Zr/0vg++KQ4tvZFCanW3Oh7oaP908I7pc/BH5CVd9xm8o7RpcwefAeEekRGt3fEBjqduKE3REaVhz+KQLn9SPxexLb8tMecOwkr1vFrfKD327cyrN+NYFc6yECr/WvEkaQsMt6papLhEnC14rID+sOfN8HBF9P6GQWgRcA797leTtyoKvqe+L+E4QIkPH2PZXHJxyJUQxLuUBYeHPlJq9xF0FZp7p3K1pMMcUUU2yJA5GSfZsxT4jGuCmlPMUUU0zxTOMTzmK+HZhazFNMMcUzialinmKKKaY4YPhEdGVMMcUUUzyrcVvy2f+u+bKbMrvtmecwOj3P8l0ZK/eAqym+VdH4WMbJP+hir6ziPvY4eBeOv+duHvnao1Qdj5pwS3FCsmo486ZF3EMfBWMxtZzu617E8l2W3imHOTSC83Vql4V8ScmXlM4jXfS/P7ixQBLDObcZRfy+/8+7jvW9WZk82/BMyMS025j5WUZ3H+bpl9dIhtC84Eh6nvr5LjIskW4PrRza7SK1nPIldzOaTVk5nVA2wdVDUU7/dg/5y4dQ57Z97zeKG5EJTOvKVrhtdeXFz+PSK+ZYPQWNl16hlRfM1/qc786w9OACdgSmkBBxbEAlfPtEqU4P6XQGfO6pD3MqX+Qn/vS1dD6ccPxPltH3P3i9W+8K28nkGWHVMu020qjj5luMZhOqpuATjyYKJijo4dE6aTMlyzOoHDIqKE7MUHU8tMu1iHh1QkVCcbRF3j0JiUXThFHb4OqgqSKATxWfCVVdkAqKuRr1O06gvR5uafmZEMMUNwjTaGA6bbTVwM21GM2muBqoheGcwdYN0MaUHjtsI06xgxKXWXrHc4qmUHSCUna5ggrDQzVap05CUaJliXZX8f3+dcsyxcGCJEn4tNtIq4EmFtKE3ukOg8NCMes4Xh/SzkYs5D0GVcrlmVlcIZgyKGY1hFQbAU2UZmvEbGPAjB1QkxLTKhnNW/qnm7RG94ILaRjSG6Crq+hwhB8Ob8vz7K1i3sYC7b/6fi69NGF0yMPCCDFgjMeIYoySvbiHe8WImXqfT5o9xxODOd597jm0G6v86Jk/YN6ucqGaodSEoaZcLtu89ejLGS4dh8QjRskbq2RphVXBe6E85ikOW4pYhsXVBNs9xaH3w8xb/iJY5eNyXsdynuKZQfFp93P+1RlVHVzbQV5Ra/XJ8oIj7S6ZcTSSgtxWHMpWScWRmzB3u1LVGPmUkU+o1NAtc4Yu5bEzczzRm4OLOdmK4Y4HhtgH3nedkkzxjGGbtmnvvIPy2CznP6uJfeVV7pq7ystnH6dth8wnq7TNkGPJEjWpmDUFhRqevqtFoZYSG64RE/UMHiueWTMglaB8vQo/+cpfYemTG1z+0g5dV+Nq1aBb1vjjv3wBR/5cmPloD977wdvymPtqMUueI1lG/4hlcLIinRkx2+lTVpbSWVRBVZitD3nl4bPcU3+aL209zMeqjFINnWTEPdnTNKTCiqfnc5ZcA58YTi9c4UqjQTsfkdsK5w0e4Uq/zqhMyTIHOKzxJNZRNBKK2YTB+RbzrSZ+NEJHB57q9m8lxtbQcCFldLIkqVV0GiMS68gSx+FGj5fPPs580uPObJGGjLgjWaEmjrYRnCoPVy26vsa5YoGur3Gx6DBwGZ10yNAlfLRxhN5SndFcSkNk2ikfZIgBDe5NSTMkSymPzbJ6uk7/OSVfd/f7eUXjUT6nPqLCMdQKr4pDScXQkDoAZ9KSkJG9EWZtPJ7gUS67AX2FT8kvkorQkJRcEhb9gCUPHzlzhCsXjpEMG8w8fhTtD3Artxatu7eKebJyi+A/+X5W7qpx9X6YPb7CYJSyeKW1nqkfFfNFL7xbn8NH6ke5WHZ4cjTL+566k6oy/NFj96AqOCdURUL28Rp2BGVL8bnC8xZZmOnx+MocvVHGaJTgnUVEEVGcM2hlSOslM60Bi6carHzu/TSfGMB7H1zzZ09xcGCec4rB3fMsPdfQme/hVSgri/OG0lmsKI/n81xNGww1ofQJbx/NMXIJy2UNI8qRfJXEOC4O2xTe4lXwaqjU4FU4s3AZvyCcO3E37SOHpy6NgwoxiBFUo9J46X10Tzc4/wUVX/fyB7i/9iTPzy6EOlEpJUIZfRQh0kEZrzVQxi01iXNY0WJ2USG5eA8fLeqhgkHJpCCVghSYNcKP3vtrPHz6GP/14kv54GvvZP49KYf+03tvSZfsq8U8msvoHTe4mZJmXjAYpfiRBQNiPaigXiiLhKVeHecNH06PcXnYZNDP8COLDGxgvVDBDoXW40o6UFZPGMp2aLBehaKya0rZO8HY8Ep8aaAwaC5k1uEbjt7RlKSfk++nMKbYNXyrzuBQQtVSmmlFWVmqyuI9gKFwlm6VY0TpJzl9n3F2dZ5+mbE8qCGiVLOGmq24MmpQeksiHms8XgUjylzep50M+Xj9biTLIDlwi5pMsQWGh2t0T1pecvdZvmfhoWghC13vueKToAswWHSD4nUIQ02xeEqpsChGwn6vsnYMQEZws/o1RS0UqjSNJwVelg35jPxJjqVL/Gr+Kbz7yRdyJEvxBTetnPev9olhNGMYHlIwytVeHVdZJFFM6rGJwzmDLw3qhWE/YzRKWe7VcU7QyoBRaJcgIEbxCoszKXgQH4RaLDX4695JXGXAC5J4TOKDT18UTUNEh6pwtVeHROmdVLJuQs0I6pkOYw8YXCtjcMjgU6Xbr62NfoxRrPFhlNVvkzQ9L2k8zpJr8OByoIYwxmON0khK6rbkqX6HYZWwUO/TSAoqH6wmExutJqC1DPpTxXwg4V2wllWRJOHpT0459erHed2RD7LoB/S80lcL0co1omS6kSttrKStFGvbHLJmIW/GWGGPvyEo5663dBn7pAfcmRR8w5EHeNeZu9EXniG5vEL12Lmb0if7WvtcLriGBwNlkQQBm9DIksShCl4MKPjKQgluuF5EMYpNPWI8SeIRUWx7iKrQ69bQ0sDIUg0tWA0KPNkoUIlhdkoog1ilanuqugm+qykOHHxmYvSF4iqDmFBfRDRaMtAvUwpvmTXB/ZCZCmtSbFTeqXEYCUrceUMinswEa2ZsNcM4XMqsTzJNcfAwVnTWMjrq+Psn/pJ78gv0vNLVhL5PScWtTdxNWrtjGNG1znhsIW+G3Ya1c3zspGXtveGoHXBfbjk812V4eIF6edBdGSJImlA2BWZK0rwiyxyjYYo6A0loBxvagihidF2RjocRTsBZXGkRA0lahUiOWgk18F6CmyPxWOupKoN3JviWnWASxdoQuWFtaKhVXajqFtNq4vsynQQ8YFAJoUxqWHtv43dXVBYRizGei/02f9B9ATNJny858n76PuedS/ewWuY80ZvFq5Bax5ytyOzGTHsjMSqoAhmO0OLaSaEpDg6SY0fRuQ7Z4T6vajxKqYauBnVWk2qDMRasY91S+cK6kt6svLfDWGFbUVKgVEMhYAUMhlcffYS3fv4R5j84y6GHUrQsdr7gVmW64TNuAmItYi0+g7QWlLI1weJFBSW4GTacE+MJxWr4RMtavaDO4Kvg9vDO4L2Qpo48q8jzirxW0qiNaNVGWBuHMUroBAATG7a1HmM9JnP4DCTPgn9xioOHcX2IboxxE3LOUFWGqrL0ipRHeoe5WHR4cX6el9XPcry2TCsdsVpkLA1qGJR6UpLI+vB2skGKB0ZFSDiZ4mBCBG01qOYazLX7nE4yGuIYaphfSsVjb0DRjmHiCGzzB9jyWoag0Cc7AYPwgsZ5Dp1ZpH9cwNzcyGtfLGazMA+dFlUDsixYKl4lKMg89GOjUYKrbHBHCMFiJghLVYKlLGDTGFfowsysV0ErGy1upSptGK5mhjJxlKXFOYNNPEnmyPOSPHEMipTBIMMYT1arKDpKdddR7OUu9PrT6IwDBOMUU4BUYeRUlpbRKA2ddGUQ60nzitV+jfet3slD9aP0XE5mKs6uLtAtcxaXWjhnqKcV1niGowaqEpS0cfgYnWEK8N3VqcV8kCEGN99icKxGJ1vEYLDC2gSeQfGsu6e2s5Svh7EynlTQm2FF8aphghDwKIftCvfOXeI97UOIyE0tY7L3ilkEaTao5pq4mlKzjsoH14II2MQFV0NlwwSfC37n0NMES1ldiNYQo9gkKEzVBDRY0Ag4J4gIrgqThyJBQXsf/pvMk2cV9ayknpaMKosvLJJBVhvRqyuj+ZzaLfiFptgbiI8uBjd2Z1lcEeuKEzQVfOpwpUWXMrpZxvuyk9SSisobRlVC2U/BCYULYXaDKsVF10YmwVio1GIqDWFy0wngA42qlTGcMRxOt3Y7GnSDL3g7xboZN2Jl2y2u6fF0zJC7Gou8u37zCzTtg2I2VEc69O6oUzUUa5TSBYVpjCdJlKKIscbWI42Y5hgtYGMU8JgkKGDvo+BEEWR9aDse5sZJP4htS0O6pQhkicN5Q6/IqCob/dgh4cTVPf3DCaaokVmLTi3mAwO7UtC4mDOaF/K8pFdZZDUJ3AZWwREifAwwV2CMcrUbln9zVQiXpApzD0vdOv1RRj0ryZKKmi1pJyO8CqtVhjimSvmAQ4xQNS3FjNDapJjX/MkTCna3SnkrOGRt5b5rLHA1axOIRhQPOFVqUnEyu4JmPkwk3wT2XDGLEUazeQh3qgffMgQFay2k1lGZYN2axJOmbi0DMChmDxiM1SCCSX9gVKwiRAUeBWSA6AIZK+cQNuUpnaGqkqDg43mp9ZB7RnMJ+UrC1Mt8sGB6Q2pXGpgipZEX9Po5th/C57SuYf6gMkjm6LSGVN6wulSHyiBFiPIZR+mUvYxq5LEzniypSIynbkt6LmPoUuTmjZwp9hFV3VA2oZXc+MTajWJdEW/cbiW4TCbh8eTiOZx0ITvIFjNQdCzDeeu8mOAAACAASURBVAGrDIoUVcFaH4aRiWNU+jg5J5RxZlWVOOEXlLgC6oXKBQoosR4VsLFDcnGYm6QhjGrYz/ClDcNdhcR6DjV6rBY5vSLDeUOVBMH1ixSMMppXRkuGljXofroYJ8NRtrHWbKcDxw6jtRTXroEqZlAh3oObCAcsK/SxJ24bmcpBgHR71C7k2OEsuXVx3iFM1IkTYlgr6oTBKF3r5NVojEuXmJQExMnjokzoi+JbghHPwKWsFDXM3+JlESTPGX7Oi+kdTUj7ii2Uzl8/TfWxs8900TZCDKOOMFrwzKc9TMzqs3EyLiV0wl5vzcd8PZRqcITJxjSuK+0IiSd3povY/CCHy4mhaAnFnIJVijIJ1qv1JNaTWResXRV8HG6unZp4nBlHVQQ/sw5D4LhmMSbZSrCOY6JAmpQk1tFfriM9G5iijJIljuP1FZ6WNqU3VM5Q2mCpj0YpYpVi3lNctjc9/Lg5+cjG+Gnd+mVKp03/7nmKtqF/xCJOyVcU8WDKdcWc9D2Ni4vwCaSY/UoXo0oymCW31VoIJT4wBSJhhKXOUAxTxGgYYdmQgo8DcSZ2/uHUsgz1KMziO4ZVSrfIkb/FHizJMi58aoLcv8rwag2zasmX5rEHTTEboegIujDiUNoFQjqJkeDGsFEp77VtVWAo1WIpsUYxQKmemggn7CgEOtxkPPzeK2YjVDXBNRxiPd5LjN9XEuuoJyWuHpzzw1FK0U8xqSfLK7yX0LA0ZuQBkvtgLa35kYO5ZGwIv3M+zK7b3OHHnM1AMyvITRUmBcbZXiZYXt4Lxiqu7nC53XORXAO9dshjGg2k3UKMAWtxh2epGgaXC2pCss5oXtapChVMCaa0DOfvI+0rjacG2O4ILl9FV3voaIRWzz6TUKsKHQ4RH+YTrPUUqcb45qCATeZiiGXs5H30DWrYpjGahyRMKJs4NxGSBNY5M+QTzb+8A1OiqdXQF93DaD5n6Z6M4Tw0P+kyLzh0gUfbh7i62qDoNKjvc5GvBxHB1aDRHjFr1/lMLEpNlLYYnCi5OkolZgLuznLeHB43maq9OeGkIVWwziM8UKoyVFj2KVVpb3q+Yl9cGa4GtCpMEqMoCLokNZ56UtJMRxxvrvBEd5aLww5pVnF0pkt3lHH1ais2rjgZGP02royUJCrgDWleYoxGDgUhrxWYxrqfuZMPyU3oQ9cVs8f7oPiNddiWw9Wy/cn6ug61qLRb6LEFfGrxeULRSSlaBpcF/uGqCYO7R9g8hACqCv3VHPXC6ks9Whpm/rpF/VKD2YcM5nKCX7zy7FTMZYX3A0wFifEkiWOYT8jN6poLS+PIqipsqGtjJtfMgygmiencsSMHIqGRUDmL/QTTy2ujsS1GYtJqcuEVbVZPKV/6ue/mtZ0PYMTj1fC+9l18bHCYv5h52YFTzABVUzk1s7JBMQM0ROiYGh6loY6+lpTe47bJ7rsejKxzbKSbJiAOG6Fj6qzqiOUYLDBWyhdch6q4eSNvXxSzT8Fmbq0haEwqMaIkxpEZF4aoomhhKa3SKzK8N6R5UCTG6FoEBoBLTIxvNmvp1cQoDmPWozRcTEABqNuSWhJm4yuXBWsc1pJXJGaY7QsmzX7AHj0CM220keMaKUVucbkNnh0JfvrRrOAT8FnMgusmuKGln4XXKEMbjOfYkvrHlKJtGC7MYQezzD90mPTJK/jLV/C93j496K1D0gRTr+HSIC8d+4yFtRSp8fsDRbdrgJs2j8Xv9+2lP0PYYkQGgU51cATc0YKnhh3eae+lYQqMeBpmxJnG0/z5QZwJNwafQjsdkkqFR0lFaBvH+0ZzPFwco2lGaxzMdyd9hmhkmbuWRW7LW2yK5DCESI+Q7aekAuecZak0LBhom43n1qRcd7ndBPY+KkMEn0GjMaIok0AuxHqKdWI8zaSgkwzC8HJg8JrQzXLS1DHTGmCNX4sZHJPOjN0Wy6uBDKkqAp9z1iix1mFMmC91MUbaoMwkfdrJiEZaMihSqiqk8qbpujXh98FYXsOEtezvPMLKmRaDQ4bhPCQDSLshftcWUHSEwRFdc1uIh9olG90a0WXjgjVdWoXMo3cN0NQxM9PFGs+F3znJwoMZjQ+xvWLezpJ/BrkjTL2GtNv4bD3hSFxw46gNlvCGYkZ3xuR8BeNOPR77t4IKQyQo5e2G03lGcXrEvacucHZlgUeXD9HORjSSgr939H28LDvLG+uv298y7xIuV47Vu9SkpFRHTSw1gd9feSFv//BLaDWHHG13+ZwjH+YVcx9l1Y9YrqJ7YsI1cT0rejyhuHZfFWYttCTlnYOjvHPlXj6z81E+vX5u7ZiU0LHJLQy/9n4FE2PwSYghrpzBSYjIqKUVwyrhXHc28uoaVvs1tOGQ1MfkEMF5wXmLj77jyq37h8MtFBMVMUCaujU6R68hVTvLKp7ut3hH9XyqyOELBM4M0fWYausZJopYGyv1bRzXblJ440UD5OQxqvkmo06KKZXaoiftCslQSXt+zecp3lI2DS6HsuPRBHzNxWScoIhkFEPDTFBA9caIRlaGZ/aG/jEFUlztOI0js9gnLlE9deHash40P6sYSBPEwZVBg6Kw66FLcUSxGTr2L6usKWNVCT9jVMckTJzzuMEs3oONax7SYltNyFKk06Y60iF9IuMjgzswQ4M4uGBDbPiTZ2Y41blKfXELa3uHKCJJM0yriTTq+MOzyKjEP/LYTfFF7Air1E2BRSlxpFisCCtVjWo1ZcULlTdcmJkBQrTEmP4zvYHbTCrumjgMIe26xPH2y5/Ee/7muYxemvB5jfMM1TM28ez247ZdYY8Vs0GSBJ8pzaygiDy6tbRitj7gyaUO/eU6ZikhXzRUhz2NO7uBWCimUhdVgnMmEh5JSNmGNQ7nWrMgyypqMdV2nKbdG2aUpeXw7CoL9T5/89gJLj+ZUx0umT20ilehlpchOqNIMJmnkRf0U8BaJldJuLln32R5jgm+46KfptNB2k0uvOoQK2eg8yh0HqtoP9rDXFgMvuCihHoNaTbIjs2gpsHgsDA86chmRrzy1FlS8XSrnH6VcW5plqJMGPXC+PNoe5W5vM8jVw4xGGXMPG+R7IWOc2cWyC+2OflAit2smLdTys+ksk4SNA8d16XlFuUgDR4M0a0Vs4aMwPXhz3rmKKzrdNV1N0ZmKnLrGNwkt8GzASZL4Y6juHaN7t1NfALH/txhSk8yLJHShxBMVS5/0mEeOnKEkw93N055bY4iYqNFbpp19NQxhkebXHpJRr6kHLlwGXf19ipmTZSFtBdcGRrqQYLlatEguZKgXUsvz/jI3FE4Fs4ZakqGWyPG3y3GLpB542gYi1el7x3v/ct7ue9NXf7wW+5j7s53sewHDN066f5m/p8bwZ4qZpOlkOdsjs8e0zA6Z5C+Je0a8iUoO0K7PmJQpBSj0K/5yCImQkwSCDHPYjSmYgehDWGNWwNCOJSPlnEiDi0MaVfwecJKVierVbTqIeNr7JtOTVgQVmo5ZpTf9hUs1Ouab1mPLzA63AyLw3ol7Sn55QFmuYfv9cE51IUeWoHkakrzyYRkkKCSUrZT/lv/HmzuaDZGGFEqFyz/1mwfGzkDhi6lUxtRz0paaUFqHZdn2hReuPCKGs2Tn8bMI32SBz++/fJaxoZ3uZ9hhBvuL5GKcyKb83rwUTGPR1ZGt7SsgTVehdQ6+s8GvTxWjju5KcaGgRhMLcfMz6GNGsXRNlXd4jLBWyhbBnFC1bCIKi6r4xMYLoTIB5+YXVl+kiQhkqjZxKcWlxvKtiJekEYdMxjctscPI3ElNdUE05tgxVCzJT5XpBKSrrAyqt2221oRUixP+JJLrkG6ItjFLjpqbDpuo/vjZrB3ijlyZEi7iZow8z2eMU+Mp5MNKUcJ+WVL/Wml83jF8HDKXTNXeHJ1hpUrTbwN9J0QODVEdM0f7H1I6y6LhKqwDDWLxwW+5mqUBBY6DfeTXkLjomJLQ9GvMTxWcnQmxEAWRUJiPK1shK8pOtfBqOIHg5u3FDefN5HiLUnK1RfNsnzGIBWky0L73BD+6sNUzm04141G0O3CpUWyRyyZETrWIu0W/uRhRocbPP3JDcq24o6NqLdGfPadD3MoXeXdi3dzZdDgxQtPcjjr0ncZI5/ACbg6X+dLXv3XfGn7r3nNb38rz/uZE9hLS+uujQmL32Qp5vChm2bKulWICN4KOjGElpgdGrJMNso6TA7GCUILTDAUQnTTx/o4ztzKTAjdXHoGoiWvi01WamBrNCGMcKsoG5HgjrMWyTLMoXmWX3aMqi6MOqGDM2XoqLqHgqtRLWgCxXMHNNtDBr0abmApO+nGTFjVLUeSptWEO47hsgRXSyibhtHRCpcl+IXO7aextBpXr974/IezVXSuxDydUb8gXO42gVAVvBqc+HUe5ht0NuRiyCXhoWKe967eTeNpoXr8PNI/EYrEeohdio/GwAGMY5YsC6tBKBQuJHOMG4dXwZcWOwA7BDMKlvBMOuBq0piwiONMvDeI8ZH8KPiFNfoLxQj4iWGqD+v6EZ39NVvGxhvYw5JVoXBCKxsxila1U2FQhdVQNEuQNL0ld4YkSTh/rMycCxZzVNBVXaiaSr4oZMuKXS22D2WLjUHVgxi0rEKm0+WcmlNaT8wwnBe680HGM0lYcr1fZvRGGQalZYc8vHqES8MWTy13GPQz3t8+xYl0KVhMM3XS0mF7/VDmaJGp80ieoXm6QVmL28fcZWvR1K6T1I79xxDDKNfFxMSuMUvhJLZqJi6S3CRyAH3MWzVs9dHI0bVjJCphyVKkXkfnOkEglaNaaDGaCZEMxoWkpHQ1tK+qLmtc12qUvF5yvN1l1BgwrBIuv+gIh93LSfoVtl8iqwPo9kL9TtfVh6YJagVNDWU7payvT75qnqL127twm6SeBbt6TQhbYjyS+LVn7I129ihPTgBuR4xfbupWLlYdHh/MY0ehPW/2WBggkxiOeeC4MsSg7SbVbB0UVoc5qpBEdrihS5GepXZZqS17kkHYfrp2hV6V87HsEN4L1SimaEfGOGf9emOMFvTaIqtKmCT0AiODKQyp8RyvraC5x6WWpK9kKzA4abi7dRmvh1lcblIUCZe1iRkJrpkhg1rwCd+k/jGzM8GvnIe4aO33oayCeyRmLlULBTMPp8w/NMBeuMqOEcbGYmqxclsLzuHOPwUXLnLoqRn8ycM8ckebqm2YsQNmbI9L3Sb95TrlcUPDFPzFY6eRx+vULgpzK8pffeh+3jt/H42LhuW7azQ6CfU8RRODJgapPDKsQnW1EwrCKQxHa8t57TnyjKqThxhub9Yn9dC1ic4xvDNrsfJrHM5mPalkjI0xzIbUOBoJ+8RQvgtEK3ltXmJixKUemOjETZ4jrSbMdnCH2vTuqHP1PosdQeNpT1UXVu8EOxRmHvVkXUf9iW6YmLczFG3B51B5od0Ycv/sBT5/5oPcky7yu3fdz4d6J/iTJ57L6lMt2o/MMvdwRdEyFO31OlFf9LQfXsY1UrqnEoqOIIVgKig7GYlt3z7ZGEO9OeJF+VM0JCy46qKfuW2H5LUSHdbpnCtYXE7x7NyIxwp5O/eDQyItbAUCH+kf4wMXj9Nc1U3Hjd0qUIOwylJc4f1G8wf21GLWLMXndt03GJEnFbNZH3FCvuKxQ79mqfR9RqXBOh6vaztOMNnyHsrGnZFNDhMmCKzxWMJCr+KVZEiIdiiElh2RxGiO8dAWCUsZaXpropE8D40rXkdqNUgdxlqwBluArCbYEcH6zFJMO1ZeH1ipRARVhbKENA3XmLDAw4UFnMMMSmoXDUMavK3zskBterZNvmp4oHkPH+icQB6r0zwv1BY9WddjC0uxbEh6YUFbFFw9DZZPEmXfyhBVpAgdopoQ2WDyuCjuPkCtwacmxG7f6ISKbPhav+ZYsUckEpY8OzAWs27hEzdhwQlJk7XOWcsKmenA/AzVbIPRoZyyYYKHJ4ZaqgE7kFDnxm3JGNQKxim2JP6G5V6ds6sLXGy2OWZXuCu7RNsMWDmW8+HsKJdkDp8luByq+pgSQFBjaJ5LqeqWoi2ULdCGo6qEohOs6b3AmNFtLCtDmC/yDpLVElNsnx4z5m0e/96MMUudj5wYY8W7UtUZDlPa1bXnrHnPRMiSCplpY9TjlpZv6Ln2TDGLEVwnZzSXoklwPYxpPO9uL/KqmY/wZ8ULmPnrS/hOnXIm8BR8dPUIi8NmWGjTurgWoESfckwoicxzEGgdYT0uWmPFM82waGszLSjVIqWQ9qF1viA/t8Tii45yR36Vs8lCcJNYHzIBc2U0l2JGOeYW1gDU2XYojPNgBJ1prlmiag3NpxzZsqG25HD1BH/HHOboDOLCBKFPLZoZbL/CLPfRLKVq50FJlg5xiowiG0AcLp16+8X1xBVNuK97FvUeqeVgDPPDj6NlPEeV5vhd1etoPcfPNhkt1ELfZoWiY+gfMdiRUl/UMFnUiMO+or5/6ctpQtUIQ/G1SbxxTHK0io3R9c51sliia+tKitG1hKTNw9bUOFIcaiEsqb7DxNp+wbsNIzY700GaDXSmhWtm2NURZqVH8dwjLD23RtkSRrOQL8HCgxXpakX29Co+T6lfauBzCdmjqUWlFeqSg2QwDskUyg+1+OC5JufvneH+hQt8+syjPL92nu86/nu073Bcen7GJddmxddYck0eHR7hweXjPPiRkzSfbjCYN/Tu9DBb8ILTT/FUt83SU4dIerfRee89xSjlXNXhmF1lxrg1q9hKIEerCkgvLJP0Gte52NZYi3Veo/oMbcyjXBq2KJdqJMON9cOrUqiJpEbCkdYqveefoP5kCz6wekOLb+ytKyM1uDRkq9nIkzHOwsvEIRVIt4c0clwWJiUKl+C8WbNoxgrXTGbn6djC3cKo2CLbZuQTxAniFDNy4Z5lmAwYcyaMlytSo/gklP1WJrvK+VAhpIotywhqJFijRkiGPq7MEXzrWMGLDSOFyq8d71OD1DM0tcHq8CDG4C1oPY2KKfjYk+UBVA7pD4OQjFmbBFJroFlHfA01wRckRQmVQ9sNXLtG1c4om8GS8gnB8mmDq8VOzwZLSDwkffbNvFRjYpmjxbzptuP3N84o3bhvfM7O4Utrq2RHf606bi1c8mYhgp2dDbHGWQbWoHmGpgmuleFqCVXTUtUN2UpOulJjOJ9R1WMURa74VMJknpE4H+CwI78+gSjgU1nr20wFST+sElM1BPHC1ZUGZ9MFntu4zF3ZZToywqmjVEuhliXX5GLZ4UrRDHMzVhksGIoZQXNHmjkaSRHyFZKQ/Xu75CPWBs6UtUm8dZRqKSpLUgKDIVLd/jpaaYhkEb/9kNGI0EgKVjqWbDnD3KBbdE9dGWUjoWgL2qho14cs9+oURcJTgw4fqp0gWxHcpcvIkTmG85aqqXGZHwkrmiiURVC2ea0ksZ56PYRz9UbZmtKGEIJnjSdLQoJJd5hTVZbVIufx3jx2KNhSSboj3KXLpL27uVy1GLoEm/g1vmZMsAqrmuVW+vjzf6eOKNhRIBeqLXqSoZIMPKZS0tUqxI2WHrwPitcIZlRhhhUqgrWCZgnVbA2cYgqHOI8ZVlSzda48r75e4QVc1owz7mFT1QSXwfCYC4k7PYspBKlCo7R9wY6gmFGqGU+yYsiWQqZm1VBcrvhmBVbpZx6beFrNIcMipX++iZT7NO43hI7Ixg7a6Dp50/iQuKCCTNrComHxBbuepr/WoScxfT8qaxMnkXwaIgx0MMAP918xm0aD1VffQ++opX9cKFue5FSPQ50evREUVQgz9c5QdjNst0G6YsivxkeuhKoBV56XkC1bZtJZxGuIajHB56sCPgFRwZSKKZX6Uz2kdCw/f5bBvMHldc6tZPwhsLjQZKms068yPnD+BP58g6QnpCsSEp0SSFrK4mcUiFVs4kmzitUypz/KMBW3jU7V1OtIu0W9XnAiCVFVQ113M1wYzdBfbHD4quKuXMUOo6uD4Cu+mf6hYUpqWpHGjq3yZp1GdgukQEMyTjWv8vApg6lyWtZumBe4HvZOMRuJK0wAJtDxjRtC5Q2rVR7WcKuqYKlaWTPAxny7jC3jiXjlcdgdrI/ax+eIBJ9yHN2uHV/FAPFghCtaVZgK+i4Ly84Yv25pGw3hQ4nc9HpdAFUr+GyrOhgniDdUQyVLBVsoPhFMpdF1Ee6nRjBlgqk8KuG/qxmKdljyKBl4TOFJ+hWjuYzBkWAZiQt+xKoVrEk7CMP5qqn4XKkfX2W2OeBKt0kxSnAjC6WhagpSCn6moj4zZGAbmCLB1ZSq7QITW81hbFjSK00d7doIYzzDWh3ZI7/hNYiyULPNVIOMreYt9sr6oGIN27xUIxrqa5ZCsfcE7Bsggm23kbkZekcs/aPC6LCDVsmdC0vc3b7MpWGLXplT+LA81tVanX49p3I1aouCuIm60FDECUXHYkrFjkIWqS1CpzbmHTGxg9M0zBmMOzw7FHzXcnG5xYeSY3RHOf0ipVqsU180JANIeorPQkdQArVWkFlYBMOHCIntkoBuUkaSpZBnZImjJhqUssoa93LPZZi+xY4ULTcqwu2iLq4Xc2zZOEozu3gmg9AwBVUDqtqNP/zeTv7FxoSGkKR2fQj1ENJyqWitWXa4oKAkKlyRwBbmnEFLwRUG16+BKD1bC8PTxK8vJyVKZQ3GKMMi9InjxTp9Kyw/U9WVoiP4WtgvlXK5aFGppZGX+HFcbKKUbaFcsdRvIaHCFIJPleJEickd1b2x117MMUODz8K9NPpMZWSwI6HqOJKOQ4zDWiVLK9q1EYWzXB1mDHo59kKDquO4/76zeBU+fmkBY5R7D1/Gq/Chs8dhEMPLEs+Rzip3NJcRoJvmrIyaYcb80Ij5mR7tfMRMNuAjcoTBqI22K2YXQnak84ZBP8Oda1CmMJjP1v24+xTBoFlCVQ/yTMZuCY36NUboWFGYmMhdew9mI5OcetCJeYpx6r4RJRVHVQcWgpXJPhI92Xab5c+7n/5Ry9InFTRmB8ggw5eGR84e5RE9iiSKJJ752R5HW90Qdz8jfHT5JNmyoaoHV4yrhdGST5WruSHtwtxHPUnfUXuyj2+kLL6wTtUQfBqU8eL9rchaqGjiya4a6k8LXGxzQdohnE4gj6Op0Rz4ekjIIneIVbRI1tpuLa1i1JPwSHPh9ghJDBw/wuhYm07t8lr1KzG4mAz98ZUFmo8b6otFCKONSBEaZrS29BTEMMmbML1mswG0S1y+0f42ItTEkUYr4FC6yvBIRb6YBPfYDdxjbxWzZW021nlDZh1J5LHoV9n6Mj4mKnHWeVDHPl+JPBCqrGVz6XixVgnfCuAMfiJ8a7yA69qWRHFZDAMj+EkHLqXyJljZKlQuXNOnYXh2K0j6wY9XVgI51OoFRpSVYYJLFKk7TBasUGs9g35GNbS0D/W4a+4qmVlf9qhpC0o19Kqcp/odHrMLtBoj7u9cYOAzLq62sEa5q3kFj/Bo4xCFCpTrsaQwwZhVCWZk1niqE/EUPqS+y0R+ixJGKn5oqa0YNFEKkwV3SSHXxG/uFcY+751I4GRMXLTF9g0+6HFdmsDazHy0mH2eYpN9zDQRgSxlOGcYLkBSr8jTisFqDiOLGRhMIbia4lPPap7TqQ0jMXzMhPUhy864ENQDof1VrWA5+0zQkSDOIS5k/lW1yFRoFZeHKCafTkyoOsLq5C5YxmEkSWQ4VDT3YDWsFuNMGIkZxTuhbzO6ZY1BFZfrug11RYyg/z97bx5tWX7V933273emO7351djd6lHdUmtokJAAC5ANNpMJih1MbMCWCUkgybKDnRVWbMexnTjEXrZhYRwny4lNsAyEIQsbm8R4EkhGAgmEhIZWz0N1jW++4xl+v50/fufcd9+rqu6uqldVT839rnWr7jvn3HPPOfec/du/vb/7u9PA/MjMjPDYzO8+KmPigWLzq8NQFr2u1/xqmNXLMAidqCBKK7w9aJgtcoBVmpoSUn9T8fXbx8qQkAEuFgAn7A0z0iTccMMi1BI12WDXSZmsCj71bOdtBnk6VX7rtstgsOuCEufMgVCGKy1aCxwBSM1z9lUY4sdFzE7eQmNPvmyp2hExIQ57ZdydajM7L+RluBxVm0CQvwUJsvt+8TIaRxQnO5TdmN37W1Rt6DpAoexG+FSZrJUkvQJ1wYhGxrOUhFJwr4ZzwyUu7C6w0hnxxOo5VpIhjy5eZlAlfHbnDFvjNjsvL6FG+aTxZFFFOyuCN/hMl2govOBO8nJvJTw4ldA6F5NuwcC1eCG32M2YdNOQDWBxRym7MfnSSggdGFjchaWnay3rlqmlSOsb/Adv+hK9bvjYUrUleGcQBupmKll70NZooBY2mLrTOg1TTT392ls2M54TgCWwcqqlFLtztAURrwbT7YZO8m3BR0r8hTa5a9Mb1/kCT80ZF8CSr/R4aaE7NRetXaHoBePp0uB0JNshV1B2laqr7D4YYceWzlKEj0IS12XBqxaF9kXBFELVCnH2cjG0WvMdh6SOKHFEsSMfx/hRRLwdkb0YYwvqQgumRZgqMVW7xUfPvB1TCL0XQwjlKOATi8uCk5eIMDq0291Bi1PnHdFuDkkyHcyNyFXFKK+GRuLTqTBRW/OYcxB4R/ccm2favLTwMN2ZzxgMMfsVgLEE5+tYGWYAH4WbAxWqyhJFPijM+fBq1YOaRgaXhRG7dJaqKRIwQZWu8Xa8QllXEDY6y824OC0qMMJsZNh5Q+nDSO5Sxce1bKiHSTVTuTQTu/axohG3pA2hL54Da8kGq6SdFj5eIu/Veg8GTBU6u1StiDL2aG6R0pCXEYMyxWvoqrE5bDPcauG9sNXr0IlyFqIJgyrhQr/HcJARbweO78ZCjyQNBrSqLGlfSPYIWfzcYCsQJyS7kO54yp5B45jWJUPnfEhOxgNHNbRE48D88DGku0p2cYSohiSlBG7xcBzF3AAAIABJREFUbWqldvW1jASXgI/0QJiiQYghX/vBn7IypjsLxlnkas1dCF5mmFndOY/ZdNpoOwsPsEC8F5wWW3CgOazUVXshwVufh4ZtfM1+asT0TNnEi0M+o1gAmwq2MOEcs9rrFYJRrQi5j0Tq9YpfqGgtTOi2crKoIraOCywwnkSYApJdDffMSAN1sj4WcUrVDl34xClJPzCQjgJqw70eXUeIqCotSb9EiirkCg49wrMe8+Fu2ofRrDvMY16P9rinvcML1zC4VgQjgp92aOCmnpM7IpR/GM4bytLSqa9t1bbkK4q2HIULdBdfd6AYF/H0wWr+N6KYKFwmW4scNbA23MlFEU2N9aiMkUhDMiwJy6Jc2djtkqYlnbSYTndN7IOX0TbXTia9Tsh9Z0EE1w0c4u6zA7qAxoHHrFbACsVCTNlOggOqStFd4PnO4nQ/plBOjhRvezyZvGUa6zMVdIbKQqFkW3md7AsCR2FaC+lmHykqfBqh1mBKF+L5pUO8Z/lz0X4hiSqSu0ChE6EX2cC5ji1SVJjBJBSXJHG9bXnHeL7FQsTkhOLbgXI5pWqbMI0Ws2+9dPpPff3qxgnN57QOh4XWZjUTo3arjISpfNWx4aG+Q9j6/Q/gUqFYDGG0YNyCBncTA0YC40JqzzSqNYHCIBlCEj4K26mA1p1uTBW88MkpBwLD+4HI01qagBeqF7uIwt67J2SdghMLAxbSCZG4OlFvqdTw7JU1Jpstkg3LwoYEw29D3iZfDhxzl1AXnNTHZmpd8WFIWh8ljCgxgsUfGGB1NyF+8nlY7OEeexP5iseKUN7CrRqLwyB1qMLwSHIJ1zN8uP0VB7azIsQatvN4SrWhe/tNkHvuimFuCkWakl4fG1zLI7HH1eJEOAEjVJU5kORrDG/zY8jM37MdThqvGqBywWPWRKcxb1MpZR5hrcdkOqUsiwlxvMazvlloO0UlqNmhEG/2g4xnHIxkc+5xlqBJXRllQoWbT+x0lBWvSBW0KWQmbia+5jvP8JbTuhpQK7dfIOEVUxSBjVIUqFek0w6GZ5JDnmMWF5BeFyoHVYU6HzQAGlU5qPmjJggJqcJ4cscMs0sE1/GQ+CkPfuqJXC/i1Mz8mz+bwTuw6qYMHjgYo1QbGAu3q1LtWhidqA1bGvoXas1ocinTVmLBMO8bZVM0seB9rYupiqNALRwcjEJMyGnEnjQriKxnuT1mVMZsaxfxsLo64IGlTd7cvcxKNORSucDYxVzOe+wVGZNBQrJhyTaE9mWPi0PYRO1++KNhgtg8HGeoagWbO8w1Yr5HgaY7dhM+kFxwVzaxiz3y1RTfuvUB4XBcesUU3Btv7ocoPORa4lSnlDqnilezrwt+g7iNdLnQn86lSiPkDuGBKIuIahjiUxCEVGS5IIodlQvkcZNVqAplHh2o5moqvxqtjGYaGzLv+18fqgz3R9O0VVKIUtV6EzZXdBBRRJ7YeMpGb8PL1Ou4Fci5S5goQk6t4rOI8lTwgk1RZ9ea5rKNEL5T1HtM5adcZhnnIWQQWWgkQxsBoVmj2JSPa8S0DHwmI91sL3GEGhOKWYxBnMO4EJ7wkanZMTMPkDFheX28ogp1wYx2sluKwd8IXCrIYh5Oqckv2IMUpgNQQVRQ9u8BR3AI8Ad/3Fk6ZSwO33LkSxG+fed6Ki09V5EvGvJ3jzixNOD8yjIMI9LLNnjGBVc93D6uOcR2xkstCfmV2oOWqi7Fngi5xKhAkWeUHsrxMlJBdy/ce5PdNT4XrfFk/mhQPBwppgzhDXHwwHZFvDMIoaxIpnROU4b7tYEoUNUNHrwPs7PtvTDYHwGakImf4S7HTXyn+X71FGcWufxEzMrZDUxTtVdLLtzoXChU8sFQPc4N6RnhMTOcGuZ02/CPdu/n7dnLvCdVPH7KnZbjGMpQuz/aN3bECHgXum1IHXfykZC1irqArfZoreI9+LKeuzcqc7McQqnlHOsY9Gz1VxQFulmDOA49BV0c9FmlUuzY4ruBlWFE9zsrNxfyFir/3OZWEJdZXYLEUnVjfGyIR6GwpIEp3LQ6cHozF4oMx2h/AFEUSqqlHo28D553aDUeKsOSGeU3EbSVBI9v9jPUOhgzM4GrRIicIjUdpakOVFt3jq4pjWZShYKPLL5juhIulmnXdO/2GyVwjSpPZj1jbS6Rp7xGudCskJHFB4ZD4mtqnr1jekbZlQmQ0etMeMvyJRTYGbYodxeIRqEoBM9Uy6Pxjhv2kCmDZyoz59zEexvvNRqFMEjcDx5ta8OHYpP6JNtXQuFT68IYO8iR/ggtinCvORe0usuK+PRJ/NridLA2wzG6uR0kSOsZGUCjUBhCS35/+RHBqwm6FBJKoQ/IJ6hSdiMmZxwPL+zV21/9/a8WX56FqSl2IceprJiIVOKQhyIwsD6x9wBLdsRXppswQ9+7WerSbecx+6jh7sws30vILliS3VDF5xJhuTOmcJZRHjwVYz1RrER1IHqWjdFobkz3p4G1EbYT8IaqjCgLcElFFHmW2mNWWiNe7i4AkOyVtC7GDJMYztQDQvNDRR691dyPKr4osecvE7VbaHyCytSawk0oQ5Wql+Bjgyn8fvk2IIsZplwMhrpZ3oQRZhMphlAcIEF4SEX2B5baWEtZh0S8QnnQc5llmU3vocaga6gKUwkJ2lAuWBvserp9J6AmDKxF0cyeZDpAh+5lISzhGgolTAfvUBFax5gbZTrqz8wY5kYLIc4qisWUqm25Uz6zff4i3Ustqp88xW8vrzK4ty4OWq0oT3ui1GEjR77Vwg5N0AxPZp6rSDGzfSsLi4xDwYi44DG3LgXD3NzXVSpYo1O9h6IXkmo2z4hjS3XPAj4S0o0Jtj9B0hhN7FQB0acRLjVABx5am0oNuMyQLxqqDPKV4OgkfaZO2K1CqjCAVGpwqhgglX0mRIPxquX0Q5d419JLNFL/jkOO1w3A1d72tU6j+4rn3/7uW+Dt8B3dXzt0wDf3fbfZMHPNh9eMQseSaBR+Zh/BQjphL88Yzjw4cVyxkAXjXU1pbWYaHzQSWBzOC0UV1VocFkRxpUGdoRJQdXSTnPs7m7yQPVgfQ0m6rUzW9g+wUZfD6tF0y/YOt72NDIaY0ytIo7RnZeqtVpnFtUwoGZ80DUYb4wc299jxTAWT7oc/8Bp43NZMdTUATD1tbITlrdvnul51YzYe1jXCEqJ1GMQIvp49BIMstfd2h1xmE4qSKjPTq+/QV1vRAxMcndLo/AEOc/isHqDLBSqU4NQQJxVFR285x3AjcJcuA9A7d4HFxR72W9/M8IwhP5lzcnWXN/W2WUlGfCR5kL2tDlkvZ7EzpqgspbMstCacbPen+7s06nFxY3E6CFXbCXE/KM6VtVSnRmFAau6JqhUcpKptEB8xOhFRdgQ1LVJrKBcTyq4hHniiQUnVtpTdmg/dkjpGLpQdmJzw+K7jzL2blM6ysdEL+uhHcrE0GGZvKAlhCSsyDVc0KDvCe1Zf4dHswjV2MRPKehV1uVl46kIkdBpCaZBtVbReSHj6vnXsfQZfa6yYWpP5ZmaWt7WDicvAtx2SOMRAOy1YaY3YZrUWTXFTumk0DSeYA0m+URGHRF79YDXJn1nvefZva30wUhoKURru86BI2cj3qw2ldETjQMrvxEXouG3DfFGt3rrHPAN1jujcFWxa+2Az0zo7WqDqJthxiRmXtfqcReskoJ1UmP6kDlk0YuQ2eA67oxAHrpkfdhy8cNMPiTm/1Ane+KQK9KGyQpzHdzM0iTDjEvJi+ntpmgRhpMpjiiokFssqNDtYaiFeMZMSqTy2KO+YHrOPoJsWVN6Qz0YIZ9g6sXVh8J7yxYAo8JvttCJw/6OR8aRRFRg+4qnUkvuIXivn0mpJ2Yk4uqZErw/qHDoYsvqpbRafzRg/lVK2T/K5lVO4VghLLFYwPBtz6UQaGkE4YRB1uJAs1TsRNDfYgcUnCr0KqYR4HGhzk7Vwb9txYFagJrAyHlZcx5HuWKKRZ/dBw+R0RdWK6LYNOw8Zxvc4Wudiuq8EveV8MdDqqm5o5dSUe5tSsFcidp47RTSG0684bKnwp4/oOs04BKHdk3DBFeyUtRwBUPbgm5c/zVm7y553IT58aDS/0Q4mDqHUYHiNCGVHie45iw5KVp60nH98EVcrFb2WBvRr4bYm/3zMVGtBJDRkXUlD8UQ0CbStqWGuOSUNFcqawATMy+gAw0J9baDdfix6NvkXOl97nBM8IZnnK8O4jNgrsyl1RZwjyoNhbkcFeRVNBwPv61LyW5D9PADvqC5eOrisFkKPVImLDjKaoOMJJoogsmiWIt0UMyqQvUFgUbgU0gifBD0N9gaIMbU+rw/Gt3KwsQ3qkVaCSIzkJZKXaH+I5jliVgPLYjBC9wb7P9liDxf3MJMC6Y8CQyMvkE4b0wrdXSSvkLzAX9kMAu63GyL4KHSbGRazRnl2EyVqRKga2JB0tsZPQxbTzicapvGxcUGHmeBFlWrpJAWtxQlVdoTC7q8X3gXhpM88SR0kABGiM6fRbhu33Ma1IkQz+jbGVCHBF7jx0X6M2YVOPVVHKduCcYLNPWqgWARNPWYUJACSEILFnB2xvjgk//QJVGByuuL+hy9xrn8GUxpGjxR8xZuf5xOtBxCfUCwp5UqFaVcsLIwZjhOKnQwphGhoSLeFE7+dE29N4PPPXLuX5M3AXP2nEeGKa/FseQKbhxujasP7sm28KkP1jK7hJd8oyimfuZ7ptzxufQk7LOg9nXN+e/HA9v664i6vjdtnmJ2jdUmpWilVT3GZZ6PVITKe9kVh6fM7oIp9/FHyJWHiYiZVRFlEQT2sLp2dJnsOY/aERa8qJEjTwOrIJzHOCUnkWIgnPHMC5N1vQytP78ldhqdWOD9YpD9JGfdTZBSRXrF0zgV62W1DqDFH9/qQF2hZoEWJWhv400mM3YvRssTnBWINDEPNfZwkUFX4wRA1JlDnRKY0Nx2PUVXMpS2iOEbz0LZK8zw0eN3cwQwSdDLBjyfTQzLqsXmBVm66LfXL1olJLcs60VPedsMsaYokCS6FdlSE6j4/a5Hr/yTo8Mrs8rpUuAl5HQyBBIMdzRjyygePeTEZU/Ysw7thmK8FVfxeH8lzouGYKIlZL5ZYeDEJM5bmFJqTr0NWoYzaUPQi4pGj89wuPomIh90gil95TKkkW+EejwcdyqzD2mdHRJsDzv6bNXY/fYYzrzjSzZzWZounPvUop6942pcmVC1L2TG4JMWlGcslxGNf0+MqopEjObeNDMdU5RFJy10DsRgyiXi2PMGHdx4jbvwMUTKJGGlJrsGo3mxJNuyHOvxMjNOu5Oy8tcfCCxOiZ85jRssHP3MDlYaHcdsMszpH92IFEjE6aSh7wqibshVVtC96/Ke/gH38UfpvXqJYUEpvycsIn1vEKiVA7RlLw75g3/geDm/OaiIAJFFFbD1FEaghsfEsJWPyExVbj/dY/uIQPv4Zum9+Lxv9DkUeI/2IZNuy+Iynffk6jS6P9CIpbm/v1vfT719zcRO7vGr5dbwXl+dwrU4LkwkcxXHeIEyaIp02Pm4Ms9/3eg9lu81MzBiYyn1e02ExYUYWiavjy4ZSDZVaFpMJma34fHr6dp7aDcH3+zDzE8tzvO4wS1NY3gyh7U9de7ulj+2/d0DnqWenjRQAer8Os0OVndn39XC751N1bSERlufzdT69cYa4X4c2BSIsUDJRi6sFYa9V7XmzOL26y5WH2rSvRMily9jJQzPHdmuz7dtnmMuK9gt7JDstuhdiqswweTrDtVosPdVHgWK9w+6Dlqrn2Rq1KKtglJtuIvtMeaZGWYwi6FWx5RB7FqpqP/asKqHyK3FMqohzoyWIPeOTEZ2LCQnQfXaP4b9eol1BPFTiURUoQ7tj/J2Yqs9xfbQytNcJZf0NDme5Z6tBm8G5VjSsWWNTxo2qBIpd/flAuQr5iMpbSm9p2ZJeNDk6Yfc5jhShzZhMZzuNAdwou+z0Wyxfw+co64YYRoLHPNslezbpZ2eM9vWodLNe8FprwLlVT9mxB7jRDQvE1p2yb4ZIcPtCGd7hPvdFhDCypjAdgZvTH51M6D9aIomnP2gFLnLk67ZSh0RpgEaQRkRJ4+DNFpXFexNock0sWgOrQyNXx5xhUkac211EEs/wrKf3UkQC+E9/gfVPX334c5N89yGtjGqpHZJY7Fd3qtTUkmkoYz8JODXMgNRKhlpLe1Lrr4TBPaCqBbAqNeTespIMWYv7c8N8TKFRMMyznq8Vw0betHs6GD5wTRdsDYUis4a44Se/HhxqwYAVwz3tHT57YkzRbXO4gVXQzNAZ7eYbCzbf3pLsOsF13V5XU75t8IRDvKyW/6xjiTLTUkpmDPOBr5Agkq7UXSoIYibOyXR9ED0yGKu4tsMld6UafY4bgCYxVTfB2xADnrYcOyRlakSJpI4Zy6yuN/tGWffvJ8zVU9rChfthp2wFGtltjmLNcROQoDPj7UF6m1PPlXGXeMsSD6/+4Ro2hrlGYKuhwF0Ls0nCkPSTmpmh5FpyKtnjxHKfcevafQUTqTDWHzOPGZAoDjzb4tUbW4pRoshRVTa4qrXna4wnjt2UpQEczLwDxtSJr5pK13RMcKVBvcWmwWsuSxuy8ZEjXqqoWt1rHMkcxwnaTimWIjTWadeOaYwZpgO6NZ7EVkxcFASLVKezq8ZjrqvZQ5upyE9pchDUCicuYuIiKm/o22wqFzDH8YKPm1DGQWfv/N4CnVeEZOfgD+fZrxIMf1/bDl0v9txUIgeDHDi0E4WRL3k0u8DuyRa/vHjqwGesGLyGxgvWevxxM8zq3NR7ueZ6AUyYejZUt6n9buLD5mAxwDSmPLPfqS6GAa3V5lwTY552r9hXF6uc4RYSpnPcIbhOwmTJ4GNPv8goKjtTcl17QSZwlfdjzLOzrIMPmwiBqVHfE16D/GylplafExaSCSvJKLQE6/VCmfFRUb3muGVo3f/xqhlPEdEaBgGoaG0VX2cmQ+eSEotO9ZgPMzPcNCZ89fJmnWdfzzmTwAa5N96kbFt+KftqIKQ7cq2IsRiETEqypGQQ33jC8fbO5w+1Xz8MNdRNMX1dOgtxFJqxNkL5sT04MrqZCkCoPWb2dZsbKcdZ/WbnTCjpRnCVwVdmv63VLJo40N1uWz8HAPlKwuAeQVPPxqhNUTZiTeE/kaCJEtmm0kpro7tvtA8jjh1JXE2TR5Ua8iqiqCtIH1goeLB1hV9ZVOTUOrK9e10Wyxx3HhoZNILEBKF8CF5wOY7Jtl0ozLr/NNVCuCdSiVg3RV0dGHBYBaWp5GuWN3+XqriZAaAxZW1jicXyZYnhy5IN/tJirXXjhF1f0DMRLUnomTHrnSG7rZUb1na/e4HWRtNBdD8OSN3eZzr1lGkpNnCg8s/PLBfRmkElUzGjqo5HNoL6EPQ3xISeZi5JwNiD8W8x3FCP8TluK0LjWZDKhAo+60NHCGkaKYR7xXlD4UJ5cijL368Ozcso9PUrLDihKoN6YV5FWAkhjdi6aRVgy5Y4gnCQjCZoca0RfI67hYa7Xamh0H2GjViP2gipFLM3RopANRhoydNVN4hUoWRSsWav/ZtmEozvphNKzIFZ+az3PNGCTGqdH62bFhCKfa74CKho2YREHEvp+Ph1MJnisCdqbOjfZYJxVqAsbfBwbNDcdZXFy37HioZ1MTXIteRnE1c+/BVlEaFOwrTWQJxUJHFFKylpxSWbvc5+g0TvQiWcaYRu5pyM44B0c0LvXMRk3RBbRzstiK1jnMcUKhgbCkcqZ9jO20yqiKq0B5J9ZWnDfTCwiBMqUVxl2I0cpTcsphN66YQzrT1WkwEGJfcx0Uiozl+Yz56OGUzpsSVhINZQ+myIsLGj6Bg6Lxa4Z14g3jsBwHNlxj/d/nIglOGfTnb4ytazxDPq9Q2/eclUTFT47fxedlz7wDYAZd1iatEO6ZiCvs8Y+pR4L9xr0QR+e3IfjyUXOGFhyYx5sL3Bb7QfuuGmG3fFYxZra23gOksOB5pk7pdY78cJG9bFVB+B2aKS/fdTzYzIoaaRHayrCGc8c7VgWlmQM7xNIt5z3BrM3pjW5ZT2xZTzS6vYXslib0Q7K0iSijRydNOc1FZ045xRlTBphwa70xJ+FbwVysXQO7K7MKGVlJzp7rGQjKd0qReGK3xu5xRbwzbjSczyeZ0b5WMIHxt8BIl1ZLLPY07TiqIXGlOId3Rfhj/90vt5YW+Vc5dCRZ4YJUkrfnHxiQP5hyY/kdqKyhsu7C2EPNQhW9rMxKIoNJVuwqTd8/tFLQY/TSobUVJTXVue9jVwdwxzlmI6bVy63wpKfZBy9F6w1pMl5XSaquwXjDQ1H41BjurYcnMNGzWxuB3+748zijwoz5X1NBZCAoETq5jdAe7KFVDd14ydx5qPBfxzLxK/eI6z5+5h/VOLXPiqNsnX9jnb3eXtC+fp2QkrUajBLTRi17V4vrvO2MVs5p2gQOYtVjxn27v04glPdF5iPdrjsXiDnhE+PlnnmfwUf/fTX8/S78asf25C+oVX8P3BLcrQzHHkMELZsRQLwom0z5ptTSlwZxd3efa+RRZfTEiBEz/1WTZ/eZGez3msOjfdhQSWwTV3r2KwwD1ue1+058AGh+xDvcz3n8cTnL3VaEBPSqzExHh6doJE/oa13e9OjNl7tKqQClwVhD4akXzvDJUJsouNMlyjpasw1UpoOpg0XOUGTcLH1MmdPI/xpUGM4iSMes6Z0OiycuhsjLmJL89jzccCWlVQVbC9R2INnfMpF19Y5XJvgZeWl+kkBeutAYlxtGxJpYbdIqNSy7BMAuPCGyLjGbpQPvjU+BQvmjU+Jffj1PCpnXu5OOzRfjGm+4ojeWX3asGpOY4NTBU6q+yULS65MZmEHnulD8a2UZ7z/X4oZb+Tx1bCy8UqmZSsuz0uuR4vTFbRcXRAUfL14O70/BuNYDwh2/XodhLa0kcKXvBlqM4qrIay2lIQL3VroxmtljrUXO2HnGm6NqBgipDAMXFtZ6Mgd1gZpTJw4qLHnTt/UIinHiXnsebjBbe5hezssnbhMuv/uhMy3JHFrXS5eO9JJouWwX1C1VbKZQdWkdgzK5p/zi6jHuRSSjQ0rH3G0b6YE22PWB3nrAyeDqJOkzkD49jCOdKtgnYm/M6ls/zTxbewHu3RMTkXd3vEA8FO7t4zm20qH3rpvTyydIWXl77Ix/Ye4t8+8yjtl6IbFvy6Ox6zKqiruwEHQ6z1e6mC5KZAaF3TtGl3h6YC07Lb+uFrPOm6xY7NG71Z8HFdwksw/kjoMv1qIkXBOB/xec9x4xCpaZcOt1McEFmyg3Va8WmkysiXI/CCaxk0Diyd2diecwpOSIeGZBc6r4yJXryM397BTSZXf+8cxxLiQkusqrIMXEbH1OwIZ7AzY/HdgHHKqIjplylDn9IvM6rckpXccFhUrtKjmGOOOeaY467izvXPmWOOOeaY43VhbpjnmGOOOY4Z5oZ5jjnmmOOYYW6Y5zj2EBEVkYdvdN1r7PODIvLRWz+6OeY4ehwrwywiL4jIWEQGIrItIv9CRO6928d1pzBz/n0R2RGRXxeR7xc5qq6wdxci8uH6d32trkRfshCR94vIudfe8nXvbzDz8jPPx0BEvuuovueNChH5EyLyyfp6XRCR/1dE3neL+/ywiHzfUR3jtXAcH/hvU9UucBq4BPzdu3w8dxrfpqo94E3A/wL8EPB/XmtDEbl2CdMxhIjcD3wNQRrlP7irB/MlBFXtNi/gJerno379k2Y7EbnrnR+OwzHMQkT+HPCjwP8MnATuA/5X4Nvv5nG9LqjqsXkBLwDfMPP3twBP1e+/FfgUsAe8DPyVQ5/9k8CLwCbw3x/e15fC61rHDLyHIHr1NuAngL8P/DIwBL4BOAP8AnAFeB74M4c++8n6ml0C/k69PAM+VF+rHeATwMnbfG5/Gfj3wN8B/vmhdT8B/D3gXxDajv4G8NDMegUert+/r/7933+NdSnwtwgG7BLwvwGt6xzPB+vj+XFgF3gS+PqZ9WeAfwZsAc8A/+nMupTwwJ+vXz/Kfve0cf17DerXmdtxfwDvB84RBu6LwD++3nHNnO9HD+1v9tp9C/D5+vq/Avw3M9v9YeB36nvl14F3HDqmHwI+A+RAdLefo/q4Fuvr/x3XWf9q12oZ+OeEZ2q7fn9Pve6vE9p5TOr9//htOf67fQFf5cZrA/8X8JMzN+LbCV7+O+oH7wP1urfWF+l9QFI/nCVvAMNcL38J+AGCAdsFfl99HdrAbxGMXgI8CDwHfGP9uY8B31O/7wJfWb//z4Ffqj9vgXcBC7f53J4B/ov6u0pmBoL6vDYJA0kE/BPgZ2bWK/Aw8E0Eo/yew+vq9z9CMKYrhKbOvwT88HWO54NABfwgEAPfWV/blXr9rxG8qwx4on5I/0C97q8BHwdOAOsEY/U/ztyn5+7A8/H++vj/BsHItF7juD7IqxvmC8DX1O+XgS+v338ZcBl4b32v/Kn6ONKZY/od4F6uMwjepWfpm+rrc82B4jWu1SrwR+vnowf8HPCLM5/9MPB9t/X47/YFvMaNNyCMzCVhJHv7dbb9UeBH6vd/GfjpmXVtoOCNY5g/DvxFggH7yZnl7wVeOrTtfwf8o/r9rwF/FVg7tM33csjzuc3n9b7691yr/34S+MGZ9T8B/B8zf38L8OTM31qf14vA2w7tuzHaQphFzHraXwU8f51j+mB9f8nMst8Evqc2Mg7ozaz7YeAn6vfPAt8ys+4bgRfq9+/nzhnmAshm1r/acX2QVzfMLxEG7IVD2/x9aoM1s+yLwNfNHNP33on76Aav1XcBF19l/XWv1TW2fQLYnvn7w9xmw3wcY8wfUNUlgqfyXwG/KiKnROS9IvLvROSKiOwC3w+s1Z85Q/CkAFDVEcEDe6PgLGFKDTPnSYiwhhIbAAAgAElEQVRDn6kThTsisgP8BUI8DeA/Ad4MPCkinxCRP1wv/8fAvwR+RkTOi8jfFJHb2Rf6TwG/oqob9d8/VS+bxcWZ9yOChz+L/xr4WVX97HW+Y516BjFzLf6/evn18IrWT1qNFwn30hlgS1X7h9adrd+fqf8+/Lk7jSuqOltPfivH9UcJA+KLIvKrIvJV9fI3AX/+0D1276H9vszxwyaw9ipx7+teKxFpi8j/LiIvisgewcFZupM5neNomAFQVaeq/w/Bc3kf4WH+Z8C9qrpIiB82AhoXgHuaz4pIizAd+ZKHiHwFwSA01K5ZQ/IywSNcmnn1VPVbAFT1aVX944Tp2t8Afl5EOqpaqupfVdW3Al9NiCH+ydt0/C3gjwFfJyIXReQiIXzwThF55w3s6juAD4jIn73O+g1CfPfxmWuxqCFpdj2clYMK5vexH3NcEZHeoXWv1O/PEwzW4c/Bwd/nduPwd73acQ0JAxcAInKgg6iqfkJVv51wr/wi8LP1qpeBv37oHmur6k+/ynEcB3yMEPP+wHXWv9q1+vPAo8B7VXUB+Np6+VRD7WgP9WocW8MsAd9OiHd9gRDr2VLViYi8B/gTM5v/PPBtIvLVIpIAf4X9i/glCRFZqD3cnwE+pKq/e43NfhPoi8gPiUhLRKyIvK025ojId4vIuqp6QngIwIvI7xeRt9cewB4hzHC7JJs+QBhc30qYEj4BvAX4CDc2GJwHvh74syLyA4dX1uf4D4AfEZETACJyVkS+8VX2eQL4MyISi8h31Mf1y6r6MiHU88MikonIOwizjw/Vn/tp4C+JyLqIrBFCac26S8CqiCzewLkdFV7tuD4NPC4iT4hIRnhGABCRRES+S0QWVbUk3BPN/fAPgO+vZ6wiIh0R+dZDg9axg6ruEs7/74nIB2ovOBaRbxaRv8mrX6seYZDfEZEV4H84tPtLhHzObT2BY/MixKvGhDhzH/gs8F31uv+IMN3oE7KkP04wWM1nP0iIkzWsjFeokxlfKq+Z8+8TElEfA/5LwNbrfwL4nw595gzhJrtIyCB/nP045IcIiZsB8Dn2k6V/nBAnHBJush/jNmXTCeGEv32N5X+sPubo8HlxKE7LwVjoA/V98H3XWJcRqFHPEYzLF5hhqRz6/g9ykJXxFPCHZtbfU99nW4R45PfPrMvqa3ahfv0YB2O9/5B9xsttZWUcWv9ax/UXCTOLl4HvZj8+n9S/03Z93T4BvG/mc99UL9up9/tz1PF3jjn7iRBr/mR9r18kMH+++tWuFeGZ+nD93DxFiL1r84wQchdP1dfrx27Hcb8h1eVEpEu4iR5R1efv9vHMMcccc9wIjm0o40YhIt9WT1c6BLrc7xJG8znmmGOOLym8YQwzoZqnSdw8AvzH+kacDswxxxxveLwhQxlzzDHHHF/KeCN5zHPMMcccbwgciejIHzTf8XvC7f5X/udeNwXvlq+JCGIt6hW8Izp1kuLRM9hhiXn5MuQ5bnfvui3VEcF0u0i7Rf74vZQLlt5vnad6+ciEz4A7fE2+RHAj1wSO/rrYNz/ExlefYO9BeOzrnmNYJTx3fg31gokDC86XBlSaWwUTe0QU7yR0olcBUU6e2OV0Z4+XfvJhVv/hb4K/+Wand+NeMVmGdDsM3vcQ5/5IBf2YxS+EOpGqPT1N8BCNAAOTNfCREg3D4U7eOqbTm9D52UVWPn4R3djC7e0dxeFd95ocKzWoOa4DEchS8qWYRIRU5HUx3KXdQjptXMtQZaGz9BxvfGgSU/SEquNZTkcktiJrF6gKaRwaEBeVRXXfJsSRw4hSeYP3gqogoixnY5aTMc8ngslStChetYnxsYGxiLXI4gJ6coWiG4IDapWiZmC7VniKxAviAAE1UCx4NFLEG8QL6gx5HpNmglvtYccTZDQKna9vUyh4bpiPK1TRqkKiCElajB49wcvfqrSfz7j/mQi8XrX9LEyaMnnnfYzWIwb3GlwGC093g/7cHG9ojN/UQ75hi685+Qr/7al/iUd44dQyFmXd9rEoDsHN1GBZFItS1NHNUi0Ow6oZE4vnmx95Gycffwh7fpPqlfPX++pjA7uyhJ5aZ+Ndy1z5mhKJSgCy1TH3PLJNbB2ZLSl8xJVRh6Ky5GVMGpd8z/2fYtGO+NBL7+XyThe52EbPJ2y/BbYf73DmIy16n4rxWzv4fv81juTmMDfMXwqwFp8Y4l5O1YnAGogjTLsNzuHzPBhmEzxisRbJUsqOpewKxaLiMsWn0dXlkCK3bdSf4+6gbBseXbvMl/de4k1ReMR7ZgMDrJiEeEY+wuNxqlgRDHb6d0mOV6VrUgyC7zrKXoJNk7t0Vq8Tjafc61KstRmvC2fu2WIwSenvtrDWc193m4VozHI8YuJjXkqWmbiYUZXQjXPe3X6OjhT8YvIEm7aDToRoKIzvqZBOxXg5pbPcwxQlOh7fFs95bpiPO8QgImRXJmSf7JJuKX6pi55YYnhvm3jgSP/9F9CixJ5YgyTGL7RxaYSoEo2hPFGytDagWOzxhm0dMscULhXe0XuFe5NNtnxBqXDJtXAIz9XmtyMlDmHHtyg1om1yLJ6hb1NicRo85yfSy5y1bWy7Yryekmxkd/nsXh3RyXX8iWW2H1tk+y2GfN1hJinjSYyOI4al4SPVg6wtDPkj9/wOa1Gf08nO9POlWn5l7+28PFrmhY/eR/sCVG3BJWDGBi8R/QcgX1li+YtdOs/2YGMHd+XK0Z7Hke5tjtsDI5hJRbqlxCPFtROqXszgjCXZM7QWemhR4JcX0FZMvpLiY4MaQZxiEkcvy9FoIXjV6qcJQsQAfu41v4HgrXAy3mXJjMgVCjVMNKZQy0SDiODE5ABcqRZwhPUGT9+3KNVSaDANkyQYHBs5qpagyfE0GRInSByhSz3y9TbjVUO+5tDMkRcRVREhpaDekA8TdmPHyCe0Tc6paJdYKhJx7Lg2v7nzAM/srJFdEdpXHIPTNhjmUtBCqFrgY2W8YUm3O8R5ien30ao6svj78bzKc0whcYS0WhTLGaNTgnFCvtQmXxaKdw7ZzSNGJx7EVJAvg0+VcrkChd7TlmiomPMZ53ZPcCYB++iDcOEybmf3bp/aHLcJLoOvbD1Pzzh2fIRXIZOSWCoyyul2Xg09O8apwYrHovTMGI9h4mMcBlunmRc6E8brXapuwnFMIVe/721svSWl7AllVwEl2TKIt4hLSCLwieJaiiyWTMYJP/m597K+3OdHHv2/udfmxCJ8vsz5+NMPYi8m2HUlX7XgA3Mj2xSksmgE3sLoNIxOtYkHbeLhaZa+OEI+9ukjOZ+5YT7usBaylKplqboKCj4S8hXPwyc3mFQxL45OIE5goSRKHPeu7FF6w/alk4gT7ARMZfCRUi23ibczgm7PHG84GIuPhJPWE4vlijM4BCMeA8QEulvjOcc4rATja/Bk4vEYnBjQ/UKHVlwyaIFPzPEyzDWtdHQqYe9B8JlDE0+0G5FsC7YAOw6DVbEo+ERBFFcaZCvhkjckeHomIhZLgsNsxLSuCOOTiks98UAwhWDHEI0Vl4XQRrEIVcdTLAk2F7KdlHYUHUnMeW6Yjznk9Al23rnGeNWgBnwEZc+jieeZi+sY48nWx5SlRc61wMPL/QSsIkuecgFYy4kSx+6wS5W1WdtegIuX7vapzXHEsGur6NkTTFah7xUrjr7vXMW+MOKxeKx4HII9pPga1sU4BA9UONpxQdVVXGK4nR0VXhdmEtb2zQ9Rnuqx9yaDW8uJriSk5yymAlME+pvLAIF4EByU3GZEY6H7IuRXWvyFM/8h93a2WY2HPDdao33B0LqiVJlg2hANBFOCT6CIBfEgDtIdSPYM+ZKSrzr27otIv+rtxBd3cU8/d0unODfMxxy+kzE6aSk7gChqFe1UoILbSXCJZ+n0LuMiphq2sTmoRPhU8W0HiWd1eUgrLrmw1CEaCj6byazrPL78RoF02kxOdqg6SolQavCMG8McaHIei5CIm1km05DFYTgEp0pmK1zm8fFdljmfLaQSoVrrMjibUiwpcauEKiHbCK6+GnCJ4LJgSG0IqxMNAsuifcVhSsMXXznJK91FlttjtkYt4r4Sjzw2t6gVbA6mCgbe27AfU4LJFVNBsSBo5ikWLcOzKb2yC8/cGttpbpiPOcy5y5z6qDJ8U5eNt0VIBpVERANh8RkouxEb0QJiPTZV1IJGwYDbbom1np3PrzLcFU5+0dG6mGMvblLB3CC/weBWFth+JMGdzKcGF4LxjaWavgem64LhNTjAytW9EiZqGWjJmfYunz8xpuh29tug3A0cumfzlYTBGYNUSnWhTToUfKyolfAsmGBUAXwMasFUAgKjE4aqJfhBzF5pGPQzfG7p9QS1Bp8AAi6FOheK+LBPHwMSvsNOIN4MGwxPG5Ld9JbZT3PDfMzhrlyBzS3a8hj2kcXpzZDuCMtf6FOsZOw+GuFTj9Q3nlrAQJJUWOsx54WFFx29p3bg8tGVk85xvOC6CeOTSmdxjAFmi6dnPWIzY4CdGnwdSfZqMIfCGqUaJlqxFI1YXRxSZp3beQo3BjGUHUOxpIiTaUxZI8HXz4HojDGNQgk2vqnwE3wEZmLQKpSnm0qCIbZhnZqwHy/B6xZPqBCMQosXqQ1/vBeeyxB3NnPD/EaHXVqEE2tMVjPEh2mULUIJ6fZbuhinnPw4FN2IrSccWKXzfBwSHi8v4A0YheEpgykXSZdaxM8aqnmM+Q2HsheRny25fzEkdp3KNEzRxJUbuNoYh5DG1foXE/VYQuKwrG26FaU4Lg3b6qTfZEkoTlREOxFxP4QtXAbREJI9DcY1kuAtm9qgJhrixD54ztFQUCOgwfiaitrqghqdhk8aI28nYErFJ7Vhr8LyYgGKJU++YOh1u+gkR8vipk5vbpiPOaTToTi9QLkQhRHbgS0UHwuDe4R0Vzj9K5dwS222vjIlyiqyzaiOnylqhJ0HI/JlwRSWqpWyeOmuTkbnuE2o2oaltV3e1A0N1T1CqRFIRXLI+JZ1AUkT4piFw0xj0BB40ABG9Ph00hSDWEPZE9qrI8aTLvGeDRWuCdiJBM5/IlQthUj2vd8oGFJjAQU7rg2vZ2qcw4Lw0vp/qLefKPFYKbqBISUOpAo6HLpQUnZSJMvA+7trmE2WgTGYhR5EM7t011CiMjNKoyL7wXwj4Dw0BG25xTvAmP3srd9PcDX609J892zMylo0z/G7fVB/LMRatD8geSmmap2gWLbYMbQ2lMmqkL9lTGmUL7xjBbwgY8ENLHsPwvBsRHY5GPFiGapMWXxW6b48hv7wbp/WXYdpt5E0Jf+yBxmvxyx+cQ85dwkdDPGTyc3t9Fbv2VtE2TY8vn6Rh9uX8czEkdUc8pyVdTueUuE80PcxxQxv2UtY2/xtRLHGo8fEMNuVJaTXRTyM+xnRyGBzqLpQLVVMNAIxmCIYUlNA4sElBEurIYHXMCzQfZU5W9S2wghqBVOEbaKxYkpAoGwJk1WhWFbSLSHb0GDgxzYY8RMrmJ0YPxrdVC7nSAyztFoQRfgTy2hak2lUwSkyc1AqclAB2pgQpJcwpTClR/Jyuu5WoFbCPiofjqF5eRBVvAhEZn+5CBoZzCBH8gLK8lgYZre3B/0+0X0r5MuWVA3prmeyavjah5/hvQvP8d0Lz/LbRcb3/sIPkOwJ1VuHmKRi+MUF4r5QLHk0VpI9j33+Irr7ezDG3BjN+reWdgvpdbn87pTBQyVqFlge5YjzcNOG2SDm7lmusg1ftfQs98abdQjC4NQciCkDZOI4bRNSici1osThqDBqp4Z4cihZGIsjMe54eMwisLxItdoNLVL7EdEw8PV9pKRLE/IoZZREJDuG9hhMocgYTB0/ltowTw1xHaaYLgcgzDhNqYEe11ds4RkvW1xbyFc9nM5xoxbRRInGgh3VIlBrbWLvQz+lm8DRhDKiCElifBLhI4MdFeAUIoOKIKVDvA83vSpSueDFzkIkLCvKWzbKQNhH44U3g0PznarBY26+x3toZbjFTn0or09W846gLpuerCY88Ph5nnv2JOmvFHRT4VefeoRP9u7l53rv4uJuj5XPQjTxXFpuUWae9acgGTheecRx9swWV544xVL3ARY/dRme+T0mM9eUoAMSxYzf9QCDMzGDByqWTvW5/J4l9u4/ycqTa3Sf3sVsbFNdulxPmS1iTSj2UQ3Sl7VO9sHv8OjNyxXfMnwsPJRcZsmMKNVQaigFKTVix7fJpOR+u8tELT8/OMNEExbMmMyUPBpfZsVUXHIJQ02mn03FsWQ821Wb83sLZDc5Zh0pxOAXWuQrac2YUKSCeBBCLcu9EXvWM7YpLk+RKsSBbal1XFmDXTqk0KgGmge/8aTFK1EdU47GHqkUl0K+BG7Bsdwb0c8ytE7Kq62LWRZi7DAN2cGbuCmOxmNOE4gsPrGoNUhRIXmJ77UgMsjEBU+4KNGyRMeTqa7rcfBKAezqCrJQx16NHM3gcEQQIwxPGP72Q7/An6u+k/hCRa/fpvdbK2iUsT1apNdXVj9yDkQYnrqHqm1Y/eQWZqfP7nct8p/d/2v8rckf5MLpBbLNFaJnnj/oRf5eQBPGylIuf3nC+C0THr3nEg/2NuEMlN7y7z76dtazJRaftMjGJlgbpFezEDfUonanygq9yjDf3evoUng03sQKbLmYAhuKRNQwrLos2RFr1nK+Un7q/HvZGrc5091jPRvw6Ml/xbpNea6KuVgtkklJJiUd41k0CRt5l8FGh97oeNwr1ULKeC2i6ihYxRaQ7XpUhDctbLOTtthptbi8t4otTTCsuWILxZaCSk2pMzUdToInrQpWgwfd0OzioSeaeOzEI05xrZh8xdNaGXP/0ha/01oOM24DGitVBuMVSzy4+fL1I/KYLZrElL0YHxvEdzCFo+rGuMRge0kYnWqHVXwIH5jSY8rgPUvlg3fbxKVn488NRILkpcjVYZE6RDH1iuvvOADVYHBtHcLwihQljCdor4PrJFjvw0NXHo8BQ6IY08pIBsqPnv9DXNxcJHmzDUT/mk85Og2TNYO39yA+LDMOdh9fwrhFRsOCn7/0bvoXe2SbFjuu3Z4ZL/L3Aky7zegPvI3hacvo4YL11T5ODedGS2S2JDGO7IE+Fzsdth5fIPu694SHzYTpbTQJnlmUK9FESbcr7KgiPr8Flds32ncLAh0TCkJmOcweppQ4i7DrE7747BnMwFI9ZPAIuVpMrTzXhC/K2qxYEbxKeMYO64DfBYgRXGKmwkJmaIkHSrxXYYuEwu03AdBYyZeEaBh+N/FMk+IeJWT3CAUpdp8OB7VhVjBVsF1VK+hk5EugKyVx5OiXGTaX/VDGMFznYkGo2pbICHo1Pfw1cSSGWeMIzWLy5YiyJbgsRRyBvF1zChuito9rfqCtq2fKEOMJ1TRhVNO6k0AwwPsXSiVU8iA1wXsGTQA/UFd0Gi+arm/sdVRnUuttkr4n3cjRxFC2I7LCQZ4fG09ekhhZXCDbdHziI49hHVx5p0ynXC4F/8iIKK2YvMszmcTEn+kSjeHyV4DrKLKT8NnN++meM6RbSrQ3uQZB6g2Ga8wGzEKPc99Z8o2PfYaoTr1/Zussl/pdlloTFtIJ3/vmj/F1X/ZkLRS/nzD7jdHD/JuNxyi8ZVzFXOl3Gb/UI72ScvK3EqJhFUJ4N/EQHhXUQFdicqqpYY4lvC98RKkRBsMr1TIrn4jItj3n2wtYo0w0wtTna2u9DN+wMQjGGy8Hnqm7CZ+E4hCbQ7JtyLYc6ZUR0TBl4mIKX2uTtyvG65bMCq3tYBus27cvXoJxnoYi6oFYXLBHxoX4tCk946WYsiPkp0ruPb1F4SybwzbRSIgHFXFfSHbDfvJVyDctNyuSejQec5NUc4qppOYC1vGcOtupwj5PkFAJ3NBSvAWSkAUFaFq8NEnB6ddIXUBxmMJSrxOtieB+xqDrPv9QfBgptdmtNgRymX531YlJ3vwgMhxTvXz+lnqcHQW0qtDxGFv4cH4R5KuKUsezuo7713ZQFV48twYTgy4pxSK4rofYo6VBgGxT6b1UIHu/91gZEkWQJiRZxXrS5/xkid0yY2PQYdRPKcuIQZrwuewMbZOTmZJYHK72vC6VC1RqcN6gKrTTgvL0iHGasbsZkW1bln53iIzzu3SC+8+KQ/G6b1gtyv/f3pvEWJal932/M9zh3TfFmHNmVXV1saqrJbZbNE0Jbqop0SZlWwBhwAvJguWVAXlhwAtvvDIM77w2vDHglQEKsA3agAWbIk3RbNBNUqSaXeyhpq7KqsrIKeY33uEMXpxz73uRXdXszMrKCDfjDwQiI/K9F/fe9+53vvP//t//S4XppHGlT8iPHb2DBtEkWHd2mkmLttlEEjJmYVcJwXnDZgLbI8jX6pCw1dsFTd+zlS145IYs6wScwGswOVQj2cUmBNhEdIEY6FzkfDCUiwkceKGQVtIMBE0Rgs+sSkm1pUgbZqnHZjIIGAxYCTbzON3a6j49ng/HHCkIXYYoqKrAxcgsBEHHSp6CDfxNt32QwT/WFKstxpmLBSstIWGL3v4OzgZoz+o62Ex0ekXh6AoAnYDcr7JoH72LAaqthJNXtykOLP1H+7jynANzVWGrCrW4BT7FFg61XaETy7Ao2eot+Peufo93FtfY//0b6CUsfnHBxmjBdJFhjArezJVidNeQfOsvMCZuuf+q0BhCIIsC3+8xLEquJyf82fEd7k9GzB/10aeKqkgo0x7frhM+GG8zTks2swWNU9ROMWsyJlUemjacZJyX/PK1H/HhfJvv6JfpfZSw8fsnmMfP1zD9pz0/oVR3jzhWHhmhzdpQyIq+rFBCMLU5w7sL1IMj1OIOjQ1jpBy2a9FWIpgctbdhbYOfsbTnuCVoISRNIalHnnQSaApTCKa3U8xOw8vFISd1j+UyxZvQom0GML8uIiUlVjEIzlIXxF05IQlCQm3CTqEeRkMkD5Npwe7WhGv9Cff7V2j6IW6pMkjybM9js2e/v54PlZGn+CzB5AKTBQu89g0V6ycfM+FWyB2evPq9FzHF9WvPi58D+cTju+eptd+dOaj4Gk+s8GL9tVll0uHfHi/Dm9wUEtEvEN7jq3PKgtYgjUOVQdJnlhqz0JQHPQ56I6TwLJqUesPTDMA5wbwMRkVKObyLxQ4tgpm4taFS/FeEY5aDAYtvvM70lmYrv8eBGfJoNuR0UoAPnWA+d8ieIUsMWjocgrlJcT7yq4CWDlzwlXBesHQpx1VBcqDJjgmKonMoAAqlEFmGf6LSZL2M1IQlFZa+CM0OFhlqMdaG83fh/NapjPb5LeYmRS8Esr4YKbNNVnSmMFF7rQHlSYTFOIlpFFjRJXnSRPq00ykTk762weSJ2EPYhbSvH35o+yHCb7QMTo9NoWItItADPokZ8zPiuQRms1HgMkW1IYNVnpGBK45tjiErbk1Eoi6w7bKJr9H6O6wLvYUDaVeEPYTtRTARCd9FG/DhLHXxRKIrnF9RGvE1u983DqRAAbVQ1KNodLK9idQau3947pSGqBrS01DsEDYhPRXsvNVgc83dN1+m3nCM3jhCK8f+vQ3Kk5Rkd0mWNTgrcRJMT9Lb3IDjkyB8v6h4HmqRdW55d5vD/2TOr915m5nN+MH0Ogf3NkgPFc2Gw20YhltztvsLerqh0DW108yaDCl895XrhspqjJM0VvHxfJNPHm9x7c8cxYMKv1x+zhN/NogsQ26McWmY3YeH2qvgq0wwyd+WCwppkE+wnsKBtS3t4ciFYSiXzF1G43VHmT+eDug9EiSTcy5wAkiB7QnM0MChRpexdpSAiIG5shq/0AgbOF9hIZkG9UYyjwlYHv7PRrPFthVbNcRkL9S7ZAzMzSDSJxFKeLRwiMJQbmuS2AYuHPiexabP7lz9fDjm2CHjWvJccWa16LLU9ftsrcWxoxX8isLouB4hWNfHOyVWGbZ8YtVbozxaIr9d/bwUeO9X1EfLSScCM0g6uQxCkE4gmQOJRqQp4hkrq88VLSXT6SvXdh7xnBurME6iTzR6KTCzPovE47YbZGopNxKK61vIuoGLHJg/L4RApCly0Kd58yWObmfsDB6gpePRbMRRWYD0mJ7H5xaVG/LEkEqLFB7nBbVVlCZBCI+WDknofJPCI9aCtdKWaijR84SeOh8LeZFn+EER3NA+BWEySYMEPjZLPq62EbZVRoExitorwIS26/Z5azeesZK89Ehz3jdCQHsfyyZ05NUjgc0EUnkar5hVGXqi8NJ3O4nQyedDghjlbV0cEiF5FHEzIdZikYsPckls524EbqFZ1LHIGItWbQIprQD7+QqlzyUwu0QGHjNa7ZnILavah4zUhm1SCLQxOMpVkAlkfFjpu+1JZ6+3unJdJu19F4id5uzF1WfpknV1hpMCvEd4EbccnqaQLDdVF+yTuWf3rSWicbhegnS9ZybwvwiE3UC4vvVIUfdDB5LLHbOjArFU7PwQiv2G4t1D8J53/8lVhq+dcPpaTtMfca02sL9/8WiMbgYhfOZK+FnHvJYhC52gtjapXr9B/V8e80s7n3B/OebPj27x3t4V/EKTjCvSHUOWNCTK0U9rEhV2RbXTnFY5k0WOlCEAF1nNdm/RBS4lHRvpguubEz75WkG1lXDrX+Ywf/GFVTEasnxpg2YUjt9632XL1ksSabilM/ZtxW9OvsYfPHyVzarBW4eeC6pJGiearCi71tyo/eRXZcrGiUcu6vMUnvwYkqmneFRTjTPKHY9ODMem4PBowMYH0Iwk5U54z2wOLhWYnuh2zkCodYnwfwBSE5JEvRIVeEEYFFB49EyiDhWnvYLTfg9fx6Ji7UnnjmohEaX8XLTP8yn+GRe32EGJ0QVCDcKLIJGL5iFeregNafxZrhlWhbz11Yw13keCZ011IVcX7+xBseKiW1WIjJXW9cDe+rbGLF+XHrk0IJVSOYQAACAASURBVAXNOEMlCpUmnx0ovmCILEMOBzSbPZpBOKcwKgqqmCWoEnASnwmECcMiq5Ei2x4gjAvbNifDOaaEgZqyTSMu0m0W8bRB+QnI0YDF125z+rLma+MDbmbHvDu5wnHZw5sQanRi6aUNSjpEzJKNCy3Mxklqo3BOIqVFq9X4pRbh8QolHW5oaAbpuS3gXitsKkHFRUOIWMpTnX5ZIll4wR8fv8LjgxGbTZgMreqQATb+bCiQOFalP7ClIp1YxPLZTHmeF4QOu9gna0ouFdjck2uHcQpvondGy0B6sdpptgme5ceTOx+pTh9267S77rVYoiqBXkLlBJk2iNTh0lYKHOXB7fG2NZ2npEKfS2BOjpe4XkI6SzC2NQfxNIMQlE0vnnyEjC5pzoiVQiIGy04y1y5mkY8WFljjhdqKaQez4paFjavdWotlx0MnobgX3oigg9TzQNQ3w3DRhXWYfsbhVzOSWcqVB9v42flIzOTtG8z+2i6TO5rFazXqMGHnu56mLzh5I2xFx++Gbdr0pWB7ePzzwdPg8S/F4tbAMjvpkUXPRpcq9GiAW5YXorDZwcc75kluuc2k24AdPSk+tS1aKuxrt5D/xSP+zuY9pPC8O7/G+492aBYpQjnEwDLoVQyzinmdUjUa6ySl1CyqlMas7qx+VnOlP3viMAPVcViGFv5rN455WG8j0nMaupRomr6EdMX/psLivKShzXstb9e7vPWHrzG+C5zOwFmSmUdPZTcDsIVqx0/Fn/VhQvEXH+Em0xd0Up8CIZDjEWI46HbWTofGj2oM7krFIK+Y2xRqGXbsNtAXwgrSadi9t7SqtOG+t2lI3GQTgnI2CZNJTHZWXNAMQuxJTyA79czfFHxpcMC9rTEnM42XCtlITEGgTFIRjN2Uwk2f7ro9P9tPIcIYlyxUboUP2WjbTNJ11bDKXsUTWa3A49ulS4Tgvi6Pc23Gpzi7KnlAnb2fOxeslndeX2HFE9+h63FvhoLyWkFThAssjMD3suDvcR5IE6qRwsSig9chKJt8rUIcr6E0Am88rhGgwGUunKMR0KiOt79wFEaL9riedP1bRxu815pdhdbIQT+0TG9vcPJywd8Yv8PL+SFvzW6xXw6wJk47TnwYHhCzZCE8Sq5xxtJhpURKhxKeXBty1XQZcss1C6B2Cik8o7TiYeZAq/O5tj/l35y6HumxoHe0cnEMTV6i88aA1bipFo7Q8einM3x9vhkzWuO1CvEiytha43qVOFJl0THQtLEjeDIHb2Yh4u8Dk3m2TiUB32a8vvNw7uJWjGUQkz8bPhMQArFXflUjMzETTJKgoX/a03y2q3MWZpRjhgnTlwSm7xh8JEnmK7Lci7VCFaw1fNApJFYqCt9xy60ptUtCi6PNodyOY5NqAS6qFPxaII5QVbx4kjNqjyCXWR1LaIQBU0B5u6Z6xTH/1z12ockeSFQlaLb7qOIzKitfMOwgY35d4FJIHiaYvmfyzSVmmjB8NwEP81uEQkgNeiHYfksgLZy8qjCFRy/CtbQZNMPIp82XeHNOFfZWnhSLZb6VK647s8UM2TsfMuX1zNj7M52ZaneH+ddvM3lJU/7qlK3BYxov+VfTO/zpg9tUlQ4LV2ZJMoPWjqrRNFYyzGrGWYmWLtzQvVDsKXRNT4XrY/3qBuxpEMLTOMWiSSiShtdG++xtjPH9Hmo4/CKv3KfDWFTjwYp4vJ46di6u48gMGH/oGP5oil8uEWlKOnMkU8XcZbFFOaCzCI3vlWwEdjr97AXzBUGkKS7Pwk53JuM8vhAzsrxhI19yIz8B5dGlw0sJmzVVohFWh3tkCZ2MS8S6llgle6YfMjmT042OEg7KKxaGBnsU5pOImeY7hzeZzHpB490IpPHoJWRHCr30+FE/WEWcnDzVtXs+OmYtcUngeGzhsJlENnGF+Wn+gl+tSl32LNu26kBxmF4k7wuHFx5hVWAy1qPyGR5w9aE8w0GvcdUQuCmXgik86aAmzxquDafsnY6xn4xXFoDnmGW2DTKqEtjCMxouOHF9ZJ0QWt19PJ9wjKYnoqF++F3rO2uKoNk1fU0yHn4+7+EvEn8ZVysVMk2CsdBggLmxxfS2Zn4T/vrVh4yTkmmTM2lyqkpjGxVeUsRsWbqQLcfPzUoS54ICQ3h6qqGnGhoXdJxu7bPlo7bZexEfW5Nqg0/U+dEZnjPZyYpbXk0uabwimVnktMRFu1tVBve0yiW4ttMv3owKh/WextuYWJ2zhllEnxstabuIu2xWeRJlkYRFk/VEsBUcdK8Tv7fc8RO76k6+vb6jluFvSBX+oIiKlkWV4mLdogtDDkS857yWCP30ap3nEphtT2FyiR045LCh3FU0QxGGgkqQtei6aiC8v+s6Y+Gi+NsE9yenxJqkRdAMBIuXDCK36MxgjYS5CoYyC4EwdJxx27INdHK59vWFCauj6YHLgnm8HTrkZsVosOTNnUdczSb84uBD/if1Nzl8b0jxqCF5PEWck6mR3p+w/YOcckOxuBb9AaRHSH9m0bOpx94q6Q9Kfv3l7+G84Ld+6xsMP/YsdwSmD/WOhb7h8S9k9K//HNvfOYHv/vDFn1Q7tOAJP5Kftg6pr+zQvHKN09cK9n/JITdq3rh5Fy0cpU04LPvsHY5D8U54VGJJEouUQfomhEdHOqOyisr2giZVOgZpxSCpmDYZx1WBlsGH2BGCcGk1B4sCKUArSyItharJEoMZjUjmo+d9tZ4alqBjVsIxkiVDWQMJp7ZHb2+G/3ivoyR6D+a4dMCjZkTpuxuJkahIhGPfae7buMu8APBFjhmkHaXgNNhUYgrP9dGE0mr+1fFt5EKFQGrATRPSY0X/nu9MikT0YbZJMLxHx7jkIDsJ1E0ntY3UaT1WmNjdp+owJWU2z0Pbd+pxmcfmq45jL0Kh3Seap7X/fI4cc1hRlPK4xK8V4OK/3dkMdqVDFkFrKDib+MbVrG1MEYVBp4YksYDGxVlcXqy46nUPjY43IryWl8FYzmaEzD5m93LUcHVrwma+5Go2YaRLSp+wNAm6CuYlYQDq+Urm2msifNCU+uhDEkbdBE1mkjfsDuZ8Y/AutVf8b/4bYeqCEcGWWnpU4mhGnmUjsP30fHzPYzFPpknwOF73xfY+VrHbu8IFjk4pRKLDQIZr2yxu5sxuCrZfOma3P+Or4wecND2+f3SdeZXSLNJwvXphIG2qLbrNdsRKk2ydDJ19savPxK/SJjRWkWnR8dHOB/mZcxKkI5GOVFoyYciUxSUSziNj9mcHUqwjF2HLd+pKjpsCsayx7S7JeeSsQi8KljYJWTQChSMR4WvuUo7s4ExidZ7wUuK1PJPdtpPhc9UwbXJmdRrEBbEZTbjWv4dY+2K1M2856nUWza0eS2SIRKvOWM/ErcCaeC8q3wXwdTuJtjfiafFcArOeNqGhxAeD+WQaOOblFYfr+RWPEyEyontT2GZLG7JeYWNmvRa4nQ7jYrwV4YbAIpWj6TtcFrJphMdnLmzn7SpgAYiBibxieF4/bch1+JQ5LyiSmo1syVBX9FTD//3wdQ7+4FeCjOhlmN7skU7yM00uLxLzN67w0W+AqD3JSdiBHO+N0VOFXnjSmefKn1dUmwmfXMm57wX/6Qf/GLlQbO2FHcjwExs+IGiqbYlTgas3heaFhxEhUMMhol8w/9duU22ouMsRqNojjSc/atDTOgxYaCz11QHLnYTZDcX8lsduGDavnjBKDIO05mDR53/++G/gl5r0MHhGiIHDpx45cKSp4fpoQqFrpnWO8ZJMhWaKxirM2p00qzOOlkXXSJJZjUkkS5MwqzKE8ORpwyCtudM/Zieb8WZvj+/1b3C/t4vrnSeVEf7Zbpxz0TCWFfftkP/x8Ov89o++wper49VTrIWTCdlhj6O6z4lzgCQXJgRmPG9X1/n+8hb6AvheCSnwiQoBt61biRj8ZGiPnlYZhycDhBMst6DeALFRU8mEWaVXHb8+BGynwu55nYYot0JNphUuuCQW3ccWcocpoCniyKmjFJ87RG6xqQ+KL1ijSM4xMEsTTItEzORkDbIiBGrpggROPcH/ylDtRYSuOhFF3V1gXg/OSeCRvRWdzyoqOKyROpAeVRikdNhG4a3s6LBiUDEulgySmn5SkSsTuEMvqZ0mleHnLKYEh7OC7R9aTCY4eV0Gq9GeODc7x2Yg2bh6ymyeY+rQTisruaJvPOiTMnwAak1dabIHCckUVBUOWi9deFwpMXXs0NQ+BusXC6EUYjzCj/rMr2vKLRGVIiHzl03IdNKeQtYOaTzzaymLq5L5bUf/1VN2BnO+PNpn2uQ8WIyYlyniMEWXwXbR6baAEzjlRFkGSUU/tlorJ7tuPifFmWxz4RMao4IqQ3pMLPTVVlEZRaIsuXb0dMNGsmBLz9mQC3qqCZ/z8xiw4FxomFrblao42y8VjtInvD/dpZ5kwfO8hXdQVYjaUDtFE9NQhe88macuBG1pPj0jf+GIbpAAnRxpDbVR2FqhAJsLbBa6M12isb0gmztj19DGF6CdjN12LbvMrwKz8qAjhdju4h3IKtSogrKAs/w18ednGDf2fDjmQuOVIDtQmEVOsgj8TTIVSKOiNrjlavzKVS5pT953vMynmQ55CWKu8FKxnCd0/p7dgwJV4pGxyOOQMgjvvA8E/bJOOJIFxoat6qhXstuboxNHJg1Tk/P+dJfFoz53Ppgye2WI/Oun5GnD6aSPs+dT/MuPDA9/tIEbG3ZfPwQClbGsUo42eohGcvD1jXAZKo+3KfXLJZUTmH5GMoV0IpEGmn74EPYeSbIjyB4vX7iLo9zc5ON/cIflNYfdCK3ibp4gGtG999MvCYTR4bOReHxuULkl79UM8or7x2M+eP8acinRM4FPwPcdTe5odjwis2xszumlDZv5klQGyZv1Ahn/yNIkgZpwIZAWSU2uGrR0mEx2BcG26SSRjn5WM0hrrhen9FQTqBAvmfuUuU1JZgZ5+uJb3f2jA/plTfq37gChwaQv6o7G+EwIichzbC8hVw258DTe4wg2oLKzD71A8srYINIaEoWdtkdWkv3lgPkywy/Drqkeh5hjphlyqkgnIhbRw0t5GejWZrDWnu0gWYQdvVsG1YaNlsS1B5dLvA4qMeEhmQgQCqM8yUyQnoTXtEG4gc01Kj0nuZxTgfPRC8CtpsqGC9DSFbETMGqbUT7qAgMn7VPf8cEdfKA3cAJZhQ+H77wyfNf1B5HqOPPcQHEYE6YZhK/gC+CMRAjPZh5MZzJpOLCaR7MBeqoQ8xLhB7y0dcxWtuBusoVx58Mxq9KSnmSUfcHL4yOMk5zWPVJtg3sWoYvNNArxOIdG0B8vSZTl5CSJCoeYicYMQC0hO3XIsn6hhvkiSRHDPrMvGXZun5Bqg/OCAzXE1gqh3KouIKAoKvpZHaRscpXp1aUmf6iRFaTTYMfYbHhIHfmwIk8bbowm5KphmFQhwCA6b+V1zriFlkEDmyqLi2oLgNKEeoOSjgzo65qtdBF66uJjGq+prUI0LkzEecFwsxl+uUQvXqLxoZ0kEYZkLTXUwq1ZNAYIKYIuWEmU8Dz5CXfd9wsUmFu0HHEnuxWURmONQtSBELZZjBONQNVxJ9+AqmLxWYJzqwy61SyLKEQQVnSFQqeDiCF48HhsGuW8dZSE2yCX06XvBoJAa7b29LHjuQRmVYVOMz2nG/Ot6iDD8aLdCqyKfesawmAswso7ua35PClxW98aPEHWB6y4vXXVXFQ7he27AsYePzaYIgS1jWTJV4s9/uXBSzT/YodBBY+/eZXlFUG9LJjVGZMyi/z2C4YQ4OJ0l6Xi/mzM/UcbbP5hWI6LfljkbA6i8MiXlhRFxatbBwAcDcbUHsrbBpla/CRFloLFdU+1rejtD1EvUJRx+B/9AvVQsHnzkJfGR11gG6Q1lQnBzTpJFduhnRNMFjnOrSgsITzeC6pdi1eehXaowvDSzknQF1u14o5jYU/HIp0Uju1sjhSeRFq0sLxePGRXT7jfbHJqCvbrIUd1EegtZViqhFwnXabd1zWZNFROM6l7GC05TAaUNsFlCl8868yKz4FYMFUVvNNIcuGjEgPmXrMt5/yH1/+I0zr/8eJkOyU+ovGS0msSHyJe6TWV1edWY3kSPlPYTGJ7wfPYpbHIp0IzkGskyWzN013QDc4IXhmR5ooxxWswvUhDuLBYmSImhJ2Hxopvbu0d1iV1XgOpwyU+qkREFBmE8VJJTyOf0gjtuRnlSxOKN16G76oGVVuEhXogu5FQXvhu/NPqBUKX4Lo38hkN4Zpz3JOeGG1f+8q0KDYrtPrGuBjYTGATQAiWfRGtDgWZbLimT5lWKVvvNFRjxdFXBM3YIquUpfCUZYJ/0YE5mp/jowlULZiWGfIgZffPJiAl81sFNhPUA0G1KdBvltwcn7KbzzBOIRKHTyWj7Tlb/QV3610oNWbkaLSnGehnHhb5LDj5ObA9x2v9OeNkpZ9OpcV4yaTOqaxm0STURlHWCaZRWCuDx4Xwq8/FoCHJDFujBeOs5NXRAZXVPCqHwZYzdumZOI4ilcHvYpiUZNIwVCWZbPiF/C639JLvyoq7YpeZzZCih5aWnqyReCQeLS2JcGhpO13w0iZI4Vi4LKg7Ehl8SM4DUVf70I7YlnPGssIhKL2ikA1/M9/jd0YH7KmrP/FlbKQxLALpwyBX4+WPUYznBa/kajydXgVfH4t/ocYVzPHPdJlHhRdtF3KU7HpFoMKINToR2rTFWmC32Wp3v/560Sk1ZOWx669taGttSF3ybIqu52ZihJarlSQGSKcEqJYTWp1UN5evXbWilC1crBhYz2TJK9PpM2bWrUCctQx6LYAL54OURYSth9eQ73uyQ838Vp93raSympvZCdN5zrhyLHcS/vavfZePZ5vc/aPbJFPB5v2gFuAfPI+r9dNBDYeIrQ3mWxk2DzrS2Ydj9EKw/wsjdAn5kcXLUCA0PVhMe8wXGT84uIOeS66+5clOLOXWmGlvA/0amJ0GnRt0YmkGffKf1P78nPHyPytxiWTyR7f4bnGb2UvQDD3JzTmjfkmuDUo6tnsL8sjhtjxv7XRnNATgvUBJ1/HCM5PivKTQIVNMpSWThuv5aTcmKhGWq8kpiTC8W15nr9rkP7v7DzmeFGyN5+wUcwpdkyvDadPjuC7OUB7OS7S05CoUyyZ1zkIn3FXbVFYzvaOpxuenY07mnt85/Wu8WdznV4t3qWP2q2gYS8VIL9l7UiGgFT6Rke6B0mumrsdQ1qTC8VG5zftHO2TLixKZg2NlMgOEID+y5Ps16WmPx9MBMrGUV01o1451IdHE4r3kLMfcZiWtZK4NqunZBLAtGIbAG4K08/6MX4+YJCQzQbIIgUmlgT5s7SeeFs9t5l/nX7qe6cbx4K20Zd08pMtwfQieXvkuYw4dPX5FYUi6vUPrSNcG8JZzDvO11gK0X71+R3/IYGSdnYYq7XSc8Sgfsre5gak00nrqEfxX13+b35x8jd+8d5vBA8PgB4eI6gUr7Hs5bmNAM1DYLGiR88eB9pndCSN1egfBzc/moajn5iF4jd9V9A4dm3/yCB4f0i96UPSYX7+BuFMzKCqyxGDTQRhJZO0LCc7yW99BAuPobVH8yuvMryqO04JTL/D9kl7SMEgqdtI5G8mCoSqDdhhB6RKqGKAtEuNU9/PShvQolaHpY6hLBqrilWyfXDaRE3bc1McoPD/0NzmoBhy/vcXgI8nB6xmzaxlXx1Nu9k87bhmCxad1IXOUwlMqQ2PDUFbrJUdNH+Mk1Zb4XOOEPi9U5Xl3coWxXiKL8LvGaxANhUhDi/mTgVkpvFzj1H3SGRpJ4KgumEwLrl2QySUAQbsfdufJzKGnFarssSwTpPIwanBLDUvZBVUB3azPlmINHDSrBE9GZkf71e3gV7YPtpXntRlxGvw0gid0cHlUVUgiVU2YfNJK5p4Sz6f4l2magWZ5VdCMHOWOQFgZRva0W40zxxay2LBChaDstUcYgaxju2U7yHXd+Gj17BWeyJy7xzyh8BDRuLrcEeAkyzsNL3/pMZ/sb/J//F+/hOh5PviPG165dY/Kw0Ez7Pw27FYf0bxg7tAGDS8EnSW1Ry0FuoTiETjtOXwz6XxA9ELgDzVeweR1y8TD/Pp1kuk10mk0gxJgDnocFykytWymIF97BfaPsAeHL+zUvLW4+ZLB9w8oPukxvFfQ9AtsUlApwfvZNd5JBM0Imr6n3rYkmxX9ouLKYEamDaNISWwkC7R0jNWSymv26yGpNLya77NwKb/16OscLPo8uruFnqmgFGogmYWb6PbHDelpTT0eUG8mvDw84je2v8Nes8njZkQmDLlssAgapylUxYZa8KgZ893JLbbSBb+28Rf8rv4q/08VrvV5IZs43tm7SqFr+lsCXJDKTbzj2M2pnMZs9dE729ijYPtJ3SArS+UU1hN3FgYVM+j3jnaQH+dkxxek9S92A9us9c5ReDWg2vRc2Zjx8GAM+xm6WesIFkHWpsuotlCRW+6H+8hlcZfeiE4C1zWkuFXCGaS8sYhoYvOK8l2wtlnQN5tcxIAdRro9S+30OQVmhc0l1bZFbNU0Lrh2pXlo7BDCRzmf79phgx7U0NNNkOoow6zJOK3zzovAekFtwiHaKI9TsbKsot9BY9SZ7WarvmiVGC3K0wwx16ERJXXcvnnIv3v9e/z3936Fl3+3Yu9v5/zXf/9/4Zo+pfQqaDfjiJlmkCD8C24c8B6MjcWH6HPtQC88g/uG5bbm5M0ws63YU6ga5HEoXAy/csJ2f8FHu5tMFynp/QQ9D9ciOQ5aZp+qMHH7+oi8auAFBma8xzc19r0PAMjiV4fIr6vbNzG7I07eGDD5Up/T3RxzQ9LPalwh2EiXZNKwmcz5UrrP1OXMTEZPNdxOD9lrNnn34S7mUcHNb3mK+wv0D+5iT07PHo9U5F//N1hWipv5Cb/WO+LDZJ8fNduxpbnEEvjWoazZVY73mh4HzYCb2TH/Vu+EI/sR36p/MW5lzwd6bhEPM/a2x2RCYoWl9grpEk5c8MpoRgl6Y4Q4neKtxTcNsnGdSZPCkUStjkVwOukzeCDQ0wsQmMWKLnVJkLFVI4nTAjO2XC2mPLAbZMdyRZPqsJuUJvZWQKdDtj0fHSuD/FZHNVmrItPzsHNvBzvLJnQpSxP9ZzK65C8kl6IrMHYdgM+4gXougTm7e0jay7gpN2mKrONnbJL/mCGID+o3SglLCUdqVUGVNqT/62qMNmNulT5trDVRW67XzY94Qsmxhs0yKEXqgaLpKx7dvsb/cDRC76dMbweC//88+nnmJuXDk20ODwcMdwT1UCOtevHFj60N5q9tUQ8E6YmgGTjmX62ZzzT1KFkbRLkyALdp+FDV/+82D/w2iQxOlNWWo9lw9D9WpCdQ7kiagafYd+R3D/HHpz/5WF40osrAn0zQjWHTOopHPZqBohqNMRr20it8ouG76arQ0kk0Jfyv+S8jG9i850lnjsF7p8jpHLv8FNMm79j58wX5cY///Ye/zD+98o2w0NWiM68Jj4vFIh04//Q0ZEb/3dbfIz2R3PrOHDU7P3/rZFJR3M/YvzGk8S4MjY0EZy48Y72kHimycYFQEm8a/GyOnJUcVQWPbA+LJBXBDEjhsUtFduKRZXMxJpfEeKGXYRJRfuzIJpbkSPOjox38XHce70BXOJdNyHhd1Bh7HTfaPnLQ0PnsmH6QytkkKjraIOtDExStUiPzuMIh6lBw7OaUGrpjcFqEVv2nHKLwfGw/P/wIgOIHz+PVvlgMr16BnU1mXx5zfNLHS5jdBJc6/vTBbWbHBYMfpvRFtBg9J4sMu9ln8rIOvPgJVJvwzTfe5d58g/eTa2GmmBFh9W79ZXUITDd+f4o6WbB4dYtqU/HwlqN/ZY5+e8zoowZIwAt6jyrMB3df3El9Gtf2E3yX7fExHB/DvT1SIAX6z/inHT+hedN7xLe/y/Db8JeadrZqmSdfIpr2n2fwkpMlg70Bs1ez2CgSJ10LyIVgoErqgcQMM5Ko+HHzOXq64LTKeWyHjGRJLppuYosoFdnp+U8uaRG8dcIEEVV78iNDelyRHw6ZHPbRM7VWvyJYA1s6bbLTIsrfVhSDbGJ9qu2tSAgv0E5jiq+lZyHA214wDXO5R+QW7xTUYjWoI2bVwsek4Rk6bM9J23N+8LM5Ugj6SqCXg/BGi6BsWHywwe7CM7hfh5WzUM+8Ffm8UNOS/sMimjIJqs2wQlRGo2ahs8kNLSbxVEsdhfNhZf/gPxjgdJ90Elq31QwW9YB8A47eTGn6YYcwfSlnbH4efe8Qc2/viz+p87aNfB5oTZYuIMSipHhUoycZ07i1VMJ1Q1UTYWmGgmaoSZO1W18ItnsLXk0Ombr0jI9zGGYqzqfV/FPQ9j3YHGwiqEcKyGgGkA5qmrnCqfUZnpCe+m73LawPPPXaEI9mEOnRpVzZ5frV7rtzmourrilCtowRMNVxrJ5YU3eIzlDtWemMv3KB2c3nuPkcHj4ieWv1+wwYPPHY87HGDxDHEwYfro5gdn0MQG0V6YnE5h53rYljjgRqLsk+EdRD+G9+45/yzd4n/OP3/iEfPNghfbdHOoHZy47yyw1+oZCVZPKKpNwacEVJxIsIzD8ruKALjJ9MyT7SZEc3OXEpKS76MXus9+SioR5BuaHor2f9SnKzOOSNJOP7Tc2JWyt0y6h4ega/hy8CIaCGgao2h6oOHHM9dlwZzXk4yaLRUZCzJfNAd7Q6Zq9E9CkPMHFAqxeQnobkJp34znwf2qHS0BQSm0G5A7ZwqLlElfKMRLcTOyjw7jIw/8zBj4fMXh6sGmR68Cd7d1g86nPtR46mLzgucmQj2PggcFo2DXTGQzPmneaUH+3tknySkU4jvzYwbG1Pmd7fZvgR1COotqAZ6HNdhC7xfODrGqYz9BKO7IChdHbkCAAACGJJREFUXAa+WAS+OZcNzdBTDwX8JeOObKst81yshcj54I3RhPui81f2YJ0kOdRsvONYXJUsbnhMAfNrUfsuYradserqazv/gMYLTDTRkkZ0Q1rbGaamF33iE4+oJcUDSf+BY3ZLsrjhsHUYr+fXma7zlMtd4vmjuTbk6A2FT1p3PY/43pjte56tP/gYtzPG5Bskc8+V3/4IEs3DX7+JTeHdxTWOTZ/iBzkb71uaXmhAKTaX/K1rH/GHv73DtX++x4O/d4PZq5Z6pC4D888AXFlCWZJMPXfrHW4kx/RlRRpVFoWsMDsN1Sz9iXPoXNSNt7JV4eCsEc35QdqgsFDVquehNZlrjGLwEWz+83fR33yN2R1BveE7w6HQ9Aau50JBty3qxkYUMwzna2aB0rBZHPQR5/e5NDxPNgJVCrbebii+9Q723/8qs680NCZBL852Sf7YvNGfEpeB+YJCVpZkFrSWYeKKxxUGU2icvhO2bxvBGWv/334JPNgsiNy/de9LJMqSHYfRQeU49O1Xd4f8s6OfZ2fuMbsjVAX5Q006uRiFnUs8H0jrOTIDNtSCbTUjF8HSNhWWpGgwRfKZnHFo3lnvEmsbwy5GYG455m7QhgomS8JB1WiKGvyypLi/ZPMH/TM0Q6uqcUmo0fjWarimG2EnfPCmCcV0sepkjgoLL2T0DYf84QJXVajaQ6VCAHdt1/HZprqnxWVgvqBQp0sG93vMryvKXY/Yrfg3X/mQrXTO7X/niLdn1/m9772BSB1f+/sfclj2+eT37pBOwX57g8bB9r0GPbeYL2nqMdz4lqPYK1le63H6c32SuWfnLUfv3vSFusxd4ouFNHC/2mCsF7yRPiQTQS2Si4bdzSn3T7Ogo/wUWASN153tZ9dle0ECs0skNnplBJuG2EXsoFykjJcet1jAn3yfnT8LMxhFLw9yNSlC5t+6CQkZlDSzeaBIVPCGEFm2eiyEOYNxODDe45clvjFhmLH3JAuHnijUMiqk2gkobffyM+AyMF9UWNcVHXzi8UbwztEVjJUsyq9gjQITBPff379G3Whs3+MTQdMPn4Yjl6CqhOWVMEprcltTD/ostyWmD719T37sn4kDu8TFRTJ3/MnjO1Q7ml8u3iMVDuvDYNYiaSBxZ6WL3rNfDvjYLCh9gWyHtyIRjUBVDmEuyNLtV6ZlopWJxq48U6rVCCxn8c52wRQRlSVxfFkH5/BV0J57I0BIhLUIIfDxcUKp8HzvwTlc3XBmant7aCp48sgGiO6vl8W/nzGIuiGZhfmGvm8Qs4TZhzv09zxf/t371He2+OA3guuK//4mKoHyyw0Man7u2j6DpGJhUhqreDgdhsD9qmUpPKm2ZMJz8t0dZC3x2Yv0mLvEF4I20HpP78GC/T/a5V98pc9/fvV3yQRUPkjndvMZd3vbZx3PjOXD421+b/fLXEtO6IuaxkssAr0UpCcVlOfXOLOOliuWRgSTskhrqKVAH2n04mzA9Mb82NDfz4T34C2+sk/awv/kp0kRdNFpyOD1QqCXMeFRq4koT4PLwHxB4dMEU6iw4kb/EJt7mkJgt4fYTJEdhWkKNg+V5XyjZNCrOC57HC0LpsuMplE0JzmiEbgrS8aDJcZKrFO41FNtSmyRPosB1iUuCuJw23aLLhc1vcee5fVQ0lVAImAoS14uDnl7cOUsxywEedqwqydsyEU3889Gy1RRWby7AH1/UoSJ2L1Vt3CngpCroP2iIXwYqwfhmJwKxxWOmXh8TxecLwPzBYUbF0xvKWwOybHGDBy8tGCynVKPR6QT2P1zQ7mpePx3GoZbc/7u7fdY2oTf+fbXyA8k/T1PMvekE4vwnk9+tc/sFcdymkGpYOCYvOEY7KU/puG+xP+PIGR0CQS8hceH7P5pQrUxpvHROF9IhqLhH23+MQuX8k7+2ur5WvHG5mN+vTil9AbnPSfO0XgZBibPSziHySxPQghBuaVYXBWdwVAz8GFQROzu+8z09qftOv1JtN5n8OyyCf7z7agqm/u1HQyY/DJj/pmBWDZkp6H6ixOopaIiD/MTY6VXNg5VS6gli3nOH+y9StVo8gNJehLc01TjSU9qZGno3x8zlwOSOjpvRSSzCzKb/hLPBu9gbeSBr2rk6YL0dMRb1U3m/jFDWdN4zUM74mE5ArsWZKzjwWLED+tgWmwRHNo+R3aAXoKoatxPSwd8gfDeo5eOZKY6bTHEOX51UFcki895nN7/5OD8KdALS3YU/GuC2RGoNUsW/Qxe1peB+YLCv/MBmx/niNEQtz0K3UytvequjtMXBNJ4Ru9oZKW58sc1clFT3agwhWJ+VdH0BcO3l/iP9rh5v49IU5pb29QbKb37M+SjI9x0djEMai7xzAie2uFddLMZYrlk4/0t/tsf/Do7gznX+hNKk7A3HXOwN+bNxX73nvuy4r23XuWflP+IK/0Zha45LgsmdcboI4fde3Ah2tB9Yxh/74jBR3lUPHjMMMPmimTaoOY18uEhnxqan0ZV8pQKlOy9R9yYbmKGKfVYo2eW9LDEJxLX0+jjJa55ugXjMjBfUPimxjY1CpCJxmuF1ApcQbWhwwcz6jNDpuCR9/bxiyVqo4iOVpHfsqHy3Faf9aCH1xJ5NMU8enxhpFCXeEb4NSOH+LM3BlU5yjLhVOXkuqGymnmZIqqoTuge71ClYLrMyLXBOMmkzphXKePG//TFsxcAsayQSiGiukIpGbTHswoxL/EveqAFcYcyr1BSoDKFXphA/yQ68N7l09NAwl/elJe4xCUucaFwWYy/xCUucYkLhsvAfIlLXOISFwyXgfkSl7jEJS4YLgPzJS5xiUtcMFwG5ktc4hKXuGC4DMyXuMQlLnHB8P8B2jR9LK2kPswAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 10 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3>Plotting images via wandb</h3>"
      ],
      "metadata": {
        "id": "nRU6TlqpVSFz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Forcing Relogin to ensure that you are logging into your account alone,\n",
        "!wandb login --relogin\n",
        "# Find your API Key and paste it in the run box\n",
        "\n",
        "# You can change the entity name and project name \n",
        "entity_name=\"safi-vamsi-cs6910\"\n",
        "\n",
        "project_name=\"Assignment 1\""
      ],
      "metadata": {
        "id": "AdP0H8En8pB5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_images_wandb(entity_name,project_name):\n",
        "  # Here we are implementing the same above logic but plotting all the images on\n",
        "  # Wandb.\n",
        "\n",
        "  # Enter the entity and project details from wandb.ai\n",
        "  wandb.init(entity=entity_name,project=project_name, name=\"log_images\")\n",
        "\n",
        "  # Loading dataset\n",
        "  xtrain, xval, xtest, ytrain, yval, ytest, labels = prepare_data()\n",
        "\n",
        "  # Creating training dataset\n",
        "  train = np.asarray(list(zip(xtrain,ytrain)))\n",
        "\n",
        "\n",
        "  sample_images=[]\n",
        "  wandb_arr=[]\n",
        "  i=1\n",
        "  plt.suptitle(\"Plotting image of each class from Fashion MNIST Dataset\")\n",
        "  while(len(sample_images)!=10):\n",
        "    n = random.randrange(0,len(train))\n",
        "    lab_index = np.asarray(np.nonzero(train[n][1]))[0][0]\n",
        "    if(lab_index not in sample_images):\n",
        "      sample_images.append(lab_index)\n",
        "      wandb_arr.append(wandb.Image(train[n][0].reshape((28,28)), caption=labels[lab_index]))\n",
        "      i = i+1\n",
        "  wandb.log({\"images\":wandb_arr})\n",
        "  wandb.finish()\n"
      ],
      "metadata": {
        "id": "-9BOTbyz3rpR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plot_images_wandb(entity_name,project_name)"
      ],
      "metadata": {
        "id": "z-clQdZt83R8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h2> Defining Various Utility functions</h2>"
      ],
      "metadata": {
        "id": "7OI2Z4lEK1Ro"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Throughout this project, we have used various functions that we have each individually defined in this section.\n",
        "<ul>\n",
        "<li> Initialization Functions</li>\n",
        "<li> Loss Functions</li>\n",
        "<li> Activation Funcitons and their derivatives</li>"
      ],
      "metadata": {
        "id": "TLTyxzJ8TJjs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3> Initialization Functions</h3>"
      ],
      "metadata": {
        "id": "cBekYpc66E6s"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resource used: https://machinelearningmastery.com/weight-initialization-for-deep-learning-neural-networks/"
      ],
      "metadata": {
        "id": "a6obe5deUR2A"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4>Xavier</h4>"
      ],
      "metadata": {
        "id": "J-vCuKLzUsy-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def Xavier(layer_sizes):\n",
        "\n",
        "  '''This function is used to get the xavier initialization for the \n",
        "  given network architecture.\n",
        "\n",
        "  It takes input layer_sizes which is a list of the number of neurons\n",
        "  in each hidden layer.\n",
        "\n",
        "  It returns a dictionary which has the various w and b parameters,\n",
        "  initialized using xavier initialization'''\n",
        "\n",
        "  params = {}\n",
        "  for i in range(1,len(layer_sizes)):\n",
        "      norm_xav=np.sqrt(6)/np.sqrt(layer_sizes[i]+layer_sizes[i-1])\n",
        "      params[\"w\"+str(i)]=np.random.randn(layer_sizes[i],layer_sizes[i-1])*norm_xav\n",
        "      params[\"b\"+str(i)]=np.zeros((layer_sizes[i],1))\n",
        "  \n",
        "  return params\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "nUVnJQfuUHqb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4> Random</h4>"
      ],
      "metadata": {
        "id": "eOfqsoS8Uv4M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def Random(layer_sizes):\n",
        "\n",
        "  '''This function is used to get the random initialization for the \n",
        "  given network architecture.\n",
        "\n",
        "  It takes input layer_sizes which is a list of the number of neurons\n",
        "  in each hidden layer.\n",
        "\n",
        "  It returns a dictionary which has the various w and b parameters,\n",
        "  initialized using random initialization'''\n",
        "\n",
        "  params = {}\n",
        "  for i in range(1,len(layer_sizes)):\n",
        "      params[\"w\"+str(i)]=0.01*np.random.randn(layer_sizes[i],layer_sizes[i-1])\n",
        "      params[\"b\"+str(i)]=0.01*np.random.randn(layer_sizes[i],1)\n",
        "\n",
        "  return params"
      ],
      "metadata": {
        "id": "1sRkD2gu46dL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3> Activation Functions </h3>"
      ],
      "metadata": {
        "id": "I7VIhXa56Zi0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resource used: https://machinelearningmastery.com/choose-an-activation-function-for-deep-learning/"
      ],
      "metadata": {
        "id": "xT0j8JKtVTO9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4> Sigmoid</h4>"
      ],
      "metadata": {
        "id": "wSPSDsgdVVSD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sigmoid(pre_act):\n",
        "  try:\n",
        "    return (1.0/(1.0+np.exp(-pre_act)))\n",
        "  except:\n",
        "    print(\"error in sigmoid\")"
      ],
      "metadata": {
        "id": "ZM7quY3l6PZZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4>Tanh</h4>"
      ],
      "metadata": {
        "id": "sIsWryV5VYCW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def tanh(pre_act):\n",
        "  return (np.tanh(pre_act))\n"
      ],
      "metadata": {
        "id": "dAvWjzlz6RjK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4>ReLU</h4>"
      ],
      "metadata": {
        "id": "m6rrPjTtVaWp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def relu(pre_act):\n",
        "  return (np.maximum(0,pre_act))"
      ],
      "metadata": {
        "id": "D8ZpvzQu6TZt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4> Output Activation : Softmax</h4>"
      ],
      "metadata": {
        "id": "vhvueCZHVhB_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def softmax(x):\n",
        "  try:\n",
        "    return(np.exp(x)/np.sum(np.exp(x)))\n",
        "  except:\n",
        "    print(\"error in softmax\")"
      ],
      "metadata": {
        "id": "8Vq4OnO96WRI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4> Derivatives of Activation Functions </h4>"
      ],
      "metadata": {
        "id": "PZcX9QpD6keQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used:<br>\n",
        "https://towardsdatascience.com/derivative-of-the-softmax-function-and-the-categorical-cross-entropy-loss-ffceefc081d1"
      ],
      "metadata": {
        "id": "3YuXUKAS4zJr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sigmoid_derivative(x):\n",
        "  return sigmoid(x)*(1-sigmoid(x))"
      ],
      "metadata": {
        "id": "6SgvWXU76reg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tanh_derivative(x):\n",
        "  return 1.0 -tanh(x)**2\n"
      ],
      "metadata": {
        "id": "EFMQdq_b6rcV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def relu_derivative(x):\n",
        "  return 1. * (x>0)"
      ],
      "metadata": {
        "id": "bUXYoFkF6rZ-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def softmax_derivative(x):\n",
        "  return softmax(x) * (1-softmax(x))"
      ],
      "metadata": {
        "id": "Up46ku0y6rRm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def derivative(A, activation):\n",
        "\n",
        "  '''This function is essentially a caller function. It takes in the \n",
        "  kindof activation funciton used as well as the value and calls the \n",
        "  respective activation functions derivative\n",
        "\n",
        "  Input is the actual data and the choice of activation funtion.\n",
        "\n",
        "  Output is the derivative of that data wrt the activation function'''\n",
        "\n",
        "  if activation == \"sigmoid\":\n",
        "    return sigmoid_derivative(A)\n",
        "  elif activation == \"tanh\":\n",
        "    return tanh_derivative(A)\n",
        "  elif activation == \"relu\":\n",
        "    return relu_derivative(A)\n"
      ],
      "metadata": {
        "id": "rHgm-ZbS6OLW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3>Loss Functions</h3>"
      ],
      "metadata": {
        "id": "56KNthDl67G7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used:<br>\n",
        " https://machinelearningmastery.com/loss-and-loss-functions-for-training-deep-learning-neural-networks/<br>\n",
        " https://towardsdatascience.com/derivative-of-the-softmax-function-and-the-categorical-cross-entropy-loss-ffceefc081d1"
      ],
      "metadata": {
        "id": "0pduqoTjWJZx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4> Mean Squared Error Loss</h4>"
      ],
      "metadata": {
        "id": "FDDkMlTiWTEY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def MSE(y, y_hat):\n",
        "  error = np.sum(((y - y_hat)**2) / (2 * len(y)))\n",
        "  return error"
      ],
      "metadata": {
        "id": "PRnQH-867Ayy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4>Cross Entropy Loss</h4>"
      ],
      "metadata": {
        "id": "uKCkovZqWXuY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def CrossEntropy(y, y_hat):\n",
        "  error = - np.sum( np.multiply(y , np.log(y_hat)))/len(y)\n",
        "  return error"
      ],
      "metadata": {
        "id": "YP4QzvRF7Xw-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4> Regularization (weight decay)</h4>"
      ],
      "metadata": {
        "id": "IR-cQwHCWd7b"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resource used: https://towardsdatascience.com/weight-decay-l2-regularization-90a9e17713cd"
      ],
      "metadata": {
        "id": "2qs5BCSMWwDG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Calculating loss \n",
        "def loss_calc(loss_name, y, y_hat, lambd, layer_sizes, parameters):\n",
        "\n",
        "  '''This function is used to calculate the L2 Regularized loss.\n",
        "  \n",
        "  Input is the loss name which denotes the type of loss, the true labels,\n",
        "  the predicted labels, lambda (i.e., for L2 Regularization), architecture \n",
        "  of the network and the parameters dictionary\n",
        "\n",
        "  Output is the L2 Regularized Loss'''\n",
        "\n",
        "  error=0\n",
        "  if(loss_name == \"squared_loss\"):\n",
        "    error=MSE(y, y_hat)\n",
        "  elif(loss_name == \"cross_entropy\"):\n",
        "    error=CrossEntropy(y, y_hat)\n",
        "    #error = -np.sum(np.sum(y_t*np.log(y_hat)))\n",
        "\n",
        "  #For L2 Regularization\n",
        "  regularized_error = 0.0\n",
        "  for i in range(len(layer_sizes)-1, 0, -1):\n",
        "    regularized_error += (np.sum(parameters[\"w\"+str(i)]))**2\n",
        "  regularized_error = error + ((lambd/(2*len(y)))*(regularized_error))\n",
        "\n",
        "\n",
        "  return regularized_error"
      ],
      "metadata": {
        "id": "MyC7PPN16-Sn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4> Accuracy <h4>\n",
        "\n"
      ],
      "metadata": {
        "id": "j1tkJKxj0QzZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used:<br>\n",
        "https://towardsdatascience.com/comprehensive-guide-to-multiclass-classification-with-sklearn-127cc500f362<br>\n",
        "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html\n"
      ],
      "metadata": {
        "id": "nwbD3kpd4ecJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def calc_accuracy(res, y_t):\n",
        "    \n",
        "    '''This function is used to calculate the accuracy of the given prediction\n",
        "\n",
        "    Input is the true labels, and the predicted labels. Here, both true and\n",
        "    predicted labels are in the probability distribution format.\n",
        "\n",
        "    Output is the single float value that denotes the accuracy of the prediction'''\n",
        "\n",
        "    acc=0.0\n",
        "    \n",
        "    for x in range(len(res)):\n",
        "      if(res[x].argmax()==y_t[x].argmax()):\n",
        "        acc+=1\n",
        "    acc=acc/len(y_t)\n",
        "    return(acc*100)"
      ],
      "metadata": {
        "id": "H-FBfniHt5I_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calc_test_accuracy(y_pred, y_t):\n",
        "\n",
        "  '''This function is used to calculate the accuracy of the given prediction\n",
        "\n",
        "    Input is the true labels, and the predicted labels.\n",
        "\n",
        "    Output is the single float value that denotes the accuracy of the prediction'''\n",
        "\n",
        "  acc=0.0\n",
        "\n",
        "  for i in range(len(y_pred)):\n",
        "    if(y_pred[i]==y_t[i]):\n",
        "      acc+=1\n",
        "  acc=acc/len(y_t)\n",
        "  return(acc*100)"
      ],
      "metadata": {
        "id": "Bghjsd1F6-QF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>2. Implementation of the Neural Network"
      ],
      "metadata": {
        "id": "2Nzcdpj7KxpQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h2> Initialization of Neural Network</h2>"
      ],
      "metadata": {
        "id": "RvRGfWa078JY"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Intuition about how to store the various parameters in our network and the overall architecture of the network was taken from the below article: https://www.kaggle.com/mtax687/l-layer-neural-network-using-numpy"
      ],
      "metadata": {
        "id": "Ukg4cZ-bYBDP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def nn_init(layer_sizes, init_type = \"random\"):\n",
        "\n",
        "  '''This funciton is used to initialize the neural network with the \n",
        "  choice of the initialization.\n",
        "\n",
        "  Input is the broad sizes of the various layers stored in a list and the\n",
        "  choice of initialization type: xavier or random\n",
        "\n",
        "  Output is the parameters dictionary which stores the various parameters\n",
        "  for each layer.'''\n",
        "\n",
        "  # The parameters are stored as a dictionary with each layer having its own\n",
        "  # key for W and b. \n",
        "\n",
        "  # initializing parameters for the neural network, \n",
        "  params={}\n",
        "  if(init_type==\"xavier\"):\n",
        "    params = Xavier(layer_sizes)\n",
        "\n",
        "  elif(init_type==\"random\"):\n",
        "    params = Random(layer_sizes)\n",
        "\n",
        "  else:\n",
        "    print(\"Enter a valid weight initilization type\")\n",
        "\n",
        "  return params\n"
      ],
      "metadata": {
        "id": "2ZICexToBSP0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h2> Forward Propagation</h2>"
      ],
      "metadata": {
        "id": "Noyewh3N8QGB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used:<br>\n",
        " https://www.geeksforgeeks.org/implementation-of-neural-network-from-scratch-using-numpy/\n",
        " <br> https://towardsdatascience.com/back-propagation-demystified-in-7-minutes-4294d71a04d7"
      ],
      "metadata": {
        "id": "wn2Qro9iZWSZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def forward_prop(X, y, params, active, layer_sizes):\n",
        "  \n",
        "  '''This function is used to forward propagate the data point and return \n",
        "  the predicted label.\n",
        "\n",
        "  Input is the given data point (only one data point) and its respective true \n",
        "  label vector, the parameters dictionary, the activation functions choice\n",
        "  and the overall architecture of the network.\n",
        "\n",
        "  Output is the predicted y label i.e., y_hat and the list of\n",
        "  various preactivations and post activations for each neuron of each layer.\n",
        "  (Each is stored as a list of list) '''\n",
        "\n",
        "\n",
        "  # Extracting only the image data not the label for the image data\n",
        "  out=copy.deepcopy(X)\n",
        "  out=out.reshape(-1,1)\n",
        "  \n",
        "  #These are stored just to make it easy to keep track of the indices along with layers.\n",
        "  h=[out] # To save the activations for each neuron in a layer\n",
        "  a=[out] # To save the preactivation for each neuron in a layer\n",
        "\n",
        "  if(active==\"sigmoid\"):\n",
        "    for i in range(1,len(layer_sizes)-1):\n",
        "      weights = params[\"w\"+str(i)]\n",
        "      biases = params[\"b\"+str(i)]\n",
        "      \n",
        "      #Actual Forward Propagation logic\n",
        "      out = np.dot(weights,h[i-1])+biases\n",
        "      a.append(out)\n",
        "      post_a = sigmoid(out)\n",
        "      h.append(post_a)\n",
        "  \n",
        "  elif(active==\"tanh\"):\n",
        "    for i in range(1,len(layer_sizes)-1):\n",
        "      weights=params[\"w\"+str(i)]\n",
        "      biases=params[\"b\"+str(i)]\n",
        "      \n",
        "      #Actual Forward Propagation logic\n",
        "      out=np.dot(weights,h[i-1])+biases\n",
        "      a.append(out)\n",
        "      post_a=tanh(out)\n",
        "      h.append(post_a)\n",
        "  \n",
        "  elif(active==\"relu\"):\n",
        "    for i in range(1,len(layer_sizes)-1):\n",
        "      weights=params[\"w\"+str(i)]\n",
        "      biases=params[\"b\"+str(i)]\n",
        "      \n",
        "      #Actual Forward Propagation logic\n",
        "      out=np.dot(weights,h[i-1])+biases\n",
        "      a.append(out)\n",
        "      post_a=relu(out)\n",
        "      h.append(post_a)       \n",
        "  else:\n",
        "    print(\"Enter a valid activation function\") \n",
        "\n",
        "  # Final step for forward propagation, using softmax.\n",
        "  weights=params[\"w\"+str(len(layer_sizes)-1)]\n",
        "  biases=params[\"b\"+str(len(layer_sizes)-1)]\n",
        "  \n",
        "  out=np.dot(weights,h[len(layer_sizes)-2])+biases\n",
        "  a.append(out)\n",
        "  y_hat=softmax(out)\n",
        "  h.append(y_hat)\n",
        "  \n",
        "  \n",
        "  #in h we  are storing values for layers right from input till output\n",
        "  #h0 is input\n",
        "  #in a we are storing values for layers right from input till output\n",
        "  #a0 is input\n",
        "\n",
        "  return h,a,y_hat"
      ],
      "metadata": {
        "id": "8VaQdyqbrqsO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h2>Back Propagation</h2>"
      ],
      "metadata": {
        "id": "wmjefdiK8cxi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resource used:<br> https://www.cs.swarthmore.edu/~meeden/cs81/s10/BackPropDeriv.pdf <br>\n",
        "https://dfdazac.github.io/06-neural-networks-numpy.html<br>\n",
        "Lecture slides from Mitesh Khapra were also used extensively."
      ],
      "metadata": {
        "id": "-qqgoKSbcr6z"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is a <b>to the point</b> coding of the pseudo code that was discussed in the class"
      ],
      "metadata": {
        "id": "5xpMKTT6a6dD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def back_prop(y, y_hat, h, a, params, loss_type, layer_sizes, activation):\n",
        "  \n",
        "  '''This is the heart of the code. It is used to back propagate the calculated \n",
        "  error and calculating the gradients for each required entity\n",
        "\n",
        "  The input is the true label, predicted label, preactivations, postactivations,\n",
        "  paramters dicionary, the type of loss considered, overall network architecture\n",
        "  and the choice of activation function.\n",
        "\n",
        "  The output is the gradient dictionary which stores the gradients calculated\n",
        "  for each parameter. '''\n",
        "\n",
        "  # We are considering point by point. i.e., we are propagating only one point,\n",
        "  # then back propagation that single point only. \n",
        "\n",
        "  #here both y_hat and y are assumed to be column vectors\n",
        "\n",
        "  # Initializing the empty dictionary to store the gradients\n",
        "  grad = {}\n",
        "\n",
        "  if loss_type == \"squared_loss\":\n",
        "    grad[\"dh\"+str(len(layer_sizes)-1)] = (y_hat - y)\n",
        "    grad[\"da\"+str(len(layer_sizes)-1)] = (y_hat - y) * softmax_derivative(a[len(layer_sizes)-1])\n",
        "\n",
        "  elif loss_type == 'cross_entropy':\n",
        "    # Here actually it should be one hot vector (As seen in class).\n",
        "    # But y does the same job (since it is also one hot encoded)\n",
        "    grad[\"da\"+str(len(layer_sizes)-1)] = -(y-y_hat)\n",
        "    grad[\"dh\"+str(len(layer_sizes)-1)] = -(y/y_hat)\n",
        "\n",
        "  for i in range(len(layer_sizes)-1, 0, -1 ):\n",
        "    #print(i)\n",
        "    # Not considering L2 Regularization here. Instead will cumulate in the update section\n",
        "    # As referred from the resource pointed in Regularization section.\n",
        "\n",
        "    grad[\"dw\"+str(i)] = np.dot(grad[\"da\"+str(i)], np.transpose(h[i-1]))\n",
        "    grad[\"db\"+str(i)] = grad[\"da\"+str(i)]\n",
        "\n",
        "    #Since we are going backwards, we wont execute these for the final iteration\n",
        "    if i > 1:\n",
        "      grad[\"dh\"+str(i-1)] = np.dot(np.transpose(params[\"w\"+str(i)]), grad[\"da\"+str(i)])\n",
        "      grad[\"da\"+str(i-1)] = np.multiply(grad[\"dh\" + str(i-1)], derivative(a[i-1],activation))\n",
        " \n",
        "  return grad\n",
        "\n"
      ],
      "metadata": {
        "id": "kAQEfBMIFfW0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4>Function to calculate gradients, batchwise</h4>\n",
        "\n",
        "Small function that calculates the gradients for a given batch of points."
      ],
      "metadata": {
        "id": "cYFgtV7ZIOCU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def grad_calculate_batchwise(X, Y, parameters, activation, layers, loss_function):\n",
        "\n",
        "  ''' This function is used to calculate the cumulative gradient of the given\n",
        "  batch of points (used for mini batch gradient descent and its variants)\n",
        "\n",
        "  Input is the current batch of data points, corresponding true labels, \n",
        "  parameters of the network, choice of the activation function, overall\n",
        "  architecure of the network and the loss function\n",
        "\n",
        "  Output is the dictionary which has the cumulative gradients collected \n",
        "  for the entire batch of data points. (structure of the dictionary is\n",
        "  same as the parameters dictionary)'''\n",
        "\n",
        "\n",
        "  #Initializing the empty dictionary\n",
        "  grads={}\n",
        "  grads.clear() \n",
        "\n",
        "  #iterate over all the points in the current batch\n",
        "  for j in range(len(X)):\n",
        "\n",
        "    #Reshaping the labels, to get a column vector\n",
        "    y = np.reshape(Y[j], (-1,1))\n",
        "\n",
        "    #Feed forward the data point\n",
        "    h,a,y_hat = forward_prop(X[j], y, parameters, activation, layers)\n",
        "\n",
        "    #backpropagate the error.\n",
        "    new_grads = back_prop(y,y_hat, h,a, parameters, loss_function, layers, activation)\n",
        "\n",
        "    #keep collecting the gradients for all the data\n",
        "    if j == 0:\n",
        "      # if j is 0 means it is the first batch and hence it will be equal to the \n",
        "      # calculated gradients\n",
        "      grads = copy.deepcopy(new_grads)\n",
        "\n",
        "    else:\n",
        "      # For remaining cases, we increment the previous gradient values with the \n",
        "      # current gradient values.\n",
        "      for k in range(len(layers)-1,0,-1):\n",
        "        grads[\"dw\"+str(k)] += new_grads[\"dw\"+str(k)]\n",
        "        grads[\"db\"+str(k)] += new_grads[\"db\"+str(k)]\n",
        "  \n",
        "  return grads"
      ],
      "metadata": {
        "id": "CJXUB3SdINv8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>3. Different Optimization Functions</h2>"
      ],
      "metadata": {
        "id": "JhnoYTOQ8g9F"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3> Mini Batch Gradient Descent </h3>"
      ],
      "metadata": {
        "id": "KKHa2-Fh8lye"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used:<br>\n",
        "https://ruder.io/optimizing-gradient-descent/"
      ],
      "metadata": {
        "id": "Fvl5rGw-5QFI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def mini_batch_gd(X_train, y_train, eta, max_epochs, layers, mini_batch_size, lambd, loss_function, activation, parameters,wandb_log=False):\n",
        "  #parameters = nn_init(layers, 'random')\n",
        "\n",
        "  ''' This function is essentially used to start the epochs, collect the gradients,\n",
        "  update the parameters according to the vanilla mini batch gradient descent algorithm.\n",
        "  It also keeps track of the train and validation accuracies for each epoch. Also log\n",
        "  these value into wandb. \n",
        "\n",
        "  wandb_log is a flag. If true, the results are logged onto wandb workspace\n",
        "\n",
        "  Input is training data, true labels for train data, validation data, labels\n",
        "  (used from global context), eta i.e., learning rate, maximum epochs, overall\n",
        "  architecture of the network, mini batch size, lambda for L2 Regularization, \n",
        "  activation funcion, parameters of the network.\n",
        "\n",
        "  Output is the updated paramters (after training), training error and validation\n",
        "  errors lists'''\n",
        "\n",
        "  \n",
        "  # Declaring an empty dicitonary for gradients\n",
        "  grads={}\n",
        "  \n",
        "  # Declaring various lists to store the loss and accuracies\n",
        "  train_errors_list = []\n",
        "  val_errors_list = []\n",
        "  train_acc_list = []\n",
        "  val_acc_list = []\n",
        "\n",
        "  # iterate till max epochs\n",
        "  for t in tqdm(range(max_epochs)):\n",
        "\n",
        "\n",
        "    # iterate over all batches\n",
        "    for i in range(0, len(X_train), mini_batch_size):\n",
        "\n",
        "      grads.clear()\n",
        "\n",
        "      # Divide the data into batches and get the current batch\n",
        "      X = X_train[i:i + mini_batch_size]\n",
        "      Y = y_train[i:i + mini_batch_size]\n",
        "\n",
        "      # Collect the gradients using the current batch of points\n",
        "      grads = grad_calculate_batchwise(X,Y,parameters,activation,layers,loss_function)\n",
        "      \n",
        "    \n",
        "      #Updating the parameters once every one batch\n",
        "      for j in range(len(layers)-1,0,-1):\n",
        "        # Here we have included the L2 Regularization\n",
        "        parameters[\"w\"+str(j)] = (1-((eta*lambd)/mini_batch_size))*parameters[\"w\"+str(j)] - (eta * grads[\"dw\"+str(j)])\n",
        "        parameters[\"b\"+str(j)] = parameters[\"b\"+str(j)] - (eta * grads[\"db\"+str(j)])\n",
        "\n",
        "    #Calculating train loss and accuracies\n",
        "    res = predict(X_train,y_train,parameters, activation, layers)\n",
        "    train_err = loss_calc(loss_function, y_train, res, lambd, layers, parameters )\n",
        "    train_acc = calc_accuracy(res, y_train)\n",
        "    train_errors_list.append(train_err)\n",
        "    train_acc_list.append(train_acc)\n",
        "\n",
        "    #Calculating validation loss\n",
        "    res = predict(X_val, y_val, parameters, activation, layers)\n",
        "    val_err = loss_calc(loss_function, y_val, res, lambd, layers, parameters )\n",
        "    val_acc = calc_accuracy(res,y_val)\n",
        "    val_errors_list.append(val_err)\n",
        "    val_acc_list.append(val_acc)\n",
        "\n",
        "    if(wandb_log==True):\n",
        "      # Logging the values into wandb\n",
        "      log_dict = {\"Train_Accuracy\": train_acc, \"Validation_Accuracy\": val_acc, \\\n",
        "                  \"Train_Loss\": train_err, \"Validation_loss\": val_err, \"epoch\": t}\n",
        "                  \n",
        "      wandb.log(log_dict)\n",
        "\n",
        "  return parameters, train_acc_list, val_acc_list\n"
      ],
      "metadata": {
        "id": "H7a01Svldakv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3> Momentum Based Gradient Descent </h3>"
      ],
      "metadata": {
        "id": "yPiDf0vS8vsO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used:<br>\n",
        "https://ruder.io/optimizing-gradient-descent/"
      ],
      "metadata": {
        "id": "-PzRbvtX5ORw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def momentum_gd(X_train,y_train, eta, max_epochs, layers, mini_batch_size, lambd, loss_function, activation, parameters,wandb_log=False ):\n",
        "  #parameters = nn_init(layers, 'random')\n",
        "\n",
        "  ''' This function is essentially used to start the epochs, collect the gradients,\n",
        "  update the parameters according to the momentum mini batch gradient descent algorithm.\n",
        "  It also keeps track of the train and validation accuracies for each epoch. Also log\n",
        "  these value into wandb. \n",
        "\n",
        "  wandb_log is a flag. If true, the results are logged onto wandb workspace\n",
        "\n",
        "  Input is training data, true labels for train data, validation data, labels\n",
        "  (used from global context), eta i.e., learning rate, maximum epochs, overall\n",
        "  architecture of the network, mini batch size, lambda for L2 Regularization, \n",
        "  activation funcion, parameters of the network.\n",
        "\n",
        "  Output is the updated paramters (after training), training error and validation\n",
        "  errors lists'''\n",
        "  \n",
        "  # Declaring an empty dicitonary for gradients and update history\n",
        "  grads={}\n",
        "  update_history = {}\n",
        "  gamma = 0.9 #Not treating this as a hyperparameter\n",
        "\n",
        "  # Declaring various lists to store the loss and accuracies\n",
        "  train_errors_list = []\n",
        "  val_errors_list = []\n",
        "  train_acc_list = []\n",
        "  val_acc_list = []\n",
        "\n",
        "  # iterate till max epochs\n",
        "  for t in tqdm(range(max_epochs)):\n",
        "\n",
        "\n",
        "    # iterate over all batches\n",
        "    for i in range(0, len(X_train), mini_batch_size):\n",
        "\n",
        "      grads.clear()\n",
        "\n",
        "      # Divide the data into batches and get the current batch\n",
        "      X = X_train[i:i + mini_batch_size]\n",
        "      Y = y_train[i:i + mini_batch_size]\n",
        "      \n",
        "      # Collect the gradients using the current batch of points\n",
        "      grads=grad_calculate_batchwise(X,Y,parameters,activation,layers,loss_function)\n",
        "      \n",
        "      #Storing the update history for each parameter.\n",
        "      if i == 0 :\n",
        "        # If i is 0, then it is the first batch and the update history\n",
        "        # will be equal to the current gradients itself\n",
        "        for j in range(len(layers)-1, 0, -1):\n",
        "          update_history[\"w\"+str(j)] = eta*grads[\"dw\"+str(j)]\n",
        "          update_history[\"b\"+str(j)] = eta*grads[\"db\"+str(j)]\n",
        "      else:\n",
        "        # else we store the update history according to momentum based gd algorihtm\n",
        "        for j in range(len(layers)-1, 0, -1):\n",
        "          update_history[\"w\"+str(j)] = (gamma*update_history[\"w\"+str(j)]) + (eta*grads[\"dw\"+str(j)])\n",
        "          update_history[\"b\"+str(j)] = (gamma*update_history[\"b\"+str(j)]) + (eta*grads[\"db\"+str(j)])\n",
        "\n",
        "    \n",
        "      #Updating the parameters once every one batch with the update_history\n",
        "      for j in range(len(layers)-1,0,-1):\n",
        "        parameters[\"w\"+str(j)] = (1-((eta*lambd)/mini_batch_size))*parameters[\"w\"+str(j)] - update_history[\"w\"+str(j)]\n",
        "        parameters[\"b\"+str(j)] = parameters[\"b\"+str(j)] - update_history[\"b\"+str(j)]\n",
        "\n",
        "    #Calculating train loss and accuracies\n",
        "    res = predict(X_train,y_train,parameters, activation, layers)\n",
        "    train_err = loss_calc(loss_function, y_train, res, lambd, layers, parameters )\n",
        "    train_acc = calc_accuracy(res, y_train)\n",
        "    train_errors_list.append(train_err)\n",
        "    train_acc_list.append(train_acc)\n",
        "\n",
        "    #Calculating validation loss\n",
        "    res = predict(X_val, y_val, parameters, activation, layers)\n",
        "    val_err = loss_calc(loss_function, y_val, res, lambd, layers, parameters )\n",
        "    val_acc = calc_accuracy(res,y_val)\n",
        "    val_errors_list.append(val_err)\n",
        "    val_acc_list.append(val_acc)\n",
        "\n",
        "    if(wandb_log==True):\n",
        "      # Logging the values into wandb\n",
        "      log_dict = {\"Train_Accuracy\": train_acc, \"Validation_Accuracy\": val_acc, \\\n",
        "                  \"Train_Loss\": train_err, \"Validation_loss\": val_err, \"epoch\": t}\n",
        "                  \n",
        "      wandb.log(log_dict)\n",
        "\n",
        "  return parameters, train_acc_list, val_acc_list\n"
      ],
      "metadata": {
        "id": "NHgO8wzQ8-qD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3> Nesterov Accelerated Gradient Descent</h3>"
      ],
      "metadata": {
        "id": "zXL670qB879E"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used:<br>\n",
        "https://ruder.io/optimizing-gradient-descent/"
      ],
      "metadata": {
        "id": "6-vfXwyi5Keh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def nesterov_gd(X_train, y_train, eta, max_epochs, layers, mini_batch_size, lambd, loss_function, activation, parameters, wandb_log=False ):\n",
        " \n",
        "  ''' This function is essentially used to start the epochs, collect the gradients,\n",
        "  update the parameters according to the nesterov accelerated mini batch gradient descent algorithm.\n",
        "  It also keeps track of the train and validation accuracies for each epoch. Also log\n",
        "  these value into wandb. \n",
        "\n",
        "  wandb_log is a flag. If true, the results are logged onto wandb workspace\n",
        "\n",
        "  Input is training data, true labels for train data, validation data, labels\n",
        "  (used from global context), eta i.e., learning rate, maximum epochs, overall\n",
        "  architecture of the network, mini batch size, lambda for L2 Regularization, \n",
        "  activation funcion, parameters of the network.\n",
        "\n",
        "  Output is the updated paramters (after training), training error and validation\n",
        "  errors lists'''\n",
        "\n",
        "  # Declaring an empty dicitonary for gradients and update history and lookahead\n",
        "  grads={}\n",
        "  update_history = {}\n",
        "  param_lookahead = {}\n",
        "  gamma = 0.9 #not treating this as a hyperparameter.\n",
        "\n",
        "  # Declaring various lists to store the loss and accuracies\n",
        "  train_errors_list = []\n",
        "  val_errors_list = []\n",
        "  train_acc_list = []\n",
        "  val_acc_list = []\n",
        "\n",
        "  #iterate till max epochs\n",
        "  for t in tqdm(range(max_epochs)):\n",
        "\n",
        "\n",
        "    #iterate over all batches\n",
        "    for i in range(0, len(X_train), mini_batch_size):\n",
        "\n",
        "      grads.clear()\n",
        "\n",
        "      #If it is the first batch, we still dont have the previous history.\n",
        "      #So, lookahead will be same as the current parameters\n",
        "      if i==0:\n",
        "        param_lookahead = copy.deepcopy(parameters)\n",
        "      \n",
        "      #If its not the first batch then we calculate lookahead according to\n",
        "      #the formula.\n",
        "      else:\n",
        "        for j in range(len(layers)-1, 0, -1):\n",
        "          param_lookahead['w'+str(j)] = parameters['w'+str(j)] + (gamma*update_history[\"w\"+str(j)])\n",
        "                                                                  \n",
        "\n",
        "      # Divide the data into batches and get the current batch\n",
        "      X = X_train[i:i + mini_batch_size]\n",
        "      Y = y_train[i:i + mini_batch_size]\n",
        "      \n",
        "      # Collect the gradients using the current batch of points\n",
        "      grads=grad_calculate_batchwise(X,Y,param_lookahead,activation,layers,loss_function)\n",
        "      \n",
        "      # Storing the update history for each parameter.\n",
        "      if i == 0 :\n",
        "        # If its the first batch, we dont have any update history yet. So, it will\n",
        "        # be same as the eta*gradients\n",
        "        for j in range(len(layers)-1, 0, -1):\n",
        "          update_history[\"w\"+str(j)] = eta*grads[\"dw\"+str(j)]\n",
        "          update_history[\"b\"+str(j)] = eta*grads[\"db\"+str(j)]\n",
        "      \n",
        "      # If its not the first batch, we cumulate the update history as per the \n",
        "      # formula.\n",
        "      else:\n",
        "        for j in range(len(layers)-1, 0, -1):\n",
        "          update_history[\"w\"+str(j)] = (gamma*update_history[\"w\"+str(j)]) + (eta*grads[\"dw\"+str(j)])\n",
        "          update_history[\"b\"+str(j)] = (gamma*update_history[\"b\"+str(j)]) + (eta*grads[\"db\"+str(j)])\n",
        "\n",
        "    \n",
        "      #Updating the parameters once every one batch with the update_history\n",
        "      for j in range(len(layers)-1,0,-1):\n",
        "        parameters[\"w\"+str(j)] = (1-((eta*lambd)/mini_batch_size))*parameters[\"w\"+str(j)] - update_history[\"w\"+str(j)]\n",
        "        parameters[\"b\"+str(j)] = parameters[\"b\"+str(j)] - update_history[\"b\"+str(j)]\n",
        "\n",
        "    #Calculating train loss and accuracies\n",
        "    res = predict(X_train,y_train,parameters, activation, layers)\n",
        "    train_err = loss_calc(loss_function, y_train, res, lambd, layers, parameters )\n",
        "    train_acc = calc_accuracy(res, y_train)\n",
        "    train_errors_list.append(train_err)\n",
        "    train_acc_list.append(train_acc)\n",
        "\n",
        "    #Calculating validation loss\n",
        "    res = predict(X_val, y_val, parameters, activation, layers)\n",
        "    val_err = loss_calc(loss_function, y_val, res, lambd, layers, parameters )\n",
        "    val_acc = calc_accuracy(res,y_val)\n",
        "    val_errors_list.append(val_err)\n",
        "    val_acc_list.append(val_acc)\n",
        "\n",
        "    if(wandb_log==True):  \n",
        "      # Logging the values into wandb\n",
        "      log_dict = {\"Train_Accuracy\": train_acc, \"Validation_Accuracy\": val_acc, \\\n",
        "                  \"Train_Loss\": train_err, \"Validation_loss\": val_err, \"epoch\": t}\n",
        "                  \n",
        "      wandb.log(log_dict)\n",
        "\n",
        "  return parameters, train_acc_list, val_acc_list\n"
      ],
      "metadata": {
        "id": "WEglGxnFE3Fg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3> RMSprop</h3>"
      ],
      "metadata": {
        "id": "xNB0DGgD9Ck7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used: https://towardsdatascience.com/learning-parameters-part-5-65a2f3583f7d"
      ],
      "metadata": {
        "id": "kO2tl8qik3le"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def rmsprop(X_train,y_train, eta, max_epochs, layers, mini_batch_size, lambd, loss_function, activation, parameters,wandb_log=False ):\n",
        "    \n",
        "  ''' This function is essentially used to start the epochs, collect the gradients,\n",
        "  update the parameters according to the RmsProp mini batch gradient descent algorithm.\n",
        "  It also keeps track of the train and validation accuracies for each epoch. Also log\n",
        "  these value into wandb. \n",
        "\n",
        "  wandb_log is a flag. If true, the results are logged onto wandb workspace\n",
        "\n",
        "  Input is training data, true labels for train data, validation data, labels\n",
        "  (used from global context), eta i.e., learning rate, maximum epochs, overall\n",
        "  architecture of the network, mini batch size, lambda for L2 Regularization, \n",
        "  activation funcion, parameters of the network.\n",
        "\n",
        "  Output is the updated paramters (after training), training error and validation\n",
        "  errors lists'''\n",
        "  \n",
        "  # Declaring an empty dicitonary for gradients and update history \n",
        "  grads={}\n",
        "  update_history = {}\n",
        "  v={}\n",
        "\n",
        "  # Declaring various lists to store the loss and accuracies\n",
        "  train_errors_list = []\n",
        "  val_errors_list = []\n",
        "  train_acc_list = []\n",
        "  val_acc_list = []\n",
        "  \n",
        "  # Initializing update_history with zeros\n",
        "  for i in range(len(layers)-1,0,-1):\n",
        "    update_history[\"w\"+str(i)]=np.zeros((layers[i],layers[i-1]))\n",
        "    update_history[\"b\"+str(i)]=np.zeros((layers[i],1))\n",
        "  # Initializing v with zeros\n",
        "  for i in range(len(layers)-1,0,-1):\n",
        "    v[\"w\"+str(i)]=np.zeros((layers[i],layers[i-1]))\n",
        "    v[\"b\"+str(i)]=np.zeros((layers[i],1))\n",
        "  \n",
        "  beta = 0.9 \n",
        "  epsilon=1e-8\n",
        "\n",
        "  #iterate till max epochs\n",
        "  for t in tqdm(range(max_epochs)):\n",
        "   \n",
        "    #iterate over all batches\n",
        "    for i in range(0, len(X_train), mini_batch_size):\n",
        "      grads.clear()\n",
        "\n",
        "      # Divide the data into batches and get the current batch\n",
        "      X = X_train[i:i + mini_batch_size]\n",
        "      Y = y_train[i:i + mini_batch_size]\n",
        "      \n",
        "      # Collect the gradients using the current batch of points\n",
        "      grads=grad_calculate_batchwise(X,Y,parameters,activation,layers,loss_function)\n",
        "        \n",
        "      # Updating the values if v and the update history using the computed gradients\n",
        "      for iq in range(len(layers)-1,0,-1):\n",
        "        v[\"w\"+str(iq)]=beta*v[\"w\"+str(iq)]+(1-beta)*grads[\"dw\"+str(iq)]**2\n",
        "        v[\"b\"+str(iq)]=beta*v[\"b\"+str(iq)]+(1-beta)*grads[\"db\"+str(iq)]**2\n",
        "          \n",
        "        update_history[\"w\"+str(iq)]=eta*np.multiply(np.reciprocal(np.sqrt(v[\"w\"+str(iq)]+epsilon)),grads[\"dw\"+str(iq)])\n",
        "        update_history[\"b\"+str(iq)]=eta*np.multiply(np.reciprocal(np.sqrt(v[\"b\"+str(iq)]+epsilon)),grads[\"db\"+str(iq)])\n",
        "\n",
        "      #Updating the parameters once every one batch with the update_history\n",
        "      for j in range(len(layers)-1,0,-1):\n",
        "        parameters[\"w\"+str(j)] = (1-((eta*lambd)/mini_batch_size))*parameters[\"w\"+str(j)] - update_history[\"w\"+str(j)]\n",
        "        parameters[\"b\"+str(j)] = parameters[\"b\"+str(j)] - update_history[\"b\"+str(j)]\n",
        "\n",
        "    #Calculating train loss and accuracies\n",
        "    res = predict(X_train,y_train,parameters, activation, layers)\n",
        "    train_err = loss_calc(loss_function, y_train, res, lambd, layers, parameters )\n",
        "    train_acc = calc_accuracy(res, y_train)\n",
        "    train_errors_list.append(train_err)\n",
        "    train_acc_list.append(train_acc)\n",
        "\n",
        "    #Calculating validation loss\n",
        "    res = predict(X_val, y_val, parameters, activation, layers)\n",
        "    val_err = loss_calc(loss_function, y_val, res, lambd, layers, parameters )\n",
        "    val_acc = calc_accuracy(res,y_val)\n",
        "    val_errors_list.append(val_err)\n",
        "    val_acc_list.append(val_acc)\n",
        "\n",
        "    if(wandb_log==True):\n",
        "      # Logging the values into wandb\n",
        "      log_dict = {\"Train_Accuracy\": train_acc, \"Validation_Accuracy\": val_acc, \\\n",
        "                  \"Train_Loss\": train_err, \"Validation_loss\": val_err, \"epoch\": t}\n",
        "                  \n",
        "      wandb.log(log_dict)\n",
        "\n",
        "  return parameters, train_acc_list, val_acc_list\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "22wMGB7d9Cwc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3> Adam </h3>"
      ],
      "metadata": {
        "id": "31e2lLRk9YSU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used:<br>\n",
        " https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/<br>\n",
        " https://www.geeksforgeeks.org/intuition-of-adam-optimizer/"
      ],
      "metadata": {
        "id": "CwBxwVKr4n7r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def adam(X_train,y_train, eta, max_epochs, layers, mini_batch_size, lambd, loss_function, activation, parameters,wandb_log=False ):\n",
        "    \n",
        "  ''' This function is essentially used to start the epochs, collect the gradients,\n",
        "  update the parameters according to the Adam mini batch gradient descent algorithm.\n",
        "  It also keeps track of the train and validation accuracies for each epoch. Also log\n",
        "  these value into wandb. \n",
        "\n",
        "  wandb_log is a flag. If true, the results are logged onto wandb workspace\n",
        "\n",
        "  Input is training data, true labels for train data, validation data, labels\n",
        "  (used from global context), eta i.e., learning rate, maximum epochs, overall\n",
        "  architecture of the network, mini batch size, lambda for L2 Regularization, \n",
        "  activation funcion, parameters of the network.\n",
        "\n",
        "  Output is the updated paramters (after training), training error and validation\n",
        "  errors lists'''\n",
        "  \n",
        "  # Declaring an empty dicitonary for gradients and update history \n",
        "  grads={}\n",
        "  update_history = {}\n",
        "  v={}\n",
        "  m={}\n",
        "\n",
        "  # Declaring various lists to store the loss and accuracies\n",
        "  train_errors_list = []\n",
        "  val_errors_list = []\n",
        "  train_acc_list = []\n",
        "  val_acc_list = []\n",
        "\n",
        "  # Initializing update_history to zeros\n",
        "  for i in range(len(layers)-1,0,-1):\n",
        "    update_history[\"w\"+str(i)]=np.zeros((layers[i],layers[i-1]))\n",
        "    update_history[\"b\"+str(i)]=np.zeros((layers[i],1))\n",
        "  # Initializing m to zeros\n",
        "  for i in range(len(layers)-1,0,-1):\n",
        "    m[\"w\"+str(i)]=np.zeros((layers[i],layers[i-1]))\n",
        "    m[\"b\"+str(i)]=np.zeros((layers[i],1))\n",
        "  # Initializing v to zeros\n",
        "  for i in range(len(layers)-1,0,-1):\n",
        "    v[\"w\"+str(i)]=np.zeros((layers[i],layers[i-1]))\n",
        "    v[\"b\"+str(i)]=np.zeros((layers[i],1))\n",
        "  \n",
        "  beta1 = 0.9 \n",
        "  beta2=0.999\n",
        "  epsilon=1e-8\n",
        "\n",
        "  #iterate till max epochs\n",
        "  for t in tqdm(range(max_epochs)):\n",
        "\n",
        "\n",
        "    #iterate over all batches\n",
        "    for i in range(0, len(X_train), mini_batch_size):\n",
        "      grads.clear()\n",
        "\n",
        "      # Divide the data into batches and get the current batch\n",
        "      X = X_train[i:i + mini_batch_size]\n",
        "      Y = y_train[i:i + mini_batch_size]\n",
        "      \n",
        "      # Collect the gradients using the current batch of points\n",
        "      grads=grad_calculate_batchwise(X,Y,parameters,activation,layers,loss_function)\n",
        "      \n",
        "      # Updating the values of v,m and the update history using the computed gradients\n",
        "      for iq in range(len(layers)-1,0,-1):\n",
        "          m[\"w\"+str(iq)]=beta1*m[\"w\"+str(iq)]+(1-beta1)*grads[\"dw\"+str(iq)]\n",
        "          m[\"b\"+str(iq)]=beta1*m[\"b\"+str(iq)]+(1-beta1)*grads[\"db\"+str(iq)]\n",
        "          \n",
        "          v[\"w\"+str(iq)]=beta2*v[\"w\"+str(iq)]+(1-beta2)*(grads[\"dw\"+str(iq)])**2\n",
        "          v[\"b\"+str(iq)]=beta2*v[\"b\"+str(iq)]+(1-beta2)*(grads[\"db\"+str(iq)])**2\n",
        "\n",
        "          # Bias Correction:\n",
        "          # calculating mt_hat and vt_hat for weights and biases \n",
        "          mw_hat=m[\"w\"+str(iq)]/(1-np.power(beta1,t+1))\n",
        "          mb_hat=m[\"b\"+str(iq)]/(1-np.power(beta1,t+1))\n",
        "\n",
        "          vw_hat=v[\"w\"+str(iq)]/(1-np.power(beta2,t+1))\n",
        "          vb_hat=v[\"b\"+str(iq)]/(1-np.power(beta2,t+1))\n",
        "          \n",
        "          update_history[\"w\"+str(iq)]=eta*np.multiply(np.reciprocal(np.sqrt(vw_hat+epsilon)),mw_hat)\n",
        "          update_history[\"b\"+str(iq)]=eta*np.multiply(np.reciprocal(np.sqrt(vb_hat+epsilon)),mb_hat)\n",
        "\n",
        "      #Updating the parameters once every one batch with the update_history\n",
        "      for j in range(len(layers)-1,0,-1):\n",
        "          parameters[\"w\"+str(j)] = (1-((eta*lambd)/mini_batch_size))*parameters[\"w\"+str(j)] - update_history[\"w\"+str(j)]\n",
        "          parameters[\"b\"+str(j)] = parameters[\"b\"+str(j)] - update_history[\"b\"+str(j)]\n",
        "\n",
        "    #Calculating train loss and accuracies\n",
        "    res = predict(X_train,y_train,parameters, activation, layers)\n",
        "    train_err = loss_calc(loss_function, y_train, res, lambd, layers, parameters )\n",
        "    train_acc = calc_accuracy(res, y_train)\n",
        "    train_errors_list.append(train_err)\n",
        "    train_acc_list.append(train_acc)\n",
        "\n",
        "    #Calculating validation loss\n",
        "    res = predict(X_val, y_val, parameters, activation, layers)\n",
        "    val_err = loss_calc(loss_function, y_val, res, lambd, layers, parameters )\n",
        "    val_acc = calc_accuracy(res,y_val)\n",
        "    val_errors_list.append(val_err)\n",
        "    val_acc_list.append(val_acc)\n",
        "\n",
        "    if(wandb_log==True):\n",
        "      # Logging the values into wandb\n",
        "      log_dict = {\"Train_Accuracy\": train_acc, \"Validation_Accuracy\": val_acc, \\\n",
        "                  \"Train_Loss\": train_err, \"Validation_loss\": val_err, \"epoch\": t}\n",
        "                  \n",
        "      wandb.log(log_dict)\n",
        "\n",
        "  return parameters, train_acc_list, val_acc_list\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "8S6l8vdT9csw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3>NAdam</h3>"
      ],
      "metadata": {
        "id": "wHNnGITB9gvO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used: <br>\n",
        " https://machinelearningmastery.com/gradient-descent-optimization-with-nadam-from-scratch/ <br>\n",
        "https://ruder.io/optimizing-gradient-descent/ "
      ],
      "metadata": {
        "id": "aiM4Ti3Jnib3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "def nadam(X_train,y_train, eta, max_epochs, layers, mini_batch_size, lambd, loss_function, activation, parameters,wandb_log=False ):\n",
        "    \n",
        "  ''' This function is essentially used to start the epochs, collect the gradients,\n",
        "  update the parameters according to the NAdam mini batch gradient descent algorithm.\n",
        "  It also keeps track of the train and validation accuracies for each epoch. Also log\n",
        "  these value into wandb. \n",
        "\n",
        "  wandb_log is a flag. If true, the results are logged onto wandb workspace\n",
        "\n",
        "  Input is training data, true labels for train data, validation data, labels\n",
        "  (used from global context), eta i.e., learning rate, maximum epochs, overall\n",
        "  architecture of the network, mini batch size, lambda for L2 Regularization, \n",
        "  activation funcion, parameters of the network.\n",
        "\n",
        "  Output is the updated paramters (after training), training error and validation\n",
        "  errors lists'''\n",
        "\n",
        "  # Declaring an empty dicitonary for gradients and update history \n",
        "  grads={}\n",
        "  update_history = {}\n",
        "  v={}\n",
        "  m={}\n",
        "\n",
        "  # Declaring various lists to store the loss and accuracies\n",
        "  train_errors_list = []\n",
        "  val_errors_list = []\n",
        "  train_acc_list = []\n",
        "  val_acc_list = []\n",
        "  \n",
        "\n",
        "\n",
        "  # Initializing update_history to zeros\n",
        "  for i in range(len(layers)-1,0,-1):\n",
        "    update_history[\"w\"+str(i)]=np.zeros((layers[i],layers[i-1]))\n",
        "    update_history[\"b\"+str(i)]=np.zeros((layers[i],1))\n",
        "  # Initializing m to zeros\n",
        "  for i in range(len(layers)-1,0,-1):\n",
        "    m[\"w\"+str(i)]=np.zeros((layers[i],layers[i-1]))\n",
        "    m[\"b\"+str(i)]=np.zeros((layers[i],1))\n",
        "  # Initializing v to zeros\n",
        "  for i in range(len(layers)-1,0,-1):\n",
        "    v[\"w\"+str(i)]=np.zeros((layers[i],layers[i-1]))\n",
        "    v[\"b\"+str(i)]=np.zeros((layers[i],1))\n",
        "  \n",
        "  beta1 = 0.9 \n",
        "  beta2=0.999\n",
        "  epsilon=1e-8\n",
        "\n",
        "  #iterate till max epochs\n",
        "  for t in tqdm(range(max_epochs)):\n",
        "\n",
        "\n",
        "    #iterate over all batches\n",
        "    for i in range(0, len(X_train), mini_batch_size):\n",
        "\n",
        "      grads.clear()\n",
        "\n",
        "      # Divide the data into batches and get the current batch\n",
        "      X = X_train[i:i + mini_batch_size]\n",
        "      Y = y_train[i:i + mini_batch_size]\n",
        "      \n",
        "      # Collect the gradients using the current batch of points\n",
        "      grads=grad_calculate_batchwise(X,Y,parameters,activation,layers,loss_function)\n",
        " \n",
        "      # Updating the values of v,m and the update history using the computed gradients\n",
        "      for iq in range(len(layers)-1,0,-1):\n",
        "          m[\"w\"+str(iq)]=beta1*m[\"w\"+str(iq)]+(1-beta1)*grads[\"dw\"+str(iq)]\n",
        "          m[\"b\"+str(iq)]=beta1*m[\"b\"+str(iq)]+(1-beta1)*grads[\"db\"+str(iq)]\n",
        "          \n",
        "          v[\"w\"+str(iq)]=beta2*v[\"w\"+str(iq)]+(1-beta2)*(grads[\"dw\"+str(iq)])**2\n",
        "          v[\"b\"+str(iq)]=beta2*v[\"b\"+str(iq)]+(1-beta2)*(grads[\"db\"+str(iq)])**2\n",
        "\n",
        "          # Bias Correction:\n",
        "          # calculating mt_hat and vt_hat for weights and biases \n",
        "          mw_hat=m[\"w\"+str(iq)]/(1-np.power(beta1,t+1))\n",
        "          mb_hat=m[\"b\"+str(iq)]/(1-np.power(beta1,t+1))\n",
        "\n",
        "          vw_hat=v[\"w\"+str(iq)]/(1-np.power(beta2,t+1))\n",
        "          vb_hat=v[\"b\"+str(iq)]/(1-np.power(beta2,t+1))\n",
        "          \n",
        "          update_history[\"w\"+str(iq)]=eta*np.multiply(np.reciprocal(np.sqrt(vw_hat+epsilon)),(beta1*mw_hat+(1-beta1)*grads[\"dw\"+str(iq)]))*(1/(1-np.power(beta1,t+1)))\n",
        "          update_history[\"b\"+str(iq)]=eta*np.multiply(np.reciprocal(np.sqrt(vb_hat+epsilon)),(beta1*mb_hat+(1-beta1)*grads[\"db\"+str(iq)]))*(1/(1-np.power(beta1,t+1)))\n",
        "\n",
        "      #Updating the parameters once every one batch with the update_history\n",
        "      for j in range(len(layers)-1,0,-1):\n",
        "          parameters[\"w\"+str(j)] = (1-((eta*lambd)/mini_batch_size))*parameters[\"w\"+str(j)] - update_history[\"w\"+str(j)]\n",
        "          parameters[\"b\"+str(j)] = parameters[\"b\"+str(j)] - update_history[\"b\"+str(j)]\n",
        "\n",
        "    #Calculating train loss and accuracies\n",
        "    res = predict(X_train,y_train,parameters, activation, layers)\n",
        "    train_err = loss_calc(loss_function, y_train, res, lambd, layers, parameters )\n",
        "    train_acc = calc_accuracy(res, y_train)\n",
        "    train_errors_list.append(train_err)\n",
        "    train_acc_list.append(train_acc)\n",
        "\n",
        "    #Calculating validation loss\n",
        "    res = predict(X_val, y_val, parameters, activation, layers)\n",
        "    val_err = loss_calc(loss_function, y_val, res, lambd, layers, parameters )\n",
        "    val_acc = calc_accuracy(res,y_val)\n",
        "    val_errors_list.append(val_err)\n",
        "    val_acc_list.append(val_acc)\n",
        "\n",
        "    if(wandb_log==True):\n",
        "      # Logging the values into wandb\n",
        "      log_dict = {\"Train_Accuracy\": train_acc, \"Validation_Accuracy\": val_acc, \\\n",
        "                  \"Train_Loss\": train_err, \"Validation_loss\": val_err, \"epoch\": t}\n",
        "                  \n",
        "      wandb.log(log_dict)\n",
        "\n",
        "  \n",
        "\n",
        "\n",
        "  return parameters, train_acc_list, val_acc_list\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "FMxarSbY9ru4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3> Template for adding new Optimization function</h3>\n"
      ],
      "metadata": {
        "id": "-8pCmbDn0cJ6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def new_optimization_function_name(X_train, y_train, eta, max_epochs, layers, mini_batch_size, lambd, loss_function, activation, parameters,wandb_log=False ):\n",
        "    \n",
        "  '''This is a template for defining and integrating new optimization\n",
        "  functions in our code'''\n",
        "\n",
        "  # Declaring an empty dicitonary for gradients and update history \n",
        "  grads={}\n",
        "\n",
        "\n",
        "  \n",
        "  '''Declare the dictionaries and other data structures as per\n",
        "  the requirement of the optimization function '''\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "  # Declaring various lists to store the loss and accuracies\n",
        "  train_errors_list = []\n",
        "  val_errors_list = []\n",
        "  train_acc_list = []\n",
        "  val_acc_list = []\n",
        "  \n",
        "\n",
        "\n",
        "\n",
        "  '''Initialize the data structures to appropriate intial values as per\n",
        "  the requirement of the optimization function'''\n",
        "  \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "  #iterate till max epochs\n",
        "  for t in tqdm(range(max_epochs)):\n",
        "\n",
        "\n",
        "    #iterate over all batches\n",
        "    for i in range(0, len(X_train), mini_batch_size):\n",
        "\n",
        "      grads.clear()\n",
        "\n",
        "      # Divide the data into batches and get the current batch\n",
        "      X = X_train[i:i + mini_batch_size]\n",
        "      Y = y_train[i:i + mini_batch_size]\n",
        "      \n",
        "      # Collect the gradients using the current batch of points\n",
        "      grads=grad_calculate_batchwise(X,Y,parameters,activation,layers,loss_function)\n",
        " \n",
        "\n",
        "      #Updating the parameters once every one batch with the update_history\n",
        "      for j in range(len(layers)-1,0,-1):\n",
        "\n",
        "\n",
        "          '''write the paramter update rule for the parameters of the network\n",
        "          as per the requirements of the optimization function'''\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    #Calculating train loss and accuracies\n",
        "    res = predict(X_train,y_train,parameters, activation, layers)\n",
        "    train_err = loss_calc(loss_function, y_train, res, lambd, layers, parameters )\n",
        "    train_acc = calc_accuracy(res, y_train)\n",
        "    train_errors_list.append(train_err)\n",
        "    train_acc_list.append(train_acc)\n",
        "\n",
        "    #Calculating validation loss\n",
        "    res = predict(X_val, y_val, parameters, activation, layers)\n",
        "    val_err = loss_calc(loss_function, y_val, res, lambd, layers, parameters )\n",
        "    val_acc = calc_accuracy(res,y_val)\n",
        "    val_errors_list.append(val_err)\n",
        "    val_acc_list.append(val_acc)\n",
        "\n",
        "    if(wandb_log==True):\n",
        "      # Logging the values into wandb\n",
        "      log_dict = {\"Train_Accuracy\": train_acc, \"Validation_Accuracy\": val_acc, \\\n",
        "                  \"Train_Loss\": train_err, \"Validation_loss\": val_err, \"epoch\": t}\n",
        "                  \n",
        "      wandb.log(log_dict)\n",
        "\n",
        "  \n",
        "\n",
        "\n",
        "  return parameters, train_acc_list, val_acc_list\n",
        "\n",
        "'''After defining the optimization function, please add the function name in \n",
        "train function if you want to perform hyperparameter sweeps'''"
      ],
      "metadata": {
        "id": "BiTvyXfO0jAH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h2>Fit</h2>\n",
        "Function to train the neural network"
      ],
      "metadata": {
        "id": "8Gahdla9-I9b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def fit(X_train, y_train, layer_sizes,wandb_log, learning_rate = 0.0001, initialization_type = \"random\", activation_function = \"sigmoid\", loss_function = \"cross_entropy\", mini_batch_Size = 32, max_epochs = 5, lambd = 0,optimization_function = mini_batch_gd): \n",
        "\n",
        "  ''' This function is actually used to run the training process. It first\n",
        "  initializes the network and calls the respective optimization function\n",
        "  to train the model. It also prints the train and validation losses for each\n",
        "  training epoch \n",
        "  \n",
        "  The input is the training data, true labels, overall architecture of the network,\n",
        "  learning rate, initialization type, activation function, loss function, mini \n",
        "  batch size, maximum epochs, lambda for regularization, and the optimization \n",
        "  function to be used \n",
        "  \n",
        "  The output is the trained models parameters'''\n",
        "\n",
        "  parameters = nn_init(init_type = initialization_type, layer_sizes = layer_sizes)\n",
        "  parameters, train_errors_list, val_errors_list = optimization_function(X_train, y_train,learning_rate, max_epochs, layer_sizes, mini_batch_Size, lambd, loss_function, activation_function, parameters,wandb_log)\n",
        "  \n",
        "  print(\"Training Accuracy:\",train_errors_list[-1])\n",
        "  print(\"Validation Accuracy:\",val_errors_list[-1])\n",
        "\n",
        "  return parameters\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "aDYOzjtnt48n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h2> Predict </h2>\n",
        "Function to predict the labels after training the model"
      ],
      "metadata": {
        "id": "6LQFPsXT9u15"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def predict(X ,y,parameters,activation,layer_sizes):\n",
        "\n",
        "  '''This function is used to simply take a models parameters\n",
        "      and run the data points using forward prop, and return the outputs \n",
        "      of all the input data points\n",
        "      \n",
        "      The input is the data points, true labels, parameters of the network,\n",
        "      Choice of the activation function, overall architecture of the network\n",
        "      \n",
        "      The output is the list of all the predicted values for all the data points'''\n",
        "\n",
        "  result = []\n",
        "\n",
        "  #Iterate over all the data points in the given data set\n",
        "  for i in range(len(X)):\n",
        "\n",
        "    #forward propagate the data point\n",
        "    h,a,y_hat = forward_prop(X[i], y[i], parameters, activation, layer_sizes)\n",
        "\n",
        "    #converting y_hat to a 1d array to match with the y\n",
        "    y_hat = y_hat.flatten()\n",
        "\n",
        "    #storing the result into the result list\n",
        "    result.append(y_hat)\n",
        "  \n",
        "  return result\n"
      ],
      "metadata": {
        "id": "OVd1N86DZ06N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>4. HyperParameter Sweeps</h2>"
      ],
      "metadata": {
        "id": "OYdDzZRu-Si9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used: <br>\n",
        "https://towardsdatascience.com/hyperparameter-tuning-a-practical-guide-and-template-b3bf0504f095 <br>\n",
        "https://docs.wandb.ai/guides/sweeps "
      ],
      "metadata": {
        "id": "z4DYfrOOHSYo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_val, X_test, y_train, y_val, y_test, labels = prepare_data()"
      ],
      "metadata": {
        "id": "Hsb7BQqXWN-5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5cbb3e7e-d73f-48c0-f582-5107f0c82e88"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of data points in train data (initially) -  60000\n",
            "Number of data points in test data (initially) -  10000\n",
            "Shape of each image - 28x28\n",
            "shape of each image (1D) -  784\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def train():\n",
        "\n",
        "  '''This function is used to exploit the wandb hyperparameter sweep \n",
        "  function to get the best hyperparameters.\n",
        "\n",
        "  It takes in no inputs and gives no outputs.\n",
        "  \n",
        "  Instead it logs everything into the wandb workspace'''\n",
        "\n",
        "  #Declaring the dictionary with the default hyperparameters\n",
        "  config_defaults = {\n",
        "      'number_hidden_layers': 2,\n",
        "      'number_neurons': 32,\n",
        "      'learning_rate': 0.001,\n",
        "      'initialization_type': \"xavier\",\n",
        "      'activation_function':'sigmoid',\n",
        "      'mini_batch_size' : 64,\n",
        "      'max_epochs': 5,\n",
        "      'lambd': 0,\n",
        "      'optimization_function': \"adam\"\n",
        "      \n",
        "  }\n",
        "\n",
        "  # Initializing the wandb run\n",
        "  wandb.init(config=config_defaults)\n",
        "  config = wandb.config\n",
        "\n",
        "\n",
        "  # Constructing the layer_sizes i.e., the architecture of our neural network\n",
        "  layer_sizes = [784]\n",
        "  for i in range(config.number_hidden_layers):\n",
        "    layer_sizes = layer_sizes + [config.number_neurons]\n",
        "  layer_sizes  = layer_sizes + [10]\n",
        "\n",
        "  #Collecting all the hyperparameters from the wandb run\n",
        "  learning_rate = config.learning_rate\n",
        "  initialization_type = config.initialization_type\n",
        "  activation_function = config.activation_function\n",
        "  loss_function = \"cross_entropy\"\n",
        "  mini_batch_size = config.mini_batch_size\n",
        "  max_epochs = config.max_epochs\n",
        "  lambd = config.lambd\n",
        "  opt_fun = config.optimization_function\n",
        "\n",
        "  #Calling the respective hyperparameters\n",
        "  if opt_fun == \"adam\":\n",
        "    optimization_function = adam\n",
        "  elif opt_fun == \"nadam\":\n",
        "    optimization_function = nadam\n",
        "  elif opt_fun == \"mini_batch_gd\":\n",
        "    optimization_function = mini_batch_gd\n",
        "  elif opt_fun == \"momentum_gd\":\n",
        "    optimization_function = momentum_gd\n",
        "  elif opt_fun == \"nesterov_gd\":\n",
        "    optimization_function = nesterov_gd\n",
        "  elif opt_fun == \"rmsprop\":\n",
        "    optimization_function = rmsprop\n",
        "  else:\n",
        "    print(\"Wrong optimization function\")\n",
        "    exit()\n",
        "\n",
        "\n",
        "  #Forming meaningful run name using the hyperparameters\n",
        "  name_run = str(learning_rate) + \"_\" + initialization_type[0] + \"_\" + \\\n",
        "  activation_function[0] + \"_\" + str(mini_batch_size) + \"_\" + str(max_epochs) + \\\n",
        "  \"_\" + str(lambd) + \"_\" + opt_fun[:4]\n",
        "\n",
        "  wandb.run.name = name_run\n",
        "  wandb_log=True\n",
        "  #Calling the fit function to train the neural network with the current hyperparameters\n",
        "  parameters = fit(X_train, y_train, layer_sizes,wandb_log, learning_rate, initialization_type, activation_function, loss_function, mini_batch_size, max_epochs, lambd, optimization_function)\n",
        "\n",
        "  \n",
        "  wandb.run.save()\n",
        "  wandb.run.finish()\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "bLaUCuD_Jsxv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sweeper(entity_name,project_name):\n",
        "  #Declaring the dictionary of all choices for the hyperparameters.\n",
        "  hyperparameters = {\n",
        "      \"learning_rate\":{\n",
        "        'values': [0.001, 0.0001]\n",
        "      },\n",
        "\n",
        "      \"number_hidden_layers\": {\n",
        "          'values' : [3, 4, 5]\n",
        "      },\n",
        "\n",
        "      \"number_neurons\": {\n",
        "        'values': [32, 64, 128]\n",
        "      },\n",
        "\n",
        "      \"initialization_type\": {\n",
        "          'values' : [\"xavier\", \"random\"]\n",
        "      },\n",
        "\n",
        "      \"activation_function\": {\n",
        "          'values': [\"sigmoid\", \"tanh\", \"relu\"]\n",
        "      },\n",
        "\n",
        "      \"mini_batch_size\": {\n",
        "          'values': [16,32,64,128]\n",
        "      },\n",
        "\n",
        "      \"max_epochs\": {\n",
        "          'values': [5, 10, 20]\n",
        "      },\n",
        "\n",
        "      \"lambd\": {\n",
        "          'values': [0, 0.0005, 0.5]\n",
        "      },\n",
        "\n",
        "      \"optimization_function\": {\n",
        "          'values': [\"mini_batch_gd\", \"momentum_gd\", \"nesterov_gd\", \"rmsprop\", \"adam\", \"nadam\"]\n",
        "      }\n",
        "\n",
        "  }\n",
        "\n",
        "\n",
        "  #Using bayes method for hyperparameter sweeps to curb the unnecessary configurations\n",
        "  sweep_config = {\n",
        "      'method' : 'bayes',\n",
        "      'metric' :{\n",
        "          'name': 'Validation_Accuracy',\n",
        "          'goal': 'maximize'\n",
        "      },\n",
        "      'parameters': hyperparameters\n",
        "  }\n",
        "\n",
        "  sweep_id = wandb.sweep(sweep_config, entity=entity_name, project=project_name)\n",
        "  wandb.agent(sweep_id, train)"
      ],
      "metadata": {
        "id": "JBZOwcGOip9K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sweeper(entity_name,project_name)"
      ],
      "metadata": {
        "id": "CPV1eT1ZpN1v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>7. Confusion Matrix for Test Dataset"
      ],
      "metadata": {
        "id": "2CrEVrWQPbwm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "From the above experiments, we have chosen the best configuration of hyperparameters and executed the prediction using the test data to finally evaluate the model.<br> <br>\n",
        "We have plotted the confusion matrix to get a clear picture of how the model is performing."
      ],
      "metadata": {
        "id": "FDYpsB3Juu7I"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resources used: https://medium.com/@dtuk81/confusion-matrix-visualization-fc31e3f30fea"
      ],
      "metadata": {
        "id": "5NJXD4lIH2Z8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_confmat_wandb(entity_name,project_name):\n",
        "    # NOTE: Change these values according to the observations\n",
        "    # made from the parallel coordinate plot from wandb.ai. These are the best \n",
        "    # hyperparameters as observed.\n",
        "    \n",
        "  config_final_test = {\n",
        "        'number_hidden_layers': 5,\n",
        "        'number_neurons': 128,\n",
        "        'learning_rate': 0.0001,\n",
        "        'initialization_type': \"xavier\",\n",
        "        'activation_function':'relu',\n",
        "        'mini_batch_size' : 32,\n",
        "        'max_epochs': 20,\n",
        "        'lambd': 0.0005,\n",
        "        'optimization_function': \"nadam\",\n",
        "        'loss_function': \"cross_entropy\"\n",
        "        \n",
        "  }\n",
        "\n",
        "  #initializing the run\n",
        "  wandb.init(config=config_final_test, project = project_name, entity=entity_name)\n",
        "  config = wandb.config\n",
        "\n",
        "\n",
        "  #Forming the layer_sizes i.e., the architecture of our neural network\n",
        "  layer_sizes = [784]\n",
        "  for i in range(config.number_hidden_layers):\n",
        "      layer_sizes = layer_sizes + [config.number_neurons]\n",
        "  layer_sizes  = layer_sizes + [10]\n",
        "\n",
        "  #storing the hyperparameters in local variables\n",
        "  learning_rate = config.learning_rate\n",
        "  initialization_type = config.initialization_type\n",
        "  activation_function = config.activation_function\n",
        "  loss_function = config.loss_function\n",
        "  mini_batch_size = config.mini_batch_size\n",
        "  max_epochs = config.max_epochs\n",
        "  lambd = config.lambd\n",
        "  opt_fun = config.optimization_function\n",
        "\n",
        "  #Choosing the correct optimization function\n",
        "  if opt_fun == \"adam\":\n",
        "      optimization_function = adam\n",
        "  elif opt_fun == \"nadam\":\n",
        "      optimization_function = nadam\n",
        "  elif opt_fun == \"mini_batch_gd\":\n",
        "      optimization_function = mini_batch_gd\n",
        "  elif opt_fun == \"momentum_gd\":\n",
        "      optimization_function = momentum_gd\n",
        "  elif opt_fun == \"nesterov_gd\":\n",
        "      optimization_function = nesterov_gd\n",
        "  elif opt_fun == \"rmsprop\":\n",
        "      optimization_function = rmsprop\n",
        "  else:\n",
        "      print(\"Wrong optimization function\")\n",
        "      exit()\n",
        "\n",
        "  #set this to TRUE to log the data to wandb\n",
        "  wandb_log=True\n",
        "\n",
        "  #calling the fit function to train the model using the best hyperparamters obtained from above\n",
        "  parameters_for_test = fit(X_train, y_train, layer_sizes,wandb_log, learning_rate, initialization_type, activation_function, loss_function, mini_batch_size, max_epochs, lambd, optimization_function)\n",
        "  res = predict(X_test,y_test, parameters_for_test, activation_function, layer_sizes)\n",
        "    \n",
        "  wandb.run.name = \"Confusion Matrix\"\n",
        "\n",
        "  # Converting the one hot encoded vectors back to label_id's\n",
        "  y_t=[]\n",
        "  for k in range(len(y_test)):\n",
        "      y_t.append(y_test[k].argmax())\n",
        "\n",
        "  y_pred=[]\n",
        "  for k in range(len(res)):\n",
        "      y_pred.append(res[k].argmax())\n",
        "\n",
        "  #calculating the test accuracy using the test data\n",
        "  test_accuracy=calc_test_accuracy(y_pred,y_t)\n",
        "  print(test_accuracy)\n",
        "  wandb.log({\"conf_mat\":wandb.plot.confusion_matrix(preds=y_pred,y_true=y_t,class_names=labels),\"Test Accuracy\": test_accuracy}) \n",
        "    \n",
        "\n",
        "  wandb.run.save()\n",
        "  wandb.run.finish()\n",
        "\n",
        "  return y_pred,y_t\n",
        "    "
      ],
      "metadata": {
        "id": "ADKEbRF9qGDX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred,y_t=plot_confmat_wandb(entity_name,project_name)"
      ],
      "metadata": {
        "id": "urk4FqnL9a-N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_local_confmat(y_true,y_pred):\n",
        "  labs=np.unique(y_true)\n",
        "  cmat=np.zeros((len(labs),len(labs)))\n",
        "\n",
        "  for i in range(len(labs)):\n",
        "    for j in range(len(labs)):\n",
        "      cmat[i,j]=np.sum((y_true==labs[i])&(y_pred==labs[j]))\n",
        "  fig, ax = plt.subplots(figsize=(10,10))\n",
        "  ax.set_title(\"Confusion matrix for Test dataset\")\n",
        "  sn.heatmap(cmat,annot=True,xticklabels=labels,yticklabels=labels)\n",
        "  plt.show()\n"
      ],
      "metadata": {
        "id": "RtPhg15lqF_v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plot_local_confmat(y_t, y_pred)"
      ],
      "metadata": {
        "id": "u8xg8pCZQ76m",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 607
        },
        "outputId": "60552ee6-a81b-43ed-e2f6-d5402180a45b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAJOCAYAAACQvoyFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3gVRcPG4d+k0EPvRQGxoCKCdFDpiBBQUFABsYLIKyqKL1g+RfG1Yi8IovSO9F6V3sFCb1JC6F0gbb4/ziYmkHISchp57uvai2w5O8/OOeRMZmd3jbUWERERkUAQ5OsAIiIiIu5Sw0VEREQChhouIiIiEjDUcBEREZGAoYaLiIiIBAw1XERERCRgqOEiAcMYk9MYM80Yc9oYM/4q9tPBGDM3M7P5ijHmbmPMtgy+9mZjzEZjzFljTI/MzuZtxph3jDEjfJ1DRDxLDRfJdMaYx4wxa40x54wxh4wxs4wx9TJh1w8BxYBC1tqHM7oTa+1Ia23TTMjjUcYYa4ypkNo21tol1tqbM1jEa8Aia22YtfarDO4DAGPMAOf9PmeMiTLGRCean5WB/T1hjFl6NZnS2P8QY0w/T+3f2+WIZCVquEimMsb0BL4A/oerkXEd8B3QOhN2fz2w3Vobkwn7CnjGmJCr3MX1wF+ZUba19jlrbR5rbR5c7/3Y+HlrbfOrzCki8i9rrSZNmTIB+YBzwMOpbJMdV8Mmwpm+ALI76+oDB4BXgCPAIeBJZ11fIAqIdsp4GngHGJFo32UBC4Q4808Au4GzwB6gQ6LlSxO9rg6wBjjt/Fsn0brFwHvAMmc/c4HCKRxbfP7XEuV/ALgf2A6cAF5PtH0NYAVwytn2GyCbs+4351jOO8fbPtH+/wtEAsPjlzmvucEpo6ozXxI4CtRPJutCIBa46Oz/Juf9G+a85m/gTSAoUZ0tAz4HjgP9UnmPL39fagHLnePclDhPcu8RUNHJFetkO5VCOeWAX53XznPqL3G54516Ou3U523O8i64PkdRzv6nOct7A7uc/W0GHky0rwpOWaeBY7gaZvHrbnHKPwFsA9qlVo4mTZqubvJ5AE3XzgTcB8TgNBxS2OZdYCVQFCjifKG956yr77z+XSAU1xf+P0ABZ/3lX4iXz5fFabgAuYEzwM3OuhKJvriewGm4AAWBk0An53WPOvOFnPWLnS+zm4CczvyHKRxbfP7/c/I/i6sRMAoIA24DLgDlnO3vwvWlHuJk3wK8lGh/FqiQzP4/wtUAzEmihouzzbPOl24uYA7waSrvxWLgmUTzw4ApTtayuBpbTyeqsxjgBSdvzlT2m/C+AKVwNXTux9XD28SZL+Lue5RKOSuAz5y6uAdXgyPx5+Ep51jiG8sbE60bwmWNL+BhXI29IFwNxfNACWfdaOANZ10OoJ6zPDewH3jSqZcquBo2t6ZUjiZNmq5u0qkiyUyFgGM29VM5HYB3rbVHrLVHcfWkdEq0PtpZH22tnYnrL9WMjuGIA243xuS01h6y1iZ3WqQFsMNaO9xaG2OtHQ1sBcITbfOztXa7tfYCMA64M5Uyo4H3rbXRwBigMPCltfasU/5moDKAtXadtXalU+5e4AfgXjeO6W1r7SUnTxLW2kHATmAVrobAG2nsDwBjTDDwCNDHyboX6E/S9ybCWvu1k/eKslPQEZhprZ1prY2z1s4D1uJqyMQfT1rvUXJ5rwOqA285dfEbMC3xNtban5xjuYSrMVXZGJMvpX1aa8dbayOcnGOBHbh6xcD1vl4PlLTWXrTWxo+/aQnstdb+7NTLBmAirkaQiHiAGi6SmY4DhdMYe1ES12mIeH87yxL2cVnD5x8gT3qDWGvP4/qr+TngkDFmhjHmFjfyxGcqlWg+Mh15jltrY52f47/cDydafyH+9caYm4wx040xkcaYM7jGhhROZd8AR621F9PYZhBwO/C186XtjsK4eokuf28S18N+N/eV2PXAw8aYU/ETUA9XT4a771FySgInnX0kzgu4GmLGmA+NMbucut3rrEqxfo0xjztXWcXnvD3R9q8BBlhtjPnLGPNUouOrednxdQCKu3kcIpJOarhIZloBXMI1riMlEbh+2ce7zlmWEedxnRKJl+TLwlo7x1rbBFfPw1ZcX+hp5YnPdDCDmdLje1y5brTW5gVex/XlmJpUH+dujMmD67TIYOAdY0xBN7Mc499ehXiX10NGHiW/Hxhurc2faMptrf0QUn2P0irrEFDAGJP7srzxHsM1ILwxrrE7ZZ3l8fWbZP/GmOudsv+D6zRhfuDP+O2ttZHW2mettSWBrsB3zhVf+4FfLzu+PNbabm4eh4ikkxoukmmstadxje/41hjzgDEmlzEm1BjT3BjzsbPZaOBNY0wRY0xhZ/uM3ntjI3CPMeY65xRAn/gVxphixpjWzhfbJVynnOKS2cdM4CbnEu4QY0x74FZgegYzpUcYrjEe55yehm6XrT8MlE/nPr8E1lprnwFmAAPceZHTSzQOeN8YE+Z8kfck4+9NvBFAuDGmmdMLksMYU98YUzqN9+gwUNoYky2FvH/jOuXU1xiTzbncPvHpvTBnn8dxNW7/d9kuLq/b3LgaGUcBjDFP4upxwZl/2BhT2pk96Wwbh+tzcpMxppPzWQ81xlQ3xlRMoRwRuUpquEimstb2x/WF9yauL4H9uP6Knexs0g/XF87vwB/AemdZRsqaB4x19rWOpI2NICdHBK6rPe7lyoYB1trjuMYpvILrS+41oKW19lhGMqXTq7h6Bs7i+mt/7GXr3wGGOqcg2qW1M2NMa1wDpOOPsydQ1RjTwc08L+DqxdoNLMU1qPgnN1+bLGvtflw9H6/z7+ehF673J7X3aCGuS7UjjTEpvRePATWd176Na3BxvGG4Th0dxDWuaOVlrx0M3OrU7WRr7WZcY3pW4GpsVMJ1FVW86sAqY8w5YCrworV2t7X2LNAU1/igCFynFeMHT19RTuq1JSLuMNaqJ1NEREQCg3pcREREJGCo4SIiIiIBQw0XERERCRhquIiIiEjAuNqHtKXpwpxvAmr0b97wD3wdId2CggKv/Zk/e+60N/IjJy6c9XWEdAuo/3iO8vlK+DpCuuw+fcjXEdItOMB+X8TGJXcXA/8XE3UwrXsyZaroY7u99l8+tHB5rx7b5QLrEywiIiJZmhouIiIiEjA8fqpIREREPCwuNu1trhFuNVyc227fguu0+TZrbZRHU4mIiIgkI82GizGmBa7nnezC9cCxcsaYrtbaWZ4OJyIiIm6wgTmIOSPc6XHpDzSw1u4EMMbcgOvhbWq4iIiIiFe503A5G99ocezG9VA4ERER8QcBetl4RrjTcFlrjJmJ65H3FngYWGOMaQNgrf3Fg/lEREREErjTcMmB6zHv9zrzR4GcQDiuhowaLiIiIj5kNcblX9baJ70RRERERCQtad6AzhhT2hgzyRhzxJkmGmNKeyOciIiIuCEuznuTj7lz59yfgalASWea5iwTERER8Sp3xrgUsdYmbqgMMca85KlAIiIikk5ZaIyLOz0ux40xHY0xwc7UETju6WAiIiIil3On4fIU0A6IBA4BDwFPeDCTiIiISLLcOVVU2lrbKvECY0xdYL9nIomIiEi6ZKGHLLrT4/K1m8tEREREPCrFHhdjTG2gDlDEGNMz0aq8QLCng4mIiIibstDg3NROFWUD8jjbhCVafgbXOBcRERERr0qt4VIX1/1bhlhr//ZSHhEREUkvP7gxnLek1nDZBbwIVDbGbAJmAXOttSe9kkxERETkMik2XKy1Y4GxAMaYKsB9wC/GmGBgPjDbWrvaKylFREQkRVnpIYvuPKsou7V2g7X2A2ttA6AlsBt4Jr2FDV+0gTb/G0nbD0bSe8hsLkXHJF2/cANt3h/Bwx+Ooss3k4g4cSa9RVzh9PmLdP12MuHvDaPrt5M5889FAGas2cbDH47ioQ9G8fhn49l28OhVl5WaoKAg1qyew+RJQz1aztX44YdP2b9vA+vXzU9Y9vbbr7J2zVxWr5rNjOkjKVGimA8TXmnN7wtYvHwqC5ZMYs7iCQD833u9WLpmJouWTeHnEV+TN19YGnvxnUD4XCTWrGl9/vrzN7ZuXsprvbr7Ok6KHu/yCNN/G8uMJWPp3PXRJOue6taB7UfXUqBgPh+lS5u/13Nyvys++N8b/L5pEWvXzGXc2EHky5fXhwnT5u91LClz53LoFYlnrLVngJ7W2i7pKejwqXOM/vV3Rr3anol9OhAbZ5m9fkeSbW4pXYSRvdozvvdjNK5cgS+mLHN7/2t2HOCtEfOuWP7T/HXUvKk00956nJo3leaneesAKFUoL4N7tGFCn8focl913huzKD2Hk249XniGLVt3pL2hDw0fPp7wVp2SLPvsswFUq96UGjXvY+bM+bzx+os+SpeyNi0fp9HdD9KsvmvM+K+LlnNvrXAa1G3Nrl176dEzXR9VrwqEz0W8oKAgvvryfVqGd6RS5Qa0b/8AFSve6OtYV7jxlhto1/FBHmr2OK3qP0aDJvW4rpzrubDFSxajboNaHNx/yMcpUxYI9Zzc74oFC5dQpWpjqlVvyo4du/26MRAIdZxuesgiGGOKG2PuAnIaY6oYY6o6U30gV0YKi42L41J0DDGxcVyMjqFI3txJ1le/qTQ5s4UCcEfZ4hw+dT5h3ZAF63ns07E8/OEovpu50u0yF/+xm/AaFQEIr1GRRX/sBuDO8iXImytHorLOZeSQ3FKqVAmaN2/ETz+N9lgZmWHp0lWcPHkqybKzZ/+tl1y5c2Gtt1Ol368LlxEb67oZ07o1myhZsriPEyUvUD4X8WpUr8KuXXvZs2cf0dHRjBs3hVbhzXwd6wo33FSWTev/5OKFS8TGxrJ6+XqatmgIwOv9evJJ36+wfvxBDoR6Tu53xfz5vyX8v1u1egOlSpfwRTS3BEIdS8pSG5zbDNet/UsD/QHjLD8DvJ7egorlz8PjDatw39tDyBEaTK1brqNOxetS3H7Syr+od+v1ACzfso99R08x8pV2WAsvDprOup0HuatCqTTLPX72H4rkczWQCufNxfGz/1xZ1orN1Kt4fXoPyW39+/elT59+5AnL47EyPKlv39fo0KEtZ06fpWmzdr6OcxnL2MmDsRaG/zyW4UPGJVn7WMe2TP5lpo+ypS7QPhclSxVn/4GIhPkDBw9Ro3oVHyZK3o4tu3j59efJXyAfFy9e5N7Gdflz0xYa3Xcvhw8dYetf/t3DFSj1nJonOrdj/IRpvo6Romuhjq+Qhca4pDY4d6gxZjjwqLV2ZHp2aozpAnQB+LrHIzx9f13O/HORxX/sYcbbnQnLlY1eP81ixpqttKh+yxWvn7FmK5v3HWFwj7YArNy2jxVb99H+4zEAXLgUzb6jp7irQik69h9HVEwsFy5Fc/qfi7T7yPXX60ut6lDnssaIMQaT0P5yWbP9AJNXbubnl9qm5xDddv/9jTl65BjrN/zBPffU9kgZnvb22x/z9tsf06tXd7p1e4L33vvM15EShDd7jMhDRyhcuCDjJv/Eju27Wbl8LQAvvdqVmJgYJo7zv1+g18Lnwl/t2rGXQV8P46fx33Dhnwts+XM72bKF8txLT/Lkw/57+uJa8d//vkBMTCyjR0/ydRS5RqX6rCJrbZwx5mUgXQ0Xa+1AYCDAhTnfWICV2/ZTqlBeCoblBKBR5RvYuCfyiobLym37+HHuWgb3aEO20OD4/fF0k2o8VPf2K8oa8YqrB2DNjgNMXbWF9zo2SbK+UFgujp4+T5F8uTl6+nxC+QDbDx6j7+gFfNutFflz58QT6tSpRsuWTbnvvobkyJGdvHnDGDrkKzo/0cMj5XnSmDGTmDJ5mF81XCIPHQHg2LETzJw+nyp33cHK5Wtp/9iDNGnWgIdaPeHbgCkIxM9FxMFIypQumTBfulQJIiIifZgoZRNGTmHCyCkA9HzjeY4dPUHj5vWZutj1h03xkkWZtGAkDzXrzLEj/vWw+0Cq58t16vQw9zdvxH3NH/F1lFQFch2nSM8qSmK+MeZVY0wZY0zB+Cm9BZUoEMbveyO5EBWNtZZV2w9QvliBJNts3X+UfmMW8cWzLSkY9u8wmtq3XM/klZv551IU4BroeyKZUz7Juff2ckxbvQWAaau3UL9SeQAOnTjLK4Nn0q9TU64vWiC1XVyVN9/8kHLlq3HjTbXo0PF5Fi1a5tdfTpercEPZhJ/DWzZl27advgtzmVy5cpI7T+6En+s3rMvWzdtp0Kge3V98mscf6caFCxd9nDJ5gfi5WLN2IxUqlKNs2TKEhobSrl1rpk2f6+tYySpY2PV/ukSpYjRt0ZBJY6ZT+9amNLyrFQ3vakVkxBEebNTB7xotEFj1nFjTJvV5pedztH3oKb/9fxcvUOtYXNx5OnR759/EfawWKJ+egiqVLU7jO2/g0Y/HEBwcxC2litC2zu18N2Mlt15XlPqVyvP5lKX8ExVNr59nAa7GzpddWlKn4nXsOXyCxz9zXe6aK3so73dqSkE3rnJ9qsldvPbzbCat3EzJAmF8/GRzAAbOXs2p8xf53/jFAIQEBTGqV/tU9nTtGzbsG+65uxaFCxdk187VvNevP/c1a8hNN91AXFwc+/Yd4D8vpHt4k8cUKVqIn0d8A0BwSDCTJkxn0YKlrNwwh2zZsjFu8k8ArFu7iddefseHSa8NsbGxvPjSm8ycMYrgoCCGDB3L5s3bfR0rWd/8/DH5C+QjJjqGvv/9iLNnPDf4PrMFQj0n97vitV7/IVv2bMycMQqA1avX+9Xvi8QCoY7TLQuNcTGeHl0ff6ooUOQN/8DXEdItKMidjjP/kj977rQ38iMnLpz1dYR0C6j/eI7y+fz3SpTk7D7tv5dVpyQ4wH5fxPrB5bcZERN10KS9Vea5tGWR1/7LZ6/YwKvHdrnUng7d0Fq70BjTJrn11tpfPBdLRERE5EqpnSq6F1gIhCezzgJquIiIiPiDAO2ZyojULod+2/n3Se/FEREREUlZmoNzjTHZgbZA2cTbW2vf9VwsERERcVsWGpzrzlVFU4DTwDrgkmfjiIiIiKTMnYZLaWvtfR5PIiIiIhmThca4uHNd3HJjTCWPJxERERFJQ2qXQ/+B6+qhEOBJY8xuXKeKDGCttXd4J6KIiIikxtqsc8v/1E4VtfRaChERERE3pHY59N8AxpgbgAPW2kvGmPrAHcAw78QTERGRNGWhq4rcGeMyEYg1xlTA9cTnMsAoj6YSERERSYY7VxXFWWtjnFv/f22t/doYs8HTwURERMRNuqooiWhjzKPA48B0Z1mo5yKJiIiIJM+dHpcngeeA9621e4wx5YDhno0lIiIibstCY1zSbLhYazcDPQCMMVWtteuBjzwdTERERORyqd3HJcRaG3PZ4h+Bqp6NJCIiIukSl3Xu45LaGJfVySwzngoiIiIikpbUGi7JNVL6eiqIiIiISFpSG+NSxBjT8/KF8custZ95LJWIiIi4T4NzAQgG8qDTQyIiIuInUmu4HLLWvuu1JCIiIpIxugEdoJ4WERER8TOp9bg0yowCwsI/yIzdeM2FiCW+jpBuuUvd4+sI6XYhJsrXEcQP7Tl9yNcRrnmxWegv8ywlC41xSbHHxVp7wptBRERERNLizi3/RURExJ9loZ40dx6yKCIiIuIX1OMiIiIS6NTjIiIiIuJ/1OMiIiIS4KzVQxZFRERE/I56XERERAKdxriIiIiI+B/1uIiIiAQ63TnXxRgTZIyp460wIiIiIqlJteFirY0DvvVSFhEREZFUuXOqaIExpi3wi7XWejqQiIiIpJMG5ybRFRgPRBljzhhjzhpjzng4l4iIiMgV0uxxsdaGeSOIiIiIZJAG5/7LuHQ0xrzlzJcxxtTwfDQRERGRpNwZ4/IdEAc0BN4DzuEasFvdg7lERETEXVlojIs7DZea1tqqxpgNANbak8aYbB7OJSIiInIFdxou0caYYMACGGOK4OqBEREREX+gMS5JfAVMAooaY94HlgL/82gqERERkWS4c1XRSGPMOqARYIAHrLVbPJ5MRERE3JOFxri4c1XRDcAea+23wJ9AE2NMfo8nExEREbmMO6eKJgKxxpgKwA9AGWCUR1OJiIiI++LivDf5mDsNlzhrbQzQBvjGWtsLKOHZWCnLnj07K5ZNZ93aeWzauJC3/+8Vj5U1fNxkHuj4HK07dGX42ElXrF+9/ndqNW1L287dadu5O9//NPKqy4yKiuKVtz6gebunePTZlzh46DAAy1evp91TL/Bgp260e+oFVq3beNVlpaZHj2fYuGEBG9bPZ/iwb8iePbtHy8uI7NmzsejXSSxbOYNVa2bz+hsvAfDNdx+ybOUMlq+aybAR35I7dy4fJ01ZUFAQa1bPYfKkob6O4pZmTevz15+/sXXzUl7r1d3XcdymevasQMsLgZlZXNxpuEQbYx4FHgemO8tCPRcpdZcuXaJx03bcVa0Jd1VrSrOm9alZo2qml7Nj914mTp3N6B+/YOLQ7/h1+Wr2HYi4YruqlW9n4tBvmTj0W7o91cHt/R88dJgn/vPaFct/mT6XvGF5mDXuJzq1f4DPvvsJgAL58/LNR+8wafj3vP/mK/R599OMH1waSpYsTvfuT1GrdguqVG1McHAw7dq18lh5GXXpUhQt7+9A3VotqFu7JY2b3EP16nfS57/9qFurBXVq3s+BAxF0ee5xX0dNUY8XnmHL1h2+juGWoKAgvvryfVqGd6RS5Qa0b/8AFSve6OtYblE9e06g5YXAzJwmG+e9ycfcabg8CdQG3rfW7jHGlAOGezZW6s6f/weA0NAQQkJD8cSzH3fv3U+l224mZ44chIQEU+3OSsz/dZnbr582ZyGPPPMibTt3p+/HXxEbG+vW6xYuWUHr+xsD0LT+3axatxFrLRVvqkDRIoUAqFDuei5eukRUVFT6D8xNIcEh5MyZg+DgYHLmyskhp+fH3yT9LIRgreXs2XMJ63PkyOGRz0dmKFWqBM2bN+Knn0b7OopbalSvwq5de9mzZx/R0dGMGzeFVuHNfB0rTapnzwq0vBCYmeVfaTZcrLWbrbU9rLWjnfk91tqPPB8tZUFBQaxdM5dDB39nwYLfWL1mQ6aXUaH89azf9BenTp/hwsWLLFmxhsjDR6/YbtOfW2jT+Xmee+Utdu7+G4Bde/cxe8GvDB/Qn4lDvyUoKIjpcxe5Ve6Ro8cpXrQwACEhweTJnYtTp5M+03Le4qXcenMFsmXzzH0AIyIi+fyLH9i1cxX7/l7PmdNnmT//N4+UdbWCgoJYumI6u/auYdHCZaxduwmA7wZ8zM49q7nppvL88L1/nh7o378vffr0I84Pzhm7o2Sp4uxP1Ot44OAhSpYs7sNE7lE9e1ag5YXAzJymLDTGJc3LoY0xe3BuPpeYtbZ8Kq/pAnQBMMH5CArKfTUZrxAXF0e16k3Jly8vE8cP5rbbbuavv7Zlahk3lL2Opzo8TJeX3yBnjhzcfGN5goKStvNuvfkG5k0cSq5cOflt+Wp69HmXmWMHs2rtRjZv3ckjT78IuE5vFSzguhCrR593ORhxmOiYaA4dPkrbzq5zqx3btebBFk3TzLVz99989t1PDPz8/Uw93sTy589HeMum3HRzbU6dOsOY0QN47NE2jBr9i8fKzKi4uDjq1W5JvnxhjBw9gIq33sSWzdt5/rnXCAoK4tP+79DmoZaMHD7B11GTuP/+xhw9coz1G/7gnntq+zrONUv1LHLtcefOudUS/ZwDeBgomNoLrLUDgYEAIdlKeayf/vTpMyz+dZlrkFUmN1wA2oY3o63TffjFgCEJPSHx8uT+t0F2T50a9Ov/LSdPncZaS6vmjXm525NX7POrD/4PcI1xeeP9/gz55uMk64sWKUTkkWMUL1qEmJhYzp3/h/z58gIQeeQoL77+Hv9761WuK10yU481sUYN67F3736OHTsBwOTJs6hV+y6/bLjEO336LEt+W0njJvewZfN2wNWomTBhGi+93NXvGi516lSjZcum3HdfQ3LkyE7evGEMHfIVnZ/o4etoKYo4GEmZRJ+70qVKEBER6cNEaVM9e16g5YXAzBxIjDEvA8/g6vT4A9eQkxLAGKAQsA7oZK2NMsZkB4YBdwHHgfbW2r2p7d+dU0XHE00HrbVfAC2u4piuSuHCBcnnfJHnyJGDxo3uYdu2XR4p6/jJUwAcijzCgl+XcX+T+knWHzt+ImH8xB+btxFnLfnz5aVWtTuZt3hpwutPnzlLRKR7Y0Qa1KvFlJnzAZi7eAk176qMMYYzZ8/xfK+3eem5J6l6x22ZdITJ27c/gpo1q5AzZw5Xpgb12Lp1p0fLzIhChQuSL18YADlyZKdBw3rs2L6b8uWvT9jm/haN2b7dM5+Pq/Hmmx9Srnw1brypFh06Ps+iRcv8+ssUYM3ajVSoUI6yZcsQGhpKu3atmTZ9rq9jpUr17HmBlhcCM3Oa/GRwrjGmFNADqGatvR0IBh4BPgI+t9ZWAE4CTzsveRo46Sz/3NkuVe6cKkp8yU4Qrh4Yd3pqPKJEiWL8NPgLgoODCAoKYsKEacxwvugz28uv9+PUmTOEhITwxivPkzcsD2MnzQCg/YMtmLtoKWMnzSA4JJgc2bLxSd/eGGO4odz1vPDs43R56Q3ibByhISG80fN5ShYvlmaZbVo2o897n9C83VPkyxvGJ317AzB64jT2H4hgwM+jGPCz6zY6A794n0IFMv9egGvWbOCXX2ayetVsYmJi2LjxL3788eov9c5sxYsXZcDATwgODiYoyDBp4kzmzF7EnHljCcsbhjHw5x9befnFt3wd9ZoQGxvLiy+9ycwZowgOCmLI0LFsdnq3JPMEWj0HWl4IzMwBJgTIaYyJBnIBh4CGwGPO+qHAO8D3QGvnZ4AJwDfGGGNTuarCpHXFhTEm8ajSGGAv8Km11q1zM548VeQJFyKW+DpCuuUudY+vI6RbjpDAesD4hehLvo6QbgH1H89hfB0gnQKxjsU7YqIOevXjfGHSh177OOZq06crzjhWx0BniAgAxpgXgfeBC8Bc4EVgpdOrgjGmDDDLWnu7MeZP4D5r7QFn3S6gprX2WErlu/OsogbpPywRERG5FiUex3o5Y0wBXCyVgCkAACAASURBVL0o5YBTwHjgvsws351TRfmAt4H4P+t/Bd611p7OzCAiIiKSQX5wYzhHY1zPNzwKYIz5BagL5DfGhDh34i8NHHS2P4jrUUIHjDEhQD5cg3RT5M4N6H4CzgLtnOkM8HP6j0VERESucfuAWsaYXMYYAzQCNgOLgIecbToDU5yfpzrzOOsXpja+BdwbZHuDtbZtovm+xhjPPihHRERE3OcHN4YDsNauMsZMANbjGhe7AddppRnAGGNMP2fZYOclg4HhxpidwAlcVyClyp2GywVjTD1r7VIAY0xdXANuRERERJKw1r6Na4hJYruBGslsexHX/eHc5k7D5TlgmDPWBVzXX3dOZXsRERHxJj/pcfGGVBsuxphgXHe3q2yMyQtgrT2T2mtEREREPCXFhkv86F9jTD1Qg0VERMRvpXFPtmtJaj0uq4GqwAZjzFRc12Kfj19prfXfB9eIiIjINcmdMS45cF1T3RDXjSKN868aLiIiIv5AY1wAKGqM6Qn8yb8NlnhZp09KRERE/EZqDZdgIA/JPz5EDRcRERF/oR4XAA5Za9/1WhIRERGRNKTWcAm0B7WKiIhkTf7zrCKPS+1ZRY28lkJERETEDSk2XKy1J7wZRERERCQt7lwOLSIiIv4sCw3OTe1UkYiIiIhfUY+LiIhIoMtCt/xXj4uIiIgEDPW4iIiIBLosNMbF4w2XkKBgTxeRqXKWvNvXEdLt7PgXfR0h3Qo+8q2vI6RLhfylfB0h3XacOujrCOlWKFdeX0dIl2P/nPF1BJEsRz0uIiIigS4L9bhojIuIiIgEDPW4iIiIBDrd8l9ERETE/6jHRUREJMDZON3HRURERMTvqMdFREQk0OmqIhERERH/ox4XERGRQKeriv5lXMp4I4yIiIhIatJsuFhrLTDTC1lEREREUuXuqaL1xpjq1to1Hk0jIiIi6ZeFLod2t+FSE+hgjPkbOA8YXJ0xd3gsmYiIiMhl3G24NPNoChEREck4XQ6dlLX2b6AM0ND5+R93XysiIiKSWdzqcTHGvA1UA24GfgZCgRFAXc9FExEREbeox+UKDwKtcI1vwVobAYR5KpSIiIhIctwd4xJlrbXGGAtgjMntwUwiIiKSHjbrXFXkbo/LOGPMD0B+Y8yzwHxgkOdiiYiIiFzJrR4Xa+2nxpgmwBlc41z+z1o7z6PJRERExD1ZaIyLu4NzewJj1VgRERERX3J3jEsYMNcYcwIYC4y31h72XCwRERFxWxa6c66793Hpa629DegOlAB+NcbM92gyERERkcuk9yZyR4BI4DhQNPPjJO+HHz5h3771rFuX9ExVt25PsGnTQtavn8/777/urTgZ0qxpff768ze2bl7Ka726e7Xs4b/9SZv+E2nbfyK9Ry7iUnTMVe1v8MJNhH80jtYfT2D5tgMARJ46xzMDZtLm04m06T+RkUv/zIzoqRow4BP+/nsda9fOTVhWqVJFFi+exJo1c5gwYTBhYXk8nqPfF2+y9K/ZTP11dKbsr3X7FsxeOYHZKyfQun0LAHLkzM6AkZ8xY9k4pv02hp5vevczFM+Xn+P0WPP7fBYtm8L8Jb8wZ9H4hOVPd+nAktUz+HXFNN7q+6oPE6YuUOoZoHTpksyfO57fNy1i08aFvPCfp30dyS2BVMdusXHem3zMrYaLMeZ5Y8xiYAFQCHjWm88pGj58PK1aPZ5k2b331iY8vCnVq99H1aqN+eKLH7wVJ92CgoL46sv3aRnekUqVG9C+/QNUrHijV8o+fPo8o5f9xagerZn4SltirWX2pt1uvbb5B2OvWLbr8EnmbNrNxFfa8t0zzfjfpOXExsURHBTEKy1r8MurbRnePZyxy7ew6/DJzD6cJIYPH0/r1p2TLPv++494880PqV69GVOnzuHll7t6NAPA5DEz6PLIi+l+3dBJ31OyTIkky/Llz0v3V5+h/X1P0a7Zk3R/9Rny5nPdMumn70bSom472jTqSJUalbm7Ye1Mye8uX36OM6JteGca392GZg0eBqDu3TVodn8jGtV7gHtrh/P91z/5OGHyAq2eY2Ji6PVaX+6o3IC69cLp1u0Jv84LgVfHkpS7PS5lgJestbdZa9+x1m72ZKjLLV26mpMnTyVZ9uyznfj00++IiooC4OjR496MlC41qldh16697Nmzj+joaMaNm0KrcO89/ik2znIpOpaY2DguRsVQJG8uNh84xtPfz+DRLyfT7cfZHD3zj1v7WvzXPppVLk+2kGBKFQyjTOG8/Ln/KEXy5qJi6cIA5M6RjfJF83PktHv7zKhly1Zz4kTSz0WFCuVYunQVAAsXLuGBB5p7NAPA2pUbOHXqTJJlZcqWYuCYL5kwbyjDpw6kXIXr3dpX3Qa1WP7rKk6fOsOZ02dZ/usq6jWszcULl1i9bB0A0dExbP59K8VLeq3TE/D95/hqdX7qEb7+fBBRUdEAHDt2wseJkhdo9RwZeYQNG109rOfOnWfr1h2UKlncx6lSF2h17JY4673Jx9wd49IHsMaY/zhTZQ/nStONN5ajbt0a/PbbFObNG8ddd/nvg6pLlirO/gMRCfMHDh6ipJf+YxfLl5vH772d+/43hib9RpMnRzaq31CSD6es4JNODRn94gM8UO0mvpm91q39HTlznuL5/73/YLF8ua9ooBw8cZatEcepdF2RTD0Wd2zZsoPw8KYAtGnTgtKlS6TxCs/o++nrvN/nUx5q0plP3vmS//vov269rliJIkQePJIwfzjiCMVKJK3HsLx5aNDsblYsWZOpmdPiy89xellrGTNpMHMWT6BjZ1ePS/kKZalV5y5mzh/DpBnDuLPK7T5OmbxAqufLXX99ae6sfDurVm/wdZRUBXIdi/uXQ/cAugC/OItGGGMGWmu/TmH7Ls72hIQUIDg488cZhISEUKBAPu65pzXVqlVm5MjvuOWWepleTqA7888lFv+1jxm92xGWMzu9RixgyOJN7Io8yXODZgMQZy2Fw3ICMGjBRub9vgeAo2f+od3nkwC4s2wxXn+wTprl/XMpmleHL6BXeC3y5MjmoaNKWdeuvejf/x169+7BjBnzEv669qZcuXNSpXolPh/8QcKybNlCAXjwkZZ06vIIANeVK80Poz4nOjqGg/sieOGJ19Lcd3BwMJ/+0I8Rg8Zy4O+INLfPqlrd14HIQ0coXLggYycPZueOPYQEh5C/QD7ub/wIVapWYuCQz6lRuYmvo14zcufOxbixg+j56tucPXvO13HkGubu5dDPADWttecBjDEfASuAZBsu1tqBwECAHDmu80i/0sGDh5gyxfXFu3btJuLiLIULF/TL7t+Ig5GUKV0yYb50qRJERER6peyVOyMoVTCMgnlcDZNGt5dlypod3FAsP8P+0+qK7Z9tdCfPNroTcI1xGffyg0nWF82bm8hT5xPmD58+T9F8uQCIjo3jleELuL/KDTSqVNZDR5S67dt3ER7eCXCdNmrevKHXMxgTxNkz52jTsOMV6yaNmc6kMdMB1xiXPj3eJWL/oYT1hw8dpUbdqgnzxUoWZfWy9Qnzffv34e/d+xk2cIwHjyB5vvwcp1fkIVev1bFjJ5g1fT5VqlYiIiKSmdNcA/w3rP+DuLg4ChUqwPHjnh2LlV6BVM/xQkJCGD92EKNHT2Ly5Fm+jpOmQKzjtNgsdAM6d8e4GCA20Xyss8xnpk6dy733ugYnVqhQjmzZQv2y0QKwZu1GKlQoR9myZQgNDaVdu9ZMmz437RdmghL5c/P7viNciIrBWsuqnRE0uP06Tp6/yKa/XbfiiY6NY2eke7+87731OuZs2k1UTCwHT5xl37Ez3F6mCNZa+o5fQrmi+el0TyVPHlKqihQpBIAxht69X2DQoJFez3D+3HkO7IugWXijhGU33+bewL9li1ZS995a5M0XRt58YdS9txbLFq0E4MXezxGWNw8fvPmZR3KnxZef4/TIlSsnufPkSvj53gZ12bplB7NnLKDu3TUBKH9DWUJDQ/2u0QKBU8+JDRrYny1bd/LFlwN9HcUtgVjH8i93e1x+BlYZYyY58w8Agz0T6UrDhn3N3XfXpnDhAuzcuYp+/T5j6NCxDBz4CevWzSMqKopnnunprTjpFhsby4svvcnMGaMIDgpiyNCxbN683StlV7quKI0rlePRLycTHGS4pVQhHq5VkbvKl+DjKSs4dzGamLg4OtS7jQrFC6S5vwrFC9DkjnK0+XQiwUFB9HmgNsFBQWzYE8n09Tu5sXiBhNNLL9xXjbsrlvHYsQ0d+lWiz8VK3nvvc/LkyUXXrq4r0KZMmc2wYeM8Vn68Twe8R426d5G/YH4WbZzGNx8Pole3/+Ptj//Lcz2fIiQkmFmT57Htrx1p7uv0qTN8/9lgxs0dAsB3/X/k9KkzFCtRlOd6PsWu7XuYuGA4AKMGj2fCyCmePLQkfPk5To/CRQrx80hXZ3BIcAi/TJjOogVLCQ0N5fNv+rF4+VSioqPp8XwfHydNXqDUc7y6darTqeND/P7HZtaucX35v/XWh8yavdDHyVIWaHXsFj8YNOstxrr5REljTFUgfhDJEmutW6OvPHWqyFNi4mLT3sjPnB2f/ktxfa3gI9/6OkK6lA0r5usI6bbj1EFfR0i3wrny+jpCuhz750zaG0mWFBN10KtnJc6//7jXvmtzvzHMp2dcUu1xMcYUTDS715kS1llr/fPcjIiISFbiBzeG85a0ThWtAyz/jmeJb9EZ5+fyHsolIiIicoVUGy7W2nLeCiIiIiIZlIXGuKR1qqhqauuttetTWy8iIiKSmdI6VdQ/lXUW8P5NMkRERCSpLHQfl7ROFTXwVhARERGRtLh7y//Hk1turR2WuXFEREQk3TTG5QrVE/2cA2gErAfUcBERERGvcavhYq19IfG8MSY/4P2HpYiIiMiVstB9XNx9VtHlzgO6VFpERES8yt0xLtP49+ZzQcCtgOcfAiMiIiJp0xiXK3ya6OcY4G9r7QEP5BERERFJUVo3oMsBPAdUAP4ABltrY7wRTERERORyafW4DAWigSVAc1yniALvUcQiIiLXMKsb0CW41VpbCcAYMxhY7flIIiIiIslLq+ESHf+DtTbGGJPatiIiIuILGpyboLIx5ozzswFyOvMGsNbavB5NJyIiIpJIWs8qCvZWEBEREcmgLNTjktEb0ImIiIh4nbv3cRERERF/pVv+i4iIiPgf9biIiIgEuiw0xkUNl8sEBeAl3wUf+dbXEdLt1Pohvo6QLkWrP+PrCFnC6Uv/+DqCiPg5NVxEREQCnM1CPS4a4yIiIiIBQz0uIiIigU49LiIiIiL+Rz0uIiIigS4LPR1aPS4iIiISMNRwERERkYChU0UiIiKBToNzRURERPyPelxEREQCnXpcRERERPyPelxEREQCnLXqcRERERHxO+pxERERCXQa4yIiIiLif9TjIiIiEujU45KUMeZhY0yY8/ObxphfjDFVPRtNREREJCl3TxW9Za09a4ypBzQGBgPfey6WiIiIuMvGWa9NvuZuwyXW+bcFMNBaOwPI5plIIiIiIslzd4zLQWPMD0AT4CNjTHY0sFdERMQ/+EFPiLe42/hoB8wBmllrTwEFgV4eSyUiIiKSDHd7XEoAM6y1l4wx9YE7gGEeSyUiIiLui/N1AO9xt8dlIhBrjKkADATKAKM8luoyP/zwCfv2rWfdunkJy4YP/5ZVq2axatUstm1bxqpVs7wVJ0N69HiGjRsWsGH9fIYP+4bs2bP7OtIVBgz4hL//XsfatXMTllWqVJHFiyexZs0cJkwYTFhYHo/nGDFtAQ/2eJcHX+jL8KkLrnp/UxauoGW3t2jZ7S2mLFwBwIVLUXR/7xtadX+bB1/oyxfDJl11Oe7Inj0bCxf/wtIV01m5ZhZ93ngxYd1bb7/Cug3zWb1uDl27dfZKnvQaNLA/EQc2sXHD1b8vnlK6dAlmzx7D+vXzWbduHt27PwlAgQL5mD59BH/8sZjp00eQP39eHydNWbOm9fnrz9/Yunkpr/Xq7us4qSpduiTz547n902L2LRxIS/852lfR3JLINWxJOVuwyXOWhsDtAG+ttb2wtUL4xXDh4+nVavHkyzr1Kk7NWs2p2bN5kyaNIspU2Z7K066lSxZnO7dn6JW7RZUqdqY4OBg2rVr5etYVxg+fDytWyf9wvz++494880PqV69GVOnzuHll7t6NMOOvw8ycd4yRn3Sm/FfvMlva/9g36Ejbr32qTf6c/DwsSTLTp89z4CxMxj5cW9GfdKbAWNncObceQA6P9CEqd/2Zdxnb7Bhyy6WrPsz04/ncpcuRRHeoiP1arekXu1wGje+h2rV76RDx7aUKlWCalWbUOOuZkycMN3jWTJi2LBxtGjZwdcxUhUTE0vv3v2oWrUx9977AF27Ps4tt9zIq68+z+LFy6hUqT6LFy/j1Vef93XUZAUFBfHVl+/TMrwjlSo3oH37B6hY8UZfx0pRTEwMvV7ryx2VG1C3Xjjduj3h13kh8OpYknK34RJtjHkUeByI/40a6plIV1q6dDUnT55Kcf1DD7Vk7Ngp3oqTISHBIeTMmYPg4GBy5srJoUOHfR3pCsuWrebEiaT1XKFCOZYuXQXAwoVLeOCB5h7NsOdAJHfcWJac2bMREhxMtdtuZP6KDew/dJTn+n5F+57/o3OfT9lzINKt/S3bsJnalSuSLyw3efPkpnbliixdv5mc2bNRo9LNAISGhlDxhjIcPn7Sk4eW4Pz5fxLKDQ0NwVrL08904OMPv054UNqxo8e9kiW9lixdxYlU/i/6g8jII2zc6GqEnjt3nq1bd1KyZDFatmzCiBETARgxYiLh4U19GTNFNapXYdeuvezZs4/o6GjGjZtCq/Bmvo6VosjII2xIUt87KFWyuI9TpS7Q6tgduhz6Sk8CtYH3rbV7jDHlgOGei+W+evVqcPjwMXbt2uvrKCmKiIjk8y9+YNfOVez7ez1nTp9l/vzffB3LLVu27Ej4Bd+mTQtKl/ZsR1uF60qyfstOTp05x4VLUSxZ/yeHj52k73cj6PNse8Z+9jqvPNmWfj+Mdmt/R06cpHjhAgnzxQrl58iJpA2UM+f+4dc1f1Drjlsy9VhSEhQUxJLl09i5ZzWLFi5j3dpNlCt3HW3atmDxb5OZ8MtPlL+hrFeyXOuuu640d955G2vWbKRo0cJERrp67yIjj1C0aGEfp0teyVLF2X8gImH+wMFDlPTzhkC8668vzZ2Vb2fV6g2+jpKqQK5jcXNwrrV2szHmv8B1zvwe4KOUtjfGdAG6AISEFCA42HPjItq1a824cf7d25I/fz7CWzblpptrc+rUGcaMHsBjj7Zh1OhffB0tTV279qJ//3fo3bsHM2bMIyoq2qPllS9TgicfbEbXd74iZ45s3FyuDBejotm0bTevfjwoYbuomBgAJi9YzshpCwHYF3mU7u99Q2hICKWKFeKLPt3SLC8mNpb/fjaYx1o0oHTxIp45qMvExcVxd51w8uULY8ToAVS89SayZc/GxYuXqH/PA4S3asq3339I86aPeCXPtSp37lyMHj2AXr3e5ezZc1est77/w/Gakjt3LsaNHUTPV99Otr7Fw/ygJ8Rb3Gq4GGPCgU9x3XSunDHmTuBda22yAzWstQNxDeIlR47rPFabwcHBtG59H3XqtPBUEZmiUcN67N27n2PHTgAwefIsatW+KyAaLtu37yI8vBPgOm3UvHlDj5fZpkld2jSpC8CXwydTuEBelqzLyfgv3rxi2wca1eGBRnUA1xiX93p0plSxf/+SLlqwAGv/3J4wf/j4KardflPC/LvfjeT6EkXp1KqRpw4nRadPn2XJbyto3PgeIiIimTZ1DgDTps7l2+8/9nqea0lISAijRw9g7NjJCePfjhw5RvHiRYmMPELx4kU5evRYGnvxjYiDkZQpXTJhvnSpEkREuHdq1FdCQkIYP3YQo0dPYvJk/75QAgKzjuVf7p4qegeoAZwCsNZuBMp7KJPbGjasx/btuzh40L8/cPv2R1CzZhVy5swBQIMG9di6daePU7mnSJFCABhj6N37BQYNGunxMo+fOgPAoaMnWLByA+H1a1GqaGHmLlsHgLWWbXsOuLWvulVuZfnGzZw5d54z586zfONm6la5FYCvR07h7PkLvPb0w545kGQUKlyQfPnCAMiRIzsNnM/wjGnzuPueWgDUu7smu3bu8Vqma9GAAR+zbdtOvvrqx4RlM2bMp2PHtgB07NiW6dPnpfRyn1qzdiMVKpSjbNkyhIaG0q5da6ZNn5v2C31o0MD+bNm6ky++HOjrKG4JxDpOU5wXJx9z9z4u0dba08aYxMu8Fn/YsK+5++7aFC5cgJ07V9Gv32cMGTKWdu1aMXbsVG/FyLA1azbwyy8zWb1qNjExMWzc+Bc//uj5BkB6DR36VaJ6Xsl7731Onjy56NrVdUXXlCmzGTZsnMdz9PxoIKfPniMkJJjXuzxK3jy5+KDnU/QbMIqB42cSExPLfXdX5+ZypdPcV76w3HRtdz+PvvohAM+1b0G+sNxEHjvJoPGzKFe6OO17/g+AR1rUp22Teh49tuLFijBg4CcEBQcTFBTEpF9mMGf2IlauWMugwZ/z/H+e4vy587zQvY9Hc2TUiOHfcu89tSlcuCB7d6+l77uf8vOQMb6OlUSdOtXo0KEtf/yxhZUrZwLw9tuf8Omn3zFixHd07tyeffsO0rGjf15VFBsby4svvcnMGaMIDgpiyNCxbN68Pe0X+kjdOtXp1PEhfv9jM2vXuL7833rrQ2bNXujjZCkLtDqWpIx140SvMWYwsADoDbQFegCh1trn0nqtJ08VeUKc9YPmZDoFBwX7OkK6nVo/xNcR0qVo9Wd8HSHdzkdd9HWEdAsNdvdvKf8QHRvj6wjip2KiDpq0t8o8Jx+u77Xv2gLjF3v12C7n7qmiF4DbgEu4bjx3GnjJU6FEREREkpPmnzfGmGBct/tvALzh+UgiIiKSLoF3siDD0uxxsdbGAnHGmHxeyCMiIiIBzBiT3xgzwRiz1RizxRhT2xhT0Bgzzxizw/m3gLOtMcZ8ZYzZaYz53RhTNa39u3tC+RzwhzFmHnA+fqG1tkeGjkpEREQyjT/c0TaRL4HZ1tqHjDHZgFzA68ACa+2HxpjeuMbM/hdoDtzoTDWB751/U+Ruw+UXZxIRERFJlnN25h7gCQBrbRQQZYxpDdR3NhsKLMbVcGkNDLOuK4VWOr01Jay1h1Iqw9075w41xhRxfj6aoaMRERERz/DiGJfEd8d3DHRuPAtQDjgK/GyMqQysA14EiiVqjEQCxZyfSwH7E+3rgLMsxYZLqmNcnHNP7xhjjgHbgO3GmKPGmP9z6+hERETkmmKtHWitrZZoSnznwRCgKvC9tbYKruElvS97vQUyfG4rrcG5LwN1gerW2oLW2gK4zj3VNca8nNFCRUREJPPYOO9NaTgAHLDWrnLmJ+BqyBw2xpQAcP494qw/CJRJ9PrSzrIUpdVw6QQ86jxU0VU51u4GOgKPpxlfREREsgxrbSSw3xhzs7OoEbAZmAp0dpZ1BuKfjjwVeNw5w1MLOJ3a+BZIe4xLqLX2iieRWWuPGmNC3TwOERERyTpeAEY6VxTtBp7E1VEyzhjzNPA30M7ZdiZwP7AT+MfZNlVpNVyiMrhOREREvMWPbkDnPIi5WjKrGiWzrQW6p2f/aTVcKhtjziSz3AA50lOQiIiIyNVKteFirQ28p/eJiIhkMQH4fOAMc/chiyIiIiI+F1jPkBcREZErqcdFRERExP+ox0VERCTAaYyLiIiIiB9Sj4uIiEiAU4+LiIiIiB9Sj4uIiEiAU4+LiIiIiB/yeI9LTFysp4vI8qoXvMHXEdKtePVnfR0hXSK/fdjXEdIt7Nnhvo6QbjmCA+vZrdGxMb6OkG7G1wHSyfo6QKCwgfbOZpx6XERERCRgaIyLiIhIgNMYFxERERE/pIaLiIiIBAydKhIREQlwNk6Dc0VERET8jnpcREREApwG54qIiIj4IfW4iIiIBDirG9CJiIiI+B/1uIiIiAQ4jXERERER8UPqcREREQlwuo+LiIiIiB9Sj4uIiEiAs9bXCbxHPS4iIiISMNTjIiIiEuA0xkVERETED7nVcDHGfOTOMhEREfE+G2e8Nvmauz0uTZJZ1jwzg4iIiIikJdUxLsaYbsDzQHljzO+JVoUByzwZTERERORyaQ3OHQXMAj4AeidaftZae8JjqURERMRtWely6FQbLtba08Bp4FEAY0xRIAeQxxiTx1q7z/MRRURERFzcHZwbbozZAewBfgX24uqJ8brSpUsyf+54ft+0iE0bF/LCf572RYx08VbmPv17MX3TRIYvGJzs+iq1KzNny1SGzB3IkLkDefKlTlddZmi2UN79/i3GLh3OwGnfUrx0MQCq330Xg2cNYNj8Hxk8awBV61a56rJSEhQUxK/LpjJm/EAArru+NPMWTWDdpgUMHvoloaGhmVbW3uNnaffjwoSp7qfTGLF65xXbrfn7KO1+XEibgfN5evhvV11uVEwsr01aTfj3c+k4ZDEHT50HYMWeIzz60yIeGrSAR39axOq9R6+6rNQMGtifiAOb2LhhgUfLuRrZs2dj/uKJLFkxjeVrZtH7jRcBuPveWixeOoXlq2fy3Q8fExwc7OOkKWvWtD5//fkbWzcv5bVe3X0dxy1BQUGsWT2HyZOG+jqKWwKxjlOjwblX6gfUArZba8sBjYCVHkuVipiYGHq91pc7Kjegbr1wunV7gooVb/RFFLd5K/PMcXPo2aF3qttsWv0HTzTtwhNNu/DzF8Pd3nfx0sX4evxnVyxv+Whzzp4+S/t6nRg7aALPv9EFgFMnTvPfJ97g8cbP0O+lD/m/L/uk72DS4bnnn2D7tn8bD++89xrff/szd1VuxOlTp+nU+eFMK6tsoTDGPdOQcc80ZPRTDcgRGkzDrIJLfQAAIABJREFUm0sm2ebMxSg+mL2JLx+uxS9dGvNJm5pu7//gqfM8PWLJ/7N33/FRVAsbx39nUwg19BJAQbFfRZAmvUmvgiAXFAVEioig6FVQpCioYHtVOlKUpohIEZDepHeQrlJCL6GFkuS8f+wSCWkbyO5m5fnez3zYnZmd88zcdXP2lNl466dt/pssIUHM6FiDViWL8MWi7QBkSx/MF8+U4ceXqtGv3hP0/GXd7Z1gMsaNm0Ldei09WsbtunLlKg3rPkeFJ+tT8cn6VKtegVKlizFk2Ce0faErZUvV4eDBw7Ro+bSvoybI4XDw5RcfUK9+Kx4tWoXmzRul+c84gFe7tOOPnXt8HcMt/nqNxcndiss1a+0pwGGMcVhrFwElPJgrUUePHmfjpm0AXLhwkZ0795A/LK8vorjNW5k3r97CubPnbum1NZ6uzoiZ3zBm3nB6fNQNh8O9t0aFGuWY/cM8ABbPWsIT5YsDsGf7Xk4eOwXAn7v+Il1IMEHBqdfycV1YWF5q1KrMuLFTYtdVrFSG6dPmADDx+2nUqZfQpLjbt/qv4xTIlpGw0Axx1v+6/RBVHwgjn2t99ozpYrfN2naAlt8uptnIhfSbvZHoGPc6phfvPkL9R+8CoPpDYaz56wTWWh7Mm5XcmdMDcG+uzFyJiuZqVHRqnF6Cli1fzekzZz12/NRy8eIlAIKCAgkKCiI6OoarV6+xb+9fACxeuIIGDWv6MGHiSpUsxr59f/Hnnwe4du0aU6ZMp0H9tJn1uvz581G7djVGj57o6yhu8cdrnBxrjdcWX3O34nLWGJMJWAZ8b4z5ArjouVjuufvuAjxe9D+sXrPR11Hc5uvM/3niYcb8NoJB4wdQ+P5CzkxF7qJagyp0aNSFF2q0JyY6hhpPV3PreLny5uR4+HEAoqNjuHjuIqHZssTZp3LdiuzatodrV6+l6rkAfPhxL3r3+ogYVwUge45sRJw9T3S08493+OGjhIXlSfVyAebuOETthwvEW//36Qucu3yVtt8to8XoRczY6hwKtv/kOebuOMyY5ysypV1VHA7D7O0H3Srr+PlI8mZxVoQCHQ4ypQvibOTVOPvM3xnOQ3mzEhyYdrtAvMXhcLB05S/s/nM1ixcuZ/26zQQGBvB4sf8A0KBRLfIXyOfjlAkLy5+Xg4fCY58fOnyEsDT+5Wzw4D68/XZ/YmJifB3FLf54jeUf7t7yvyFwGXgNaAmEAn0T29kY0x5oD2ACQnE4Mt5mzPgyZszAlMkj6P5Gb86fv5Dqx/cEX2fetXUPTUq1IPLSZZ6sWpoBo/vybPnnKVG+OA8+eh+jZg8BIF1IOs6cdH6r/nBkX8LuyktgUCB58udhzDznOJIpI39i9pQ5yZZZ+P5CdHqnPd3++2aqn0/NWlU4eeIUmzdtp1wF97tjUsO16BiW7DnKq5UfibctOsbyx9GzDP9veS5HRfP82CU8FpaNNX+d4I+jZ2n57WIArkRFkz2DszWm24+rOHz2ElHRMRw5d4lmIxcC8N+S99Ko6N3J5tl74hxfLNrOkBZlU+8k/VhMTAwVyzYgS2hmvps4hIcevo+2L7zGhx/1JDg4mEULl8dWbuX21KlTnRPHT7Jh41YqVnzS13HuWNY/6oypwq2Ki7X2ojEmD1ASOAX86uo6Smz/4cBwgMDg/Kk+SSswMJAfJo9g4sRp/PyzT8YIp1hayHzpwqXYx78vXM3rH3YlNFsWjDH8+sM8hg4cGe8177R7D3COcen52Vt0eaZ7nO0njp4kd1huThw5SUCAg4xZMhJxxtldlStfTj4c1Yd+XQdw+O/weMe+XaXLPEGtOtV4qkYl0oWkI3PmTAz8uBehWTMTEBBAdHQ0YfnzEh5+LNXLXr7vKA/mzUqOTCHxtuXJHEJo+jykDw4kfXAgT9yVk13Hz2Et1H/0Ll6tEr+y81nTMoBzjMt7MzcwqlWFONtzZ07P0XOXyJMlPVExMVy4co2s6YMBOHYuku5TV9Gv/hMUzJYp1c/Vn52LOM+ypauoVr0iX305ijo1WgBQpWp57i1S2MfpEhZ++CgFC/wzbqpA/nyEhx/1YaKklS1bgnr1alCrVlVCQtKRJUtmxo75ktYvvOrraInyt2sscbk7q6gZsAZ4BmgGrDbGNPVksKSMGD6YP3bu5fMvhvsqQoqlhczZc2WLffzQ4w9iHIaIM+dYt3wDletVJGuOrABkzpqZPPnd615ZPm8ldZ6pAUDlupVYv8LZBZYpS0Y+GTeAoR+OZOu67al8Jk593x/Efx4oT9FHKtP2hddYtuR32rd9nWVLV9OwcS0AWrRszK+z5qd62XO2H6JWAt1EAJXvz8emg6eIiokh8loUWw+f5p4cmSlVKBe/7TzM6YtXAIiIvEp4xKUEj3GzSvfli+1ymv9HOCXvzoUxhnOXr9Jlykq6Vn6EYgVzpM7J+bkcObOTJTQzACEh6ahStRx7du8nZ67sAAQHB9O1e3u+HTXBlzETtXbdJooUKUyhQgUJCgqiWbOGzJg5z9exEtWr10AK31OC++4vQ8tWnVi0aEWarrSA/11jd8RY47XF19ztKuoJlLTWHgcwxuQC5gM/eipYYsqVLclzrZqyZesO1q11vtHefXcgv85Z6O0obvNW5ve/7kWxJ4uSNXso09ZNZtSgMQQGOf8v/nn8DKrUrUTj5xsQFR3N1ctX6N2pPwB/7fmbER+P5vOJH2OMISoqmk97fsGxw8m3VMycNJt3v3yHycvHc+7seXp36gdAkxcbU6BQGC92e44XuzmnXb/W4k3OnvL8wM733/2YUWM+p+e73dmyZQfjx/6QqsePvBrFqr+O06v2P1O8f9jwJwDPFC/MPTmzUPbe3DQbsRBjoPHjhSiS2znu55VKD9Nh4gqstQQGOHi7ZtF4g3sT0vjxu+n5yzrqD5lHlpBgPmpUEoDJ6/Zz4MxFhi3fxbDluwAY2qJcnAHBqem78V9TqeKT5MyZnb/2r6NP30F8O2aSR8q6VXnz5OKb4Z8QEODA4XAw7afZzJ2ziL7936JG7So4jIPRIyewbIlPJkYmKzo6mq6v9WL2rAkEOByMGTuZHTt2+zrWv4qusX8z1o3b7RljtlprH73huQPYfOO6xHiiq0jiKp3rAV9HSLEdEf5178Lwr33WwHjLMr/k/nT3tCJzcHpfR0iR81cjfR0hxXz/fTll/PUPSNTVw1691LserO21S/XAzl99+jZyt8VljjFmLnB9rltzYLZnIomIiIgkLLkfWSwC5LHW9jDGPA2Ud236Hfje0+FEREQkeWnhjrbeklyLy+fA2wDW2p+AnwCMMY+6ttX3aDoRERGRGyRXccljrd1680pr7VZjTCGPJBIREZEUuZN+HTq56dBZk9jmX6PoRERExO8lV3FZZ4x56eaVxph2wHrPRBIRERFJWHJdRa8B04wxLfmnolICCAYaezKYiIiIuEeDc12stceAssaYKsB/XKtnWWvT7t3eRERE5F/L3d8qWgQs8nAWERERuQVp4Vb83uLWbxWJiIiIpAXu3jlXRERE0iirFhcRERGRtEctLiIiIn5ON6ATERERSYPU4iIiIuLnNKtIREREJA1Si4uIiIif06wiERERkTRILS4iIiJ+TrOKRERERNIgtbiIiIj4Oc0qEhEREUmD1OLyL7D6xC5fR/jXy/zSeF9HSLFL++f4OkKKZbinlq8jpEi29Jl8HSHFzkRe8HWEFAl0BPg6gl/QrCIRERGRNEgVFxEREfEb6ioSERHxcxqcKyIiIpIGqcVFRETEz91B959Ti4uIiIj4D7W4iIiI+DmNcRERERFJg9TiIiIi4ud0AzoRERGRNEgtLiIiIn4uxtcBvEgtLiIiIuI3kmxxMcYUT2q7tXZD6sYRERGRlLLcOWNckusqGpzENgtUTcUsIiIiIklKsuJira3irSAiIiJya2LuoFvnuj041xjzH+BhIOT6OmvtOE+EEhEREUmIWxUXY0xvoDLOistsoDawHFDFRURExMdi7qAxLu7OKmoKVAOOWmtfBIoCoR5LJSIiIpIAdysukdbaGCDKGJMFOA4U9FwsERERkfjcHeOyzhiTFRgBrAcuAL97LJWIiIi4TdOhb2Kt7eR6ONQYMwfIYq3d4rlYIiIiIvHd8g3ojDHFdQM6ERER39Mt//8x2LV8DawGhuPsLlrtWucTNWtUZvu2pezcsZw3e3T2VYwU8bfM/pZ3xPDBhB/azKaNC3wdJUW8dZ2/mzqLxm270ajNa4yfOjPe9oUr1vB0u+40bf8GzTu+yYatf9x2mRHnzvNSj77Uff4VXurRl4jzFwCYOX8pT7frTuN23WnV5R127fvrtstKir+8lx0OBwuXTeP7yUMBKF+xDAuW/sTS32fw1ZCBBAQE+DhhwgoUCGP+vB/YsnkRmzctpMsrbX0dKUHDhn3CgQMbWL/+t9h148d/zerVv7J69a/s2rWC1at/9WFCcVeSFRdrbRXXTeiOAMWttSWstU8AxYDD3gh4M4fDwZdffEC9+q14tGgVmjdvxEMP3eeLKG7zt8z+lhdg3Lgp1K3X0tcxUsRb13nPnweYOns+E74eyI8jBrNk1XoOHD4SZ58yxR9l6ojB/Dh8EH3f6ETvwUPcPv7aTdvo+dFX8daPmvgzpYs/yqxxX1G6+KOMmjgNgAL5cvPtZ32ZNvJTXm7VlD6fDr29E0yCP72X23d8nt279gFgjOGrIQN56cXuVHyyPgcPhvPsfxv7OGHCoqKi6PFmHx4rWoVy5evTseMLafIajx//Aw0aPB9n3XPPdaZ06dqULl2badN+Zfr0OT5Kd/ssxmuLr7k7q+gBa+3W60+stduAhzwTKWmlShZj376/+PPPA1y7do0pU6bToH5NX0Rxm79l9re8AMuWr+b0mbO+jpEi3rrO+w8c4tEH7yN9SDoCAwIo8djDzF+2Os4+GdKnxxjnB1Lk5SuxjwG+nTydZzu9xdPtuvP1mMlul7to5Voa1qgMQMMalVm0Yi0Ajz/yIKGZMwHw2MP3c+zE6ds5vST5y3s5X1genqpZme/G/QhA9uxZuXrtGvtdrVFLFq2gXoMaPkyYuKNHj7Nx0zYALly4yM6de8gfltfHqeJbvnwNZ5L4jGjatB6TJ0/3YiK5Ve5WXLYYY0YaYyq7lhGATwbnhuXPy8FD4bHPDx0+Qlga/I/kRv6W2d/y+itvXef7Ct3Fhq1/cDbiPJGXr7Bs9UaOnjgVb78Fy1dT/4VX6dxzAH3fcI7HX7luE38fPsLErwfy4/BB7Ni9j3VbdrhV7qkzZ8mVIxsAObNn5VQCfzSm/bqA8qWK3cbZJc1f3ssfDHyHPu99QkyMc6TCqVNnCAwIoGix/wBQv2EtwvKnvdw3u/vuAjxe9D+sXrPR11FSpHz5Uhw7dpJ9Hu629KQYLy6+5u506BeBjkBX1/OlQKJtycaY9kB7ABMQisOR8XYyishtuOfuArR5thHt3+pH+pB0PFikEAGO+N9ZqpUvTbXypVm3ZQdfjZnEyE96s3LdZn5ft5lnXu4BwKXIyxw4dIQSjz3Mfzv/j6vXorgUeZmI8xdo2v4NALq91IpyJR+Pc2xjDJi4TcxrNm7jp18XMu7z/h46c//wVM3KnDhxmi2btlO2fKnY9e3bdKf/h28TnC6YxQtXEBOdFv5kJC5jxgxMmTyC7m/05rxrPJO/aNasIVOmqLXFX7g7Hfoy8JlrcWf/4TgH8hIYnD9Vf/op/PBRChYIi31eIH8+wsOPpmYRqc7fMvtbXn/lzev8dJ1qPF2nGgBfjPyePLlyJLpvicce5tCRY5yJOIe10LZFY5rVj99NMeHrgYBzjMvPcxfzwVuvxNmeI1tWTpw6Q64c2Thx6gw5sv5zs+1d+/6i9+AhDBnQk6yhmVPjFBPkD+/l0mWKU6t2Vao/VZGQkHRkypyJb4Z/Qqf2Pahf2zluq3LVctxbpJBvgyYhMDCQHyaPYOLEafz8s38NcA0ICKBhw1qULVvX11FuS9qu1qYut7qKjDHljDG/GWN2G2P2X188HS4ha9dtokiRwhQqVJCgoCCaNWvIjJnzfBHFbf6W2d/y+itvXudTZyIAOHLsBPOXr6ZOtQpxth84fARrnd8xduzez7WrUWTNkplyJYvy85yFXIqMBODYiVOxx0pO5bIlmD5vMQDT5y2mStmSsRm6vT+IAW93oVDBsCSOcPv84b3cv8+nFH24Ek88Vo2X2nRn+dJVdGrfg5w5swMQHBxEl9deYszoST5OmrgRwwfzx869fP7FcF9HSbGqVcuze/c+Dh9OWxVaSZy7XUWjgG4475ob7bk4yYuOjqbra72YPWsCAQ4HY8ZOZseO3b6MlCx/y+xveQG+G/81lSo+Sc6c2flr/zr69B3Et2PS7gc9ePc6d3//E86eu0BgYAA9X21HlkwZmTJjLgDN6tfkt6WrmPHbEgIDA0kXHMwn73bDGEPZEo+z/+/DtOzSE4AMISEMfOdVcmRL/qfK2j7bmDf6DWbarwvIlycXg9/tDsDQ8T9y9tx5+n8xEoCAAAeTh3zskfP2x/fydZ27tqNGzco4HA7GjJrI8qWrfB0pQeXKluS5Vk3ZsnUH69Y6K4XvvjuQX+cs9HGyuMaN+z8qVHiSnDmzsXfvavr3/5QxYybTrFkDJk/+xdfxbltamO3jLeb6t6wkdzJmtbW29K0UkNpdRSLinkv7/W9qZ4Z7avk6QopkS5/J1xFS7Eykf40/CXSkzfvXJOfy5QNerUnMytPCa39r6x6b6NNakrstLouMMZ8APwFXrq/UnXNFRER8L+bOaXBxu+JyvbWlxA3rLFA1deOIiIiIJM7dWUVVPB1EREREbk3MHTTGxd0WF4wxdYFHgJDr66y1fT0RSkRERCQh7k6HHgo0B7oABngGuNuDuURERETicfeW/2Wttc8DZ6y1fYAngfs9F0tERETcZb24+Jq7FZdI17+XjDFhQBSQzzORRERERBLm7hiXmcaYrMDHOG9CBzDSM5FEREQkJXTLfxdjTEljTF5rbT9r7VkgE7AV+AE3f7dIRERE7izGmABjzEZjzEzX88LGmNXGmL3GmMnGmGDX+nSu53td2wsld+zkuoqGAVddB68IDHSti8D1I4oiIiLiWzHGeG1xU1fgjxuefwR8Zq0tApwB2rrWt8U5frYIzgaRj5I7cHIVlwBr7WnX4+bAcGvtVGvtu0ARd9OLiIjIncEYUwCoi2tIiTHG4Lxh7Y+uXcYCjVyPG7qe49pezbV/opKtuBhjro+DqQbc+KtZbt8DRkRERDzHm7OKjDHtjTHrblja3xTnc+BN/hl6kwM4a62Ncj0/BOR3Pc4PHARwbY9w7Z+o5CofE4ElxpiTOGcWLcMZuojr4CIiInIHsdYOJ5HhIsaYesBxa+16Y0xlT5SfZMXFWvuBMWYBzqnP8+w/PyXtwHkzOhEREfGxNDSrqBzQwBhTB+ed9rMAXwBZjTGBrlaVAsBh1/6HgYLAIVcPTyhwKqkCkr2Pi7V2lbV2mrX24g3rduuXoUVERORG1tq3rbUFrLWFgGeBhdbalsAioKlrt9bAdNfjX1zPcW1feEMjSYI0TkVERMTPxaT931h8C5hkjOkPbARGudaPAsYbY/YCp3FWdpKkiouIiIikOmvtYmCx6/F+oFQC+1zG+fuHblPFRURExM/FkPabXFKLu79VJCIiIuJzanERERHxc2nhV5u9RS0uIiIi4jdUcRERERG/4fGuopK57vd0Ealq7Yndvo6QYsVy3uvrCCn254Wjvo6QIrlCsvo6QopluKeWryOk2KEy9/k6QooUWLXH1xFSLMDhX99Xo2KifR3BL/jBdOhU41/vYBEREbmjaXCuiIiIn0tDt/z3OLW4iIiIiN9Qi4uIiIif03RoERERkTRILS4iIiJ+TrOKRERERNIgtbiIiIj4Oc0qEhEREUmD1OIiIiLi59TiIiIiIpIGqcVFRETEz1nNKhIRERFJe9TiIiIi4uc0xkVEREQkDVLFRURERPyGuopERET8nLqKbmKMKefOOhERERFPcrer6P/cXCciIiJeZr24+FqSXUXGmCeBskAuY0z3GzZlAQI8GUxERETkZsmNcQkGMrn2y3zD+nNAU0+FEhEREffF3EE3oEuy4mKtXWKMWQ48Zq3t46VMIiIiIglKdlaRtTbaGBPmjTAiIiKScppVFN8mY8wvxpjnjDFPX19SUlDPwW8ya/NPfLdgdJL7PVT0AZb9PZ8qdSum5PAJypI1M19M/IQpy8fzxcRPyByaCYAajasz/reRfDd/FMOn/x9FHr73tstKyojhgwk/tJlNGxd4tJx3P32LuVumM2nhmAS3V6xZngnzv+X730Yx9tfhFC316G2XmSVrZr6aNJipyyfw1aTBsde4VuOnmDD/WyYuGMOoX77hPg9eY4fDwcJlPzNhyjAA/m/IQNZvWcCi5dNZtHw6/3n0IY+VDdD/814s3z6HX5ZMTJXjNWxelzmrfmTOqh9p2LwuACHp0zH0+0+ZtWIKM5ZOonuvzqlSVkrVrFGZ7duWsnPHct7s4dkMuSZPIseY0eQYNZIcw4fF257h2ebObaNGkmPMt+RZtACTOXMCR0qBoCBC33+PnBO+J/vQbwjImxeA4BJPkGPEMGeeEcMILl7s9spJhjev860YNmwQBw9sZMP6+bHrevd+g3Vr57Fm9RxmzfyefPny+DBh8tL6NZbEuVtxCQFOAVWB+q6lXkoKmjVlDt1avpV0GIeDTj3bs2bJ2pQcmmJPFqXXZ/GP/Vzn/7Ju+QaalX+Odcs38Fzn/wJw5OAROjV9jVbV2zL68/H876PXU1ReSo0bN4W69Vp6tAyAmZPn8GrLHoluX7tsPf+t/iItn2pLv+4D6TXoTbePXfzJx+n92dvx1rd+pSVrl2+gSfn/snb5Blq/0gqA8INHeLlJF1pUe4FRn43lnY8Tz3W7Xu7Ymj2798VZ9/67H1OlfEOqlG/Itq1/eKxsgJ8nzaL9s11T/Lqx04YQVjBfnHWhWbPQ+Y12NK/VhmY1X6TzG+3IEur8Yzz6m++pW64ZT1drRbFSRalQ9clUye8uh8PBl198QL36rXi0aBWaN2/EQw/d59EyT3ftxqm27TjV/uV42y5Nmuzc1rYdF4YP5+rmzdjz5906bkDevGT/4vN469PXrYM9f4GT/23JpSk/kqlDewBiIiI48793OPVCGyI+HEhoz3du78SS4IvrnFLjx/9A/QbPxVn36adDKVGyBqVK12L27Pn0fCfl/014iz9c45SK8eLia25VXKy1LyawtElJQZtWb+Hc2XNJ7vNMm8YsnrWMM6fOxlnfskNzRs0awvjfRtLu9RfcLrNCzbLM/mEuALN/mEvFWs5bz2xdt53zERcA2L5hB7nz5UzBmaTcsuWrOX3mbPI73qaNqzdz7kzi1zjyUmTs4/QZ0mNvmNfWquOzjJ09jAnzv6X9Gy+6XWalmuWZOWUOADOnzKFyrfIAbFm3LfYab92wndz5cqXkVNyWLywPT9WszHdjf/DI8d2xbtVGzt703i5YKD/DJ33Bj7+NZfwvwylc5G63jlWuShlWLllNxNlznIs4z8olqylf9UkuR15hzYr1AFy7FsWOLTvJG5Y71c8lKaVKFmPfvr/4888DXLt2jSlTptOgfk2vZkhMSLVqXJ7/T4tmyFNPkX3YEHKMGkmWN7qDw73vaCHlyxE5x/l+vrxkCemKPwFA1J69xJw65Xz855+YdOkgKCiVz8IpLV/n65YvX82Zmz7Tzp+/EPs4Q8YMcT5f0hp/uMaSuCT/azbGvOn69/+MMV/evKRmkFx5c1KpVgV+Gjc9zvpSFUtQoHAB2tbtyPM1XuLBx+7n8dKPuXXM7Dmzc+r4aQBOHT9N9pzZ4+1T/9k6/L5oze2fgJ+oXKsCPywdz2fjPqJf94EAlK5UkrsKF6B1nZdp+VQbHnz0AYqVLurW8bLnzMap484P9FPHT5E9Z7Z4+zRsUY+Vi1an3knc4IOBPenz3sfExMT9HtDzvW4sWfkL/Qe8TXCwZ/7AJKXPoHf44O1BNH2qNZ+8/wXvfZR0a+N1efLl4ujh47HPj4UfJ89Nlb7MWTJRpWYFfl+WspbJ2xWWPy8HD4XHPj90+AhhYXk9Vp7Fkn3wJ+QYMYz09ZNo4E2XjnSlS3F5yVIAAu6+i5CqVTjd6RVOtW0H0TGEPFXdrTIdOXMRffyE80l0NDEXL2BCQ+MWV6kS13bvgWvXbum8kuPt65ya+vR5k717V9Pi2cb06TvI13ES5c/XODG6j8s/rrexr0vJQY0x7YH2AIVD7ydPxuTH9r7WpzNffzgMe1M1vXSlEpSuVIKx80YAkCFDegoWLsCm1VsYOeMbgtIFkSFDerJkzRy7zzcfDGd1At1NNx+7eNnHqd+iDi83fjUlp+fXFs9ZxuI5yyhWuigd3mxL5+bdKVOpJKUrleT730YBztaYgvcUYOPqzXw7cyjB6YJInyE9WbJmid3n//oPZVWC1zju8yfKFqNBi7q81Cj1+5Br1KrMyZOn2LxpO+XKl4pd3//9wRw7doLg4CA+/bI/r3Zrz6CPvk718hOTIWN6ipV8lM9GDYhdd73y1PjZejzX/lkA7ipcgGETPuPatSgOHwinywvJd90FBAQwaFh/vhsxmUN/hye7vz873bkLMSdP4sialWyfDiLqwAGubd4Sb7+QcmW5unVbbDdRuieeIOiB+2PHxZh0wcScdbYOZO3fj4B8+TBBgThy5yHHqJEAXPrxRyJ/nZNspsBChcjcoT1nXvdc16c/6937Y3r3/pgePTrTseML9Ov3qa8jyb9QctOhZ7j+HZuSg1prhwPDAZ7MX8WtCtqDjz1Av2/eAyA0eyhPVi1NdFQ0GMO4rybw83cz4r2mXf1OgHOMS91mtejf7aM420+fPE2O3M5Wlxy5s3PI7ZM/AAAgAElEQVTm1JnYbfc+dA9vf/IG3Z/7X5LdK/9WG1dvJv9dYYRmD8VgGPN/3zPtu1/i7fdivQ6Ac4xL/Wa16dNtQJztp0+eIUfuHJw6foocuXPEucZFHrqHXoPepGurHkR44BqXKv0EtWpXo/pTlUgXko7MmTMxZMQndHzJ+Ufl6tVrTPxuKp1fbZvqZSfFGAfnz13g6aqt4m2bNmkm0ybNBJxjXN5+tS/hB4/Ebj925ASlyhWPfZ4nLDdrVmyIfd5n8Nv8vf8g44ZP8uAZJCz88FEKFvjnS0iB/PkIDz/qsfJiTp50/nv2LFeWLSfooYcSrrhUrcrlBXEHvkfOmcuF4SPi7Xu217uAc4xL6Nv/43TX124q8wQBuXMRc+IEBATgyJgJGxEBgCNXLrJ+0I+IDwYQHe65SqO3r7MnTJo0jek/j0uzFZd/wzW+2Z10Hxd3f6vofmPMcGPMPGPMwutLagZp8uR/ebpMC54u04JFs5Yw6J3PWTp3BasXr6Ve89qkzxACOLuUsuXI6tYxl89bSZ1nnP2WdZ6pybK5KwHnH4OBI/rSt+sADu4/lJqnkaYVKJQ/9vEDj95PUHAQEacj+H3JGho8W4f0GdIDKbvGS+etoF6zWgDUa1aLJXOXA5Anf24+Htmf3q9+wAEPXeP+fQbz2EMVKf5oVdq/2I3lS1fR8aUe5MnzT9dK7XrV+WPHHo+Un5iLFy5y6EA4NetXi133wCPuDfxbsWgV5SqVIUtoZrKEZqZcpTKsWLQKgK7/60DmLJkY0Ms3fwzWrttEkSKFKVSoIEFBQTRr1pAZM+d5pCwTEoJJnz72cXDJEkTt/zP+fhkzEvx4Ua4sXxG77sr6DYRUroQjq/M9bDJnxpHHvRkuV1asJH0t5/s5pFIlrmxwVhpNpkxk+2gA54cN59q2bbd1bsnx5nVOTUXuLRT7uH69Guzatdd3YZLhr9dYnNz9degfgKHASCD6Vgrq83Uvij/5OFmzhzJ93RRGDhpDYJDzVwOmjY/fmnLdmqXrKHTf3Yz4xdnUf+lSJH26fBhvAG9Cxn09kQ+G9qZ+izocPXSMXh2c99Br0+15smTLwhsfOr9tRUdF06ZOh1s5Lbd8N/5rKlV8kpw5s/PX/nX06TuIb8ek/jfm/t+8xxNPFiNr9lBmrvuR4YO/JTDQeY1/Gv8LVetWom7TmkRFRXE58grvdHwfgNVL1lK4yN2MnjEEgEsXL/Fel/5uXeOxX33PgKF9aPBsXY4ePsrbL/cGoF23FwjNFspbA7oBEBUVTeva7VP9nBMydOQgcuTMjjGGbVv/4I3Xenu0vEFD+1Gq3BNkzZ6VRZtm8NXHI+jR8T16f/wWHbq3ITAwgF9//o1d25OvQEWcPceQT0cxZd4YAL4ZPJKIs+fIky83Hbq3Yd/uP5m6YDwAE0b9wI/fT0/iaKkrOjqarq/1YvasCQQ4HIwZO5kdO3Z7pCxHtmxk/aCf80lAAJfnL+DqmjWkb9AAgMhfnK2DIRUqcGXtOuzly//k/PtvLowcRbbBg8BhICqKc599QcyxY8mWe2nWbLL2fIecE74n5vw5It7vC0CGpxsTkD8/mVq3JlPr1gCcef2N1Dzlf/J78TrfqnHjvqJihTLkzJmdfXvX0K//YGrVrMr9999LTEwMBw4c4pUunpt5dbv84RqnVFqY7eMt5uZxHwnuZMx6a+0Tt1KAu11FacXaE/735i2W07P3ofGEPy/4V7NsrhD3WqDSkj1nD/s6QoodKuNfU1ILrPJua15qCHBzhlVaER3jn3+So64e9mrnzcC7W3ntb+3//v7Opx1Tyf3I4vVpODOMMZ2AacCV69uttac9mE1EREQkjuS6itbjnP10vXZ1c9voPameSERERFLEr7o2blNyFZfmwEFr7REAY0xroAnwF/C+R5OJiIiI3CS5zs6huLqGjDEVgQHAWCAC13RnERER8a0YrNcWX0uuxSXghnEszYHh1tqpwFRjzCbPRhMRERGJK9mKizEm0FobBVTDdTdcN18rIiIiXuCfc69uTXKVj4nAEmPMSSASWAZgjCmCs7tIRERExGuSu+X/B8aYBUA+YJ7956YvDqCLp8OJiIhI8nw/8sR7ku3usdauSmCd/92lTURERPyexqmIiIj4uTtpjIt/3ftZRERE7mhqcREREfFzMT799SDvUouLiIiI+A21uIiIiPi5tHBHW29Ri4uIiIj4DbW4iIiI+Lk7p71FLS4iIiLiR1RxEREREb+hriIRERE/pxvQiYiIiKRBanERERHxc3fSdGiPV1zWntDvMXraxpP7fB3hX+/s5Yu+jpBiQQH+972kwKo9vo6QIpEHF/o6QoqlL1jV1xFSJGNwiK8jSBrjf59sIiIiEsed096iMS4iIiLiR9TiIiIi4uc0q0hEREQkDVKLi4iIiJ+7k2YVqcVFRERE/IZaXERERPzcndPeohYXERER8SNqcREREfFzmlV0A2NMgDGmmzfCiIiIiCQl2YqLtTYaaOGFLCIiInILrBf/52vudhWtMMZ8BUwGYn+0xVq7wSOpRERERBLgbsXlcde/fW9YZwH/+rUuERER8WtuVVystVU8HURERERujQbn3sQYk8cYM8oY86vr+cPGmLaejSYiIiISl7v3cRkDzAXCXM93A695IpCIiIikTAzWa4uvuVtxyWmtnYKrNcpaGwVEeyyViIiISALcHZx70RiTA9ddhY0xZYAIj6USERERt/m+HcR73K24dAd+Ae41xqwAcgFNPZZKREREJAHuVlzOAJWABwAD7OKfKdIiIiLiQ2lh7Im3uDvG5Ucgj7V2u7V2G/AkMNpzsURERETic7fFpQPwszGmPlAcGADU8VgqERERcduddB8Xd29At9YY8yowD7gMVLfWnvBoMhEREZGbJNlVZIyZYYz5xRjzC/A2kAG4AoxyrfOJmjUqs33bUnbuWM6bPTr7KkaK+Ftmf8sLyuwJBQrkY86cSWzYMJ/163+jc+cXAfjww3fYtGkBa9bMYfLkYYSGZvFx0sR56xqP/+EXGrV+hYbPd2b8lOmJ7rf1jz0UrdKIeYtX3HaZEefO0677u9Rp8TLtur9LxPkLAMyct5jGL3ShcesutOz4Jjv3/nnbZSUlrb+PAdKlC2bh4p9Y/vtMVq39lbd7do3d9m7v11m/cT5r1s/l5Y6tfZjy1t1JP7JorE08hDGmUlIvttYuSa6AwOD8qXqWDoeDP7Yvo1adFhw6dIRVv8+m1XOd+OOPPalZTKryt8z+lheUOSFBAe72BCcub97c5M2bm02btpEpU0ZWrpxJs2btyZ8/L4sXryQ6Opr+/f8HQK9eA2+7vGvRUbd9jBt5+hpHHlwIwJ79f9OjzydMHDaYoMBAOvR4n/de78hdBcLi7B8dHc1L3d8jXXAwjetWp0blcm6Vs2bjVqb/uoAP3ol738/BQ74lNHNm2rVqysjvfuTc+Qt07/gCG7f+wT2FChKaORPLVq3nm28nMnHYIADSF0zdn5jz9DXOGBySKscByJgxAxcvXiIwMJC5v03mrTf78cAD91Kh4pN0fLkH1lpy5srByROnbrusiAv7TCpEdlu7Qk29VqMY+dePXj23myXZ4mKtXZLU4q2QNypVshj79v3Fn38e4Nq1a0yZMp0G9Wv6Iorb/C2zv+UFZfaUo0ePs2nTNgAuXLjIzp17CQvLw4IFy4iOdt6Dcs2ajeTPn8+XMRPlrWu8/++DPPrQ/aQPSUdgYAAlHn+E+Ut/j7ffhKkzeapSWbJnC42zfvTEn2jevjuNX+jCV6MnuF3uouVraFjLWRFpWKsqC5evBqDYow8RmjkTAI898gDHTpy81VNLlj+8j6+7ePESAEFBgQQFBWKtpW27lnw88P+4/iU+NSotvhDjxcXX3P2tojLGmLXGmAvGmKvGmGhjzDlPh0tIWP68HDwUHvv80OEjhIXl9UUUt/lbZn/LC8rsDXfdVYDHH3+EtWs3xVn//PPNmDt3sW9CJcNb17hI4bvZsGUHZyPOEXn5CstWrefo8biVhWMnTrFg2SqaN6odZ/2KNRs5cCicScMGM3X0F+zYtZd1rspick6dOUuunNkByJkjG6fOnI23z08zf6N86Sdu8cyS50/vY4fDwbKVM9j75xoWLVzB+nWbKVz4Lp5uUpfFS3/mx59Gc8+9hXwdU5LhblvyV8CzwA9ACeB54P7EdjbGtAfaA5iAUByOjLcZU0R8KWPGDEycOJQePfpy3jWOAuDNN18hOjqKSZOm+TCd791bqCBt/vs07V/vTfqQdDxQpDAOR9zvhR/93wi6dWgdb/3KtRtZuXYTTds6u4EuRUby96FwSjz+H1q8/AZXr13jUmQkEecu0KSNc1xG9w6tKVeqeJzjGGO4uf1+zYYt/DTrN8Z/ffvdeP8GMTExVChbn9DQzHw3cSgPPXw/wemCuXz5CpUrNqJ+gxp8PWQgtWs86+uoKZYWxp54i9ud4NbavcaYAGttNPCtMWYjzgG7Ce07HBgOqT/GJfzwUQre0G9cIH8+wsOPpmYRqc7fMvtbXlBmTwoMDGTixKFMnvwz06fPiV3fqlVT6tSpRu3aLXyYLmnevMZN6tWgSb0aAHw+fBx5c+WMs337zr306OMcZ3Im4hzLVq0nICAArKVdy6Y0a1gr3jGvj0tJbIxLjmxZOXHyNLlyZufEydNkz5Y1dtuufX/y3sdfMfST3mT14OBpf3kf3ygi4jzLlv5O9eoVCQ8/yoxf5gIw45d5fD3kYx+nk+S4ewO6S8aYYGCTMeZjY0y3FLw2Va1dt4kiRQpTqFBBgoKCaNasITNmzvNFFLf5W2Z/ywvK7ElDh37Mrl17+fLLkbHrnnqqEt27d6Bp07ZERl72YbqkefMaX++mOXLsBAuW/k6d6hXjbJ87ZSTzXEuNSmXp1b0D1SqUoWyp4kybPZ9LlyIBZ5dSQl0+CalcrhTT5zgHCE+fs5Aq5UvFZnit1wAG9OxGoYL5U+sUE+Qv7+McObMTGpoZgJCQdFSpWp7du/cxa8ZvVKhYBoDyFUqzz8MzsOT2udvi8hzOisorQDegINDEU6GSEh0dTdfXejF71gQCHA7GjJ3Mjh27fRHFbf6W2d/ygjJ7StmyJWjZsglbt/7BqlWzAejd+xMGD36fdOmCmTnzO8A5QPfVV3v6MmqCvHmNu707kLMR5wkMDKBntw5kyZyJydN/BaB5w9qJvq5cqWLs//sgLTu+CUCGDCEM6NWdHDe0niSmXcsmvN77Y36a9RtheXMzuI/zGEPGTCIi4jz9PxsKQEBAAFNGfHq7p5ggf3gfA+TNk4uhwz/BERCAw+Fg2k+zmDtnEat+X8eIUZ/R6ZU2XLxwkS6dE+xISPPSwqBZb0lyOnScHY1JD9xlrd2VkgJSu6tIRNyTGtOhvS21p0N72vXp0P4ktadDe1pqTof2Jm9Ph25dqInX/taO/Wtq2p0OfZ3rVv+bgDmu54/78gZ0IiIi8o8Ya722+Jq741TeB0oBZwGstZuAwh7KJCIiIpIgd9uSr1lrI4yJ0zrk+2qXiIiI3FF/kN2tuGw3xvwXCDDG3Ae8Cqz0XCwRERGR+NztKuoCPILzBxYnAueA15J8hYiIiHhFDNZri6+51eJirb0E9HQtIiIiIj7hVsXFGHM/8AZQ6MbXWGv9a16diIjIv5Bu+R/fD8BQYCQQ7bk4IiIiIolzt+ISZa0d4tEkIiIickvupDvnujs4d4YxppMxJp8xJvv1xaPJRERERG7ibotLa9e/b9y0/p5UzCIiIiK3IC3M9vGWJFtcjDEljTF5rbWFrbWFgT7ANmAmUMIbAUVERESuS66raBhwFcAYUxEYAIwFIoDhno0mIiIi7rBe/J+vJVdxCbDWnnY9bg4Mt9ZOtda+CxTxbDQRERHxJ8aYgsaYRcaYHcaY7caYrq712Y0xvxlj9rj+zeZab4wxXxpj9hpjthhjiidXRrIVF2PM9XEw1YAbf8Pd3fExIiIicmeIAl631j4MlAE6G2MeBv4HLLDW3gcscD0HqA3c51raA8nOYE6u8jERWGKMOQlEAssAjDFFcHYXiYiIiI+llenQ1tojwBHX4/PGmD+A/EBDoLJrt7HAYuAt1/px1loLrDLGZDXG5HMdJ0FJVlystR8YYxYA+YB5rgODs6Wmy62emIiIiPgnY0x7nK0j1w231sYb92qMKQQUA1YDeW6ojBwF8rge5wcO3vCyQ651t1ZxAbDWrkpg3e7kXiciIiLe8U+7glfKGk4yE3SMMZmAqcBr1tpzxpgbX2+NMbcc2N0b0ImIiIgkyxgThLPS8r219ifX6mPGmHyu7fmA4671h4GCN7y8gGtdolRxERER8XMxWK8tSTHOppVRwB/W2k9v2PQL/9zMtjUw/Yb1z7tmF5UBIpIa3wKaGSQiIiKppxzwHLDVGLPJte4dYCAwxRjTFvgbaObaNhuoA+wFLgEvJleAKi4iIiJ+Lg3NKloOmEQ2V0tgfwt0TkkZqrj8CzhMYu8RuZNFRUf5OkKKBTj8q/c6Q8Gqvo6QYpHhy3wdIUXSh1XwdQRJY1RxERER8XNp4Vb83uJfX29ERETkjqYWFxERET+X3GyffxO1uIiIiIjfUIuLiIiIn/PmnXN9TS0uIiIi4jfU4iIiIuLn0sp9XLxBLS4iIiLiN9TiIiIi4ud0HxcRERGRNEgVFxEREfEb6ioSERHxc7oBnYiIiEgapBYXERERP6cb0ImIiIikQWpxERER8XMa4yIiIiKSBqnFRURExM/pBnQiIiIiaZBaXERERPxczB00q8itiosxpnsCqyOA9dbaTakbSURERCRh7ra4lHAtM1zP6wFbgA7GmB+stR97IpyIiIgk785pb3F/jEsBoLi19nVr7evAE0BuoCLwgoeyJShdunT8vmIm69f9xuZNC+n93uveLP6W1axRme3blrJzx3Le7NHZ13GS9corbdm4YT6bNi6gS5e2vo7jFmX2rPvvv5d1a+fFLqdO7uTVLu18HSueYcMGcfDARjasnx+7bsCHPdmyeRHr1s5jyuQRhIZm8WHC5DkcDtaumcvP08Z6tJzxU36mUasONGz5MuMnT4u3fc2GLZSp0YQmrTvTpHVnhoz+/rbLvHr1Kq+/O4DazdrQ4qXXOHzkGAAr12ygWZsuNH6uI83adGH1es825vvbZ7L8w92KS27gyg3PrwF5rLWRN633uCtXrlC9RjOeKPEUT5SoQc0alSldqrg3I6SYw+Hgyy8+oF79VjxatArNmzfioYfu83WsRD3y8AO0bdOCsuXq8USJGtSpU5177y3k61hJUmbP2717HyVK1qBEyRqUKl2LS5ci+Xn6r76OFc/48T9Qv8FzcdYtWLiMYsWrU6JkDfbs2Z/m/1C92qUdf+zc49Ey9uz/i6m/zGHiyM+ZOvYblqxcw4FD4fH2K170P0wd+zVTx35NxzYt3T7+4SPHeOGVN+Ot/2nmPLJkzsSvU0bzXPNGfPrNaACyZc3CVx+9z7TxQ/ig1+u83XfQrZ9cMvztM9kdMVivLb7mbsXle2C1Maa3MaY3sAKYYIzJCOzwWLpEXLx4CYCgoEACg4LS/K2OS5Usxr59f/Hnnwe4du0aU6ZMp0H9mr6OlagHHyzCmjWbiIy8THR0NMuWrqJRo9q+jpUkZfauqlXLs3//3xw4cNjXUeJZvnw1Z86cjbNu/vylREdHA7B6zUbyF8jni2huyZ8/H7VrV2P06IkeLWf/Xwd59JEHSB8SQmBgACUef5T5S1a4/foZcxfybLuuNGndmT4ffxl7fZOzcNnvNKxTHYAalSuwev0mrLU8dH8RcufKAUCRwndz+coVrl69mvITc4O/fSZLXG5VXKy1/YCXgbOupYO1tq+19qK11v0qeCpxOBysWzuPI4e3sGDBUtas3ejtCCkSlj8vB2/4JnPo8BHCwvL6MFHStu/YRfnypciePSvp04dQq1ZVChQI83WsJCmzdzVv1pDJk3/2dYxb8kLrZsydu8jXMRI1eHAf3n67PzExMR4tp8g9d7Nh83bORpwj8vJllv2+lqPHTsTbb/O2P3i6dSc6vP4ue/f/DcC+vw4wZ8ESxg8dzNSxX+NwOJg5z71revzEKfLmzglAYGAAmTJm4GzEuTj7/LZ4OQ8/UITg4ODbPMuE+dtnsjvupBYXt6dDW2vXGmP+BkIAjDF3WWsPJLSvMaY90B7ABITicGRMjayxYmJiKFGyBqGhWZj6wygeeeQBtm/flapl3Ml27tzLJ4O+YfasCVy8eInNW7a7/W3KV5TZe4KCgqhXrwY9ew3wdZQUe+utLkRFRTNxYvzxHGlBnTrVOXH8JBs2bqVixSc9Wta9he6iTctnaN+tJ+lDQnjgvntwOOJ+l334gXv5bepYMmRIz9KVa3j17b7MnjyK1es2sWPnXp5t2xVwduFnz5YVgFff7svh8GNci7rGkWMnaNLa2S3XqllDGtetkWyuvfv/5tNvRjP8sw9S+Yzl38Ld6dANgMFAGHAcuAvYCTyS0P7W2uHAcIDA4Pweq55FRJxj8ZIVzkFWabjiEn74KAVv+CZdIH8+wsOP+jBR8saMmcSYMZMA6Nf3LQ4dPuLjRMlTZu+oVasKGzdu5fjxk76OkiLPPfcMdWpXo1btZ30dJVFly5agXr0a1KpVlZCQdGTJkpmxY76k9QuveqS8JvVr0sTVRfL50DGxLSHXZcr4z5fOimVL0X/w15w5G4G1lga1q9Ot44vxjvnlgPcA5xiXnh8MZsxXcSed5s6Vg6PHT5I3dy6ioqK5cPESWV2DpY8eP0HXd/rx4btvcJcHWx/98TNZ/uHuGJd+QBlgt7W2MFAdWOWxVEnImTN77IyAkJAQqleryK5d+3wRxW1r122iSJHCFCpUkKCgIJo1a8iMmfN8HStJuVx9zQULhtGoUW0mTUr73QLK7B3Nmzfyu26iGk9V5vXuHWjStA2RkZd9HSdRvXoNpPA9Jbjv/jK0bNWJRYtWeKzSAnDKNRboyNHjLFiygjpPVY6z/eSp07FjCLfu2EWMtWQNzUKZEo/z2+Llsa+POHee8KPH3CqzSvkyTJ/tnPE1b/EySj9RFGMM585foFOP3rzW4UWKP5bgd+JU44+fycmx1npt8TV3u4quWWtPGWMcxhiHtXaRMeZzjyZLRL58eRg96nMCAhw4HA5+/HEGs2bPT/6FPhQdHU3X13oxe9YEAhwOxoydzI4du30dK0mTJw0nR45sXLsWxatdexJxUx90WqTMnpchQ3qqV6tIp05v+TpKosaN+4qKFcqQM2d29u1dQ7/+g3mzxysEpwtm9qwJAKxZs4FXurzj46S+1+2d/pw9d47AwEB6vt6JLJkzMXnaLACaN67LvEXLmTxtFgGBAYQEB/NJn/9hjOHewnfT5aXnaf9aT2JsDEGBgfTs3omwvHmSLfPpejV5u98n1G7WhtAsmfmkz/8AmDh1BgcPhTP02wkM/db5/9Pwzz3TXeSPn8nyD+NO7ckYMx9oBAwAcuLsLipprS2b3Gs92VUkTg5jfB1B0qC08M0opW4eY5HWeXoArSdcCl/m6wgpkj6sgq8j3JKoq4e9+sFcKqyS1/6DXxO+xKd/dNz9lGgIXAK6AXOAfUB9T4USERERSYhbXUXW2ouuhzHGmFnAKeuPX+dERET+hWwamKbsLUm2uBhjyhhjFhtjfjLGFDPGbAO2AceMMbW8E1FERETEKbkWl6+Ad4BQYCFQ21q7yhjzIDARZ7eRiIiI+NCd1AmS3BiXQGvtPGvtD8BRa+0qAGvtTs9HExEREYkruRaXG4fMR9607c6p3omIiKRhaeFW/N6SXMWlqDHmHGCA9K7HuJ6HeDSZiIiIyE2SrLhYawO8FURERERujca4iIiIiKRBbv86tIiIiKRNd9IYF7W4iIiIiN9Qi4uIiIif051zRURERNIgVVxERETEb6irSERExM/FaDq0iIiISNqjFhcRERE/p8G5IiIiImmQWlxERET8nMa4iIiIiKRBanERERHxcxrjIiIiIpIGqcXlX+BO6tsU9xlfB7gF0TExvo6QIpmD0/s6QoqlD6vg6wgpEhm+zNcR/MKd9HdALS4iIiLiN5JtcTHGpLPWXklunYiIiPiGxrjE9bub60REREQ8KtEWF2NMXiA/kN4YU4x/usyzABm8kE1ERETccCeNcUmqq6gm8AJQAPj0hvXngXc8mElEREQkQYlWXKy1Y4Gxxpgm1tqpXswkIiIiKaAxLnEtMMZ8aoxZ51oGG2NCPZ5MRERE5CbuVFxG4eweauZazgHfejKUiIiISELcuQHdvdbaJjc872OM2eSpQCIiIpIy1vrXzRtvhzstLpHGmPLXnxhjygGRnoskIiIikjB3Wlw64hykG4pzSvRpoLVHU4mIiIjbYu6gwbnJVlystZuAosaYLK7n5zyeSkRERCQB7tzyPxToDVR0PV8C9LXWRng4m4iIiLjB3kE3oHNnjMtoNKtIRERE0gDNKhIREfFzd9IYF80qEhEREb+hWUUiIiJ+7k4a46JZRSIiIuI33JlVlAPnrKLygDXGLMc5q+iUp8OJiIhI8mLuoBYXd8a4TAJOAE2Apq7Hkz0ZSkRERCQh7oxxyWet7XfD8/7GmOaeCiQiIiIpYzWrKI55xphnjTEO19IMmOvpYCIiIiI3S7TFxRhzHrA4ZxK9Box3bQoALgBveDydiIiIJOtOmlWUaIuLtTaztTaL61+HtTbItTistVm8GfJmNWtUZvu2pezcsZw3e3T2ZRS3+Vtmf8s7Yvhgwg9tZtPGBb6OkiL+dJ3vv/9e1q2dF7ucOrmTV7u083WsJPnT+8LhcLBkxS9M+mE4AHfdXYDfFv3I+s0LGDX2C4KCgnycMHHeus7jp/xMo1YdaNjyZcZPnhZv+5oNWyhTowlNWnemSevODBn9/W2Xedv0J2kAABUYSURBVPXqVV5/dwC1m7WhxUuvcfjIMQBWrtlAszZdaPxcR5q16cLq9bovq7e401WUpjgcDr784gPq1W/Fo0Wr0Lx5Ix566D5fx0qSv2X2t7wA48ZNoW69lr6OkSL+dp13795HiZI1KFGyBqVK1+LSpUh+nv6rr2MlyZ/eFx06vcDuXXtjn7/f702GfP0tTxStRsTZCJ5r/YwP0yXNG9d5z/6/mPrLHCaO/JypY79hyco1HDgUHm+//2/vzKOsKM4+/PwYQEA2EZTghorBLYqKa1BxQ9w1GJCokaghSmLcjQZUIHHXxKNiCIkKohgwBBfUiBsIalRUQFBjFHBhFQUUP0EY3u+PqgvNcLeZuTOXC+9zzj3TU11d9avqqreqq6qr9917T0YPG8ToYYO48Nz8Nc2Zt4Bev7lqPfd/jR1H0yaNeWbU/Zzd41T+dO/9AGzRvCn33NKfMcP/wg39LueagbdXPXFOpSi5jssB++/Dxx/PZtasT1m5ciWjRj3OyScdW2xZWSk1zaWmF2DipNf5avGSYsuoFKWYzymOPLITM2d+wqefzim2lKyUSrlo06Y1Xbp25sFho9a4HXb4QTw+5t8APPLwGI4/8ZhiyctJbeTzzNmf8aM92tOwQQPq1i2jY4cf8fyEV/K+/slnX+SM8y+m2zm/ZsCtd1FeXp7XdS9OfI1Tjj8agC6dD+X1t6ZgZuz2w3Zs1WpLANrtuAPLV6zg+++/r3zCCsRqrNZ+xabkOi5ttmnNZ4le9udz5tGmTesiKspNqWkuNb2lSinnc4/upzBy5GPFlrHRcOOt/bi+3y2sXh0ahRZbbsHSJd+saVznzplPmzZbF1Ni0Wm30w68PXUGS5Z+zXfLlzPxtTeZv+CL9fxNnf4+PzmnDxdcfi0fzfwEgI9nf8q/X5jA8MF3MHrYIOrUqcPYcS/lFe/CL76k9VYtAahbt4zGmzdiydJ192F9bvwkdm/fjvr161czlU4+5PM6NPFbRbuY2QOSWgGNzWxWFv+9gd4AKmtGnTqbF0Ss4zjFp169epx4Yhf69rup2FI2Co7tegSLvviSqVNm8ONDDyy2nA2Wndtuz7ln/pTel/alYYMGtN9lJ+rUWffZe/f2O/Pc6GE0atSQl199g99eM5CnR97H65On8N4HH3HGeRcDsGLFClps0RyA314zkDlzF7By1UrmLfiCbueEtWZndT+F007oklPXRzM/4U/33s+QP99Q4BRXjk1pcW4+O+deD3QE2gMPAPWAh4AfZ7rGzIYAQwDq1t+moLk5d858ttu2zZr/t93mB8ydO7+QURScUtNcanpLlVLN565dj+Cdd95l4cJFxZayUXDgQfvR9fijOKbL4WzWYDOaNGnMzbf2o1nzJpSVlVFeXk6bbVozd+6CYkstOt1OOpZucTr1zsFD14yEpGi8+dqH5MMOOYA/3jGIxUuWYmacfNzRXHrhL9YL866brgPCGpe+N9zB0HtuXef8Vq22ZP7CRbTeqhWrVpWz7Nv/o3mz8H7K/IVfcPHv/8CN117B9om67NQs+UwVnQacDHwLYGZzgSY1KSobb06eQrt2O9K27XbUq1eP7t1P4cmx44olJy9KTXOp6S1VSjWfe/Q41aeJCsjA/rezZ/tO7L1HZ87rdQkTJ7xG7/MuZ+LLr3PKaV0B6HnmaTzz1PNFVlp8vozraObNX8gLE17h+GM6r3N+0ZdfrRl5ePe9/7LajObNmnJQxw48N37SmuuXfv0Nc+fn1xE8otNBPP50yPtx4ydy4H57I4mvv1lGnyuv55ILfsG+e+1RoBRWndVmtfYrNvlMFX1vZibJACQVdd6nvLyciy/px9NPjaCsTh2GDhvJe+99WExJOSk1zaWmF+Ch4YM4/LCDadmyBbNnTmbAwNt5YOg/ii0rK6WYz40aNeToow6jT5/fFVtKXpRiuUjR/9pbuW/onfS99jKmTXuP4cMeLbakjNRWPl/6+z+y5OuvqVu3Ln0v70PTJo0ZOeYpAHqcdgLjXprEyDFPUVa3jAb163PbgKuRxM477sBFv/w5vS/py2pbTb26del7WR/atM69bugnJx7LNX+4jeO6n0uzpk24bcDVADwy+kk++3wugx8YweAHRgAw5M4b2DJOQTk1h3LNi0m6AtgFOAa4CTgXGGFmd+cTQaGnihzHyQ8VW0AVKDVj0aR+w2JLqDTffP9dsSVUiu/mTiy2hCpRr+VOtVoFt2jcrtaqz+JlHxXVvOQccTGz2yUdA3xNWOdynZk9V+PKHMdxHMdxKpDXW0Wxo+KdFcdxHMfZANkQ9lepLfL9VlEyRwRYsbf9dxzHcRxn0yNjx8XMivbmkOM4juM4+bMp7eOS83VoSeelcbu5ZuQ4juM4juNkJp81Lt0kLTezhwEkDQJKbym94ziO42ykbAj7q9QWeXVcgCckrQa6AkvM7NyaleU4juM4jrM+2Rbntkj8ez7wGPAKMEBSCzP7qqbFOY7jOI6TG/O3igB4i3XfKhJwQvwZsFONq3Mcx3Ecx0mQ7a2iHWtTiOM4juM4Ti7y2oBO0iFA26R/M3uwhjQ5juM4jlMJfHFuAknDgZ2BKUB5dDbAOy6O4ziO49Qq+Yy4dAR2t01pdxvHcRzHKSE2pSY65wZ0wHSgdU0LcRzHcRzHyUU+Iy4tgfckvQGsSDma2ck1pspxHMdxnLzx16HXpX9Ni3Acx3Ecx8mHnB0XM5uQ/F9SJ6AnMCH9FY7jOI7j1Cab0hqXfF+H3gf4GfBTYBYwuiZFOY7jOI7jpCPblv8/JIys9AQWASMBmdkRtaTNcRzHcZw82JRGXLK9VfQBcCRwopl1MrO7WbuPi+M4juM4znpI6irpv5I+knR1ocPP1nH5CTAPeEnS3yQdRfhekeM4juM4GxBWi79sSCoDBgHHAbsDPSXtXqBkAlk6Lmb2mJmdAewKvARcAmwl6S+SuhRShOM4juM4GwUHAB+Z2Uwz+x74B3BKISPI562ib4ERwAhJWxAW6P4OGJdPBKu+n1NjozSSepvZkJoKv9CUml4oPc2lphdcc21QanrBNdcGpaY3GzXZ1lZEUm+gd8JpSCIftwE+S5z7HDiwkPHns3PuGsxssZkNMbOjCimiGvTO7WWDotT0QulpLjW94Jprg1LTC665Nig1vRsEsR/QMfGr1c5fpToujuM4juM4WZgDbJf4f9voVjC84+I4juM4TqF4E9hF0o6S6gNnAE8UMoK8NqDbgCm1uclS0wulp7nU9IJrrg1KTS+45tqg1PRu8JjZKkm/AZ4FyoD7zWxGIePQprRpjeM4juM4pY1PFTmO4ziOUzJ4x8VxHMdxnJKhRjoukraUNCX+5kuak/i/fpbr2kqanuHcQElHZzjXS1KbCm5nSOorqbOkQ4qRnmIhqTxqmy7pUUmNcvgfL6ljPJ4tqWXtKM1OIh0zJE2VdLmkkulsS2ot6R+SPpb0lqSn4zfAKhNGc0l9akBb35iv02IeV3ufhWQ5qo6fSsa5XjoylWFJJ2fafrwQdiKTnuqGmQi7s6SxhQqvCvGn6uNUSW8XIr/yiPNUSSZp1zz9Z7r3yyoZb6X8ZwlnvbbJqT41sjjXzL4EOgBI6g8sM7Pbqxnmdenc4/bCvYDpwNzEqeOAu4CTgGXAq9WIO2t6JNU1s1VVDb+ySCozs2zfjfrOzFJ6HwYuAP5UK+KyIEmEdVWr87wkmY6tCBshNgWurxBureZ/PsS0jgGGxR2okbQ3sDXwYSWCag70Ae4toLaDgROBfc1sRTT0G1wHPBeVTYeZPUGatxsk1QU6U007sSHna4HqSLI+HgvcBBxebXHZ6QlMin+vz+F3Q6QX67dNTjUp2tOrpD0kvRF78NMk7RJPlSl8G2mGpHGSGkb/QyWdHo9nS7pF0tuEAt0ReDiG1TA2Gh2ArwiN9qXx3KFxVOfFGOcLkrZPhD9Y0mRJH0o6MYf+lP/XgVsldZD0nxjuGIVdhiuOZrSUNDtb+iWdlXD/a+yYIWmZpDskTQUOrkRWTwTaVXxak3SPpF450niZwqjNdEmXRLebJf064ae/pCvi8ZWS3ozpGRDd2ip8bOtBQgXeLl1cuTCzhYTNon6jQC9JT0h6EXhB0uaS7o95946kU2L86+Vz9PtUfHKcLqlHVTTl4AhgpZkNTqRhKjBJ0m0x3ndTcUtqHMvj29E9tUX2zcDOUf9tBdL2A2CRma2IuhaZ2VxJ18X7N13SkFiPUmX4lpiPH0o6NLo3VBhRel/SGKBhKgKFT4NMjvV4QIF055WOeO6iRF7uGjX1knRPPE7W31FUsBOF1BPt1YA0ejKV2baSJkb/aUc2JO0fr9lZ0n6SJiiM6j0r6QfRz3hJd0qaDFxcxTRloimwOMaTqewi6dpY/ydJeiRlK/JBUmOgE3Ae4ZXalHvnmLZ/SvpA0sOpsprw01DSM5J+mSbc9exUhvj/HMvvC5JaRbdMdn49d4X2ap22Kd+0Ozkwsxr9Af2BK9K43w2cGY/rE4xeW2AV0CG6jwLOisdDgdPj8WzgqkRY44GOif/3BR5MFz/wJHBOPD4XeCwR/r8JnbldCNsUN8iUnuh/LFAW3acBh8fjgcCdFbUBLYHZWdK/W9RXL7rfC/w8HhvQPc88Xxb/1gUeBy4kPFGOTfi5B+iVRuPsqHM/4F1gc6AxMAPYJ/4mJMJ5j9AZ6UJ4tVAxD8cCh8V7uho4qAplZ1katyWEUYte8R61iO43JspKc8KoxuYZ8rkb8LdEmM1qoNz/FvhzGvduwHOE1wS3Bj4lNHh1gaaJcvJRzMu2wPQCa2sMTIl5dG+i3LZI+BkOnJQoH3fE4+OB5+PxZYRXHQH2ItTdjsmwYjrHA3ulq6s1lI7ZwEXxuA/w93jcC7gnUd+T9bc/aexUDevJVGYbEW0PwRZNjsedo+ZDgLeA7YF6hFGiVtFPj8Q9GQ/cW8ByUx7T9wGwFNgvumcqu/tH/w2AJsD/KpPHwJnAffH41UR8nWP82xJszWtAp0RetwWeJ9rO6J6yiWntVJq4jbV247pEuclk53Paf/8V7lfM9QKvAb+X9DtgBzP7LrrPMrMp8fgtQiFMx8gsYXcFnslw7mDClAME49wpcW6Uma02s/8BMwkfmMzGo2ZWLqkZ0NzMJkT3YYRGOxvp0n8UocPwpqQp8f+dov9yYHSOMFM0jNdPJjSM9+V5XZJOwBgz+9bMlgH/Ag41s3cIH9tsozD1sdjMPiMYhC7AO8DbhLxLjaJ9Ymb/qYKGXDxnZl/F4y7A1THd4wnGcnvS5/O7wDFxFOFQM1taA9oy0Ql4xMzKzWwBMIFg4AXcKGkawehuQ+jYFJx4P/cjjGB9AYxUGH07QtLrkt4FjgT2SFz2r/g3WScPAx6KYU4jGO8U3RVGRN+J4RT067A50pFJb0UetexTrjWtJ1OZrQf8Ld6HR1k373YjNLwnmdmnQHtgT+C5GE4/QoOeIpudrCzfmVkHM9uVYGMfjCMdmcruj4HHzWy5mX1DeCirDD0JH+gj/u2ZOPeGmX1uYdp5Cuve48eBB8zswTRhZrNTSVazNu8eAjplsvNVtP9ONai1DegkncbaOcrzzWxEHKY9AXha0q8InYUVicvKSQw/V+DbLNF1ITzZVpaKm9rk2uQmm4YUq1g7JddgTcDp0y/Cmohr0oSzvBJGds1cdApJSR3raKkCjwKnA61ZW7kF3GRmf60Qb1vyy6ecSNqJUCYWRqdkuAK6mdl/K1z2fsV8NrMXJe1LGD34o6QXzGxgITQmmEHIo3w5E2hFeKpcqTClWJ17lJVYlsYD42MD+SvCqElHM/tMYS1XMv5UvSwnh92QtCNhVHJ/M1ssaSg1lJY06TinEnoLUi6roSdtmY15vwDYm1BnlydOzyPk5T6EdRMCZphZpunjgqcRwMxeU1jD04pQjwpadiW1IHSefyTJCCN3JunK6KViO5G8x68AXSWNMLOKNjytncoD3/BsA6LWRlzMbEzsrXcws8mxEZppZncResh7VSP4bwhDkcTeb10LC2rXORd5lbXzpWcS1oCk+KmkOpJ2Jox0VGwE0xKf2Bcn5sbPJjxJQxi63C8er2nIMqT/BeB0hYWoSGohaYd8NOTBJ8DukjaT1JwwmpONicCpkhpJ2hw4jbV5NZKQh6cTOjEQdkk8N85LI2mbVDoKQZxjHkwYsk1nRJ4lrGtIrcvYJ/5dL58VVvn/n5k9BNxGmFosNC8Cmyl8RTWVhr0IU109JJXFNB0GvAE0AxZGw38EkLrvFctvtZHUXmvXlEFYD5Yq64viPcyn0/Uy8LMY5p6srcNNCQ3mUklbExbKF5wM6fikisFVO5+roCdtmSWUhXlxNOFsQqOdYgmhE36TpM6E+9ZKYWEwkupJSo6U1QgK63TKgC/JXHZfAU6S1CCWqazrBitwOjDczHYws7Zmth0wC8hn/dF1hPU3g9Kcy9dO1WFtHfgZMCmTnc9h/wtef53ibvnfHThb0kpgPmG+t2kVwxoKDJb0HXAHYbgyxZPAPxUWjF0Ufw/EnvsXwC8Sfj8lNCJNgQvMLPmkk4tzooZGhJGjVLi3A6NiA/ZUwv966TezryT1A8YpvPa7Evg1VTfGa4hP0aMIC2RnEYZKs/l/Oz4pvxGd/h6niTCzGZKaAHPMbF50GydpN+C1aIeXAWcRnoaqSmrKqx5h5Go4md+O+gNwJzAt5t0sgqFMV872B26TtJqQxxdWQ2NazMziKOOdcZpqOaETewlhLcRUwlPcVWY2X+HtryfjU/pkwjoCzOxLSa8obBPwjJldmSa6ytIYuDt2YFcR1iT0JjSK0wn59GYe4fyFUJfeB94nTINgZlMlvRPT8BmhAasJMqWjMg1kinXshJlNzHVBAfRkKrP3AqMl/Zyw7m6dURMzW6Dw8sAzhHV6pwN3pR7aYpgF3WI9kqqPEEYuzolT5ZnK7puSniBMIS4gTNHmOy3bE7ilgtvo6J7P9NfFwP2SbjWzq1KOWezUwgrXfwscEO3xQsLaIchs5zO5D2Vt23RwYkmEUw02ui3/Jf2d0MhWak1FbKTHmtk/a0SY4zjOJoakxma2LDboLwO9zeztYutySptS/8jiepjZ+cXW4DiO4wAwRNLuhDUvw7zT4hSCjW7ExXEcx3GcjZeS2T7dcRzHcRzHOy6O4ziO45QM3nFxHMdxHKdk8I6L4ziO4zglg3dcHMdxHMcpGf4f61aE1PA4y20AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x720 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>8. Comparing Squared loss and Cross Entropy loss</h1>\n"
      ],
      "metadata": {
        "id": "xVJ98-swLSTo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Here we are trying to compare which loss function is better i.e., cross entropy loss or squared loss.<br> <br>\n",
        "We again run a hyperparameter sweep but this time also including loss function as one of the hyperparameters. Using this we can easily compare both the loss functions at once and arrive at a correct decision.<br> <br>\n",
        "\n",
        "We have not used \"bayes\" method here and preferred \"random\" method because \"bayes\" method is kindof heuristic and will slowly stop using the loss function that is not performing well."
      ],
      "metadata": {
        "id": "NMUc5jqAtwOL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def compare():\n",
        "\n",
        "  '''This function is used to exploit the wandb hyperparameter sweep \n",
        "  function to get the best hyperparameters.\n",
        "\n",
        "  This is used to compare the two loss functions. Its same as the previous code\n",
        "  but with added loss functions as hyperparameters.\n",
        "\n",
        "  It takes in no inputs and gives no outputs.\n",
        "  \n",
        "  Instead it logs everything into the wandb workspace'''\n",
        "\n",
        "  config_defaults = {\n",
        "      'number_hidden_layers': 2,\n",
        "      'number_neurons': 32,\n",
        "      'learning_rate': 0.001,\n",
        "      'initialization_type': \"xavier\",\n",
        "      'activation_function':'sigmoid',\n",
        "      'mini_batch_size' : 64,\n",
        "      'max_epochs': 5,\n",
        "      'lambd': 0,\n",
        "      'optimization_function': \"adam\",\n",
        "      'loss_function': \"cross_entropy\"\n",
        "      \n",
        "  }\n",
        "\n",
        "  wandb.init(config=config_defaults)\n",
        "  config = wandb.config\n",
        "\n",
        "\n",
        "  #Forming the layer_sizes i.e., the architecture of our neural network\n",
        "  layer_sizes = [784]\n",
        "  for i in range(config.number_hidden_layers):\n",
        "    layer_sizes = layer_sizes + [config.number_neurons]\n",
        "  layer_sizes  = layer_sizes + [10]\n",
        "\n",
        "  #Collecting all the hyperparameters from the wandb run\n",
        "  learning_rate = config.learning_rate\n",
        "  initialization_type = config.initialization_type\n",
        "  activation_function = config.activation_function\n",
        "  loss_function = config.loss_function\n",
        "  mini_batch_size = config.mini_batch_size\n",
        "  max_epochs = config.max_epochs\n",
        "  lambd = config.lambd\n",
        "  opt_fun = config.optimization_function\n",
        "\n",
        "  #Calling the respective hyperparameters\n",
        "  if opt_fun == \"adam\":\n",
        "    optimization_function = adam\n",
        "  elif opt_fun == \"nadam\":\n",
        "    optimization_function = nadam\n",
        "  elif opt_fun == \"mini_batch_gd\":\n",
        "    optimization_function = mini_batch_gd\n",
        "  elif opt_fun == \"momentum_gd\":\n",
        "    optimization_function = momentum_gd\n",
        "  elif opt_fun == \"nesterov_gd\":\n",
        "    optimization_function = nesterov_gd\n",
        "  elif opt_fun == \"rmsprop\":\n",
        "    optimization_function = rmsprop\n",
        "  else:\n",
        "    print(\"Wrong optimization function\")\n",
        "    exit()\n",
        "\n",
        "  #Forming meaningful run name using the hyperparameters\n",
        "\n",
        "  name_run = str(learning_rate) + \"_\" + initialization_type[0] + \"_\" + \\\n",
        "  activation_function[0] + \"_\" + str(mini_batch_size) + \"_\" + str(max_epochs) + \\\n",
        "  \"_\" + str(lambd) + \"_\" + opt_fun[:4]\n",
        "\n",
        "  wandb.run.name = name_run\n",
        "  wandb_log=True\n",
        "  #Calling the fit function to train the neural network with the current hyperparameters\n",
        "  parameters = fit(X_train, y_train, layer_sizes,wandb_log, learning_rate, initialization_type, activation_function, loss_function, mini_batch_size, max_epochs, lambd, optimization_function)\n",
        "\n",
        "  \n",
        "  wandb.run.save()\n",
        "  wandb.run.finish()"
      ],
      "metadata": {
        "id": "ZhrLpiWRqGOT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def loss_compare_sweeper(entity_name,project_name):\n",
        "  #Declaring the dictionary of all choices for the hyperparameters.\n",
        "  hyperparameters = {\n",
        "      \"learning_rate\":{\n",
        "        'values': [0.001, 0.0001]\n",
        "      },\n",
        "\n",
        "      \"number_hidden_layers\": {\n",
        "          'values' : [3, 4, 5]\n",
        "      },\n",
        "\n",
        "      \"number_neurons\": {\n",
        "        'values': [32, 64, 128]\n",
        "      },\n",
        "\n",
        "      \"initialization_type\": {\n",
        "          'values' : [\"xavier\", \"random\"]\n",
        "      },\n",
        "\n",
        "      \"activation_function\": {\n",
        "          'values': [\"sigmoid\", \"tanh\", \"relu\"]\n",
        "      },\n",
        "\n",
        "      \"mini_batch_size\": {\n",
        "          'values': [16,32,64]\n",
        "      },\n",
        "\n",
        "      \"max_epochs\": {\n",
        "          'values': [5, 10, 15]\n",
        "      },\n",
        "\n",
        "      \"lambd\": {\n",
        "          'values': [0, 0.0005, 0.5]\n",
        "      },\n",
        "\n",
        "      \"optimization_function\": {\n",
        "          'values': [\"mini_batch_gd\", \"momentum_gd\", \"nesterov_gd\", \"rmsprop\", \"adam\", \"nadam\"]\n",
        "      },\n",
        "\n",
        "      \"loss_function\": {\n",
        "          'values': [\"cross_entropy\", \"squared_loss\"]\n",
        "      }\n",
        "\n",
        "\n",
        "  }\n",
        "\n",
        "\n",
        "  #Using random method for hyperparameter sweeps \n",
        "  sweep_config = {\n",
        "      'method' : 'random',\n",
        "      'metric' :{\n",
        "          'name': 'Validation_Accuracy',\n",
        "          'goal': 'maximize'\n",
        "      },\n",
        "      'parameters': hyperparameters\n",
        "  }\n",
        "  sweep_id = wandb.sweep(sweep_config, entity=entity_name,project=project_name)\n",
        "  wandb.agent(sweep_id, compare, count = 50)"
      ],
      "metadata": {
        "id": "1QeeVSeeqGLm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss_compare_sweeper(entity_name,project_name)"
      ],
      "metadata": {
        "id": "EYOmvodmqGI4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>10. MNIST Trials </h1>"
      ],
      "metadata": {
        "id": "8bmuT3iRPsCg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The MNIST database (Modified National Institute of Standards and Technology database) is a large database of handwritten digits that is commonly used for training various image processing systems. It contains 60,000 training images and 10,000 testing images. Each image is of the size 28*28 pixels.<br>\n",
        "Source: https://en.wikipedia.org/wiki/MNIST_database\n",
        "<br> <br>\n",
        "We are using the hyperparameters tuned for the Fashion-MNIST dataset and trying to build models to recognize the handwritten digits of the MNIST Dataset. <br>\n",
        "The main aim is to check whether the learnings from one image classification task can be applied to another task or not."
      ],
      "metadata": {
        "id": "OrZmMTzMm81x"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from keras.datasets import mnist\n",
        "#sklearn library is used only for train test validation split\n",
        "\n",
        "\n",
        "def prepare_data_mnist():\n",
        "\n",
        "  '''This function is used to load the data, define the class labels, performing\n",
        "      the train-test-validation split, normalizing the data, flattening each data\n",
        "      point, converting the class labels to one hot encoded vector.\n",
        "\n",
        "      It return all the split data sets '''\n",
        "\n",
        "  # Loading data from online source\n",
        "  (train_x,train_y),(test_x,test_y)=mnist.load_data()\n",
        "\n",
        "  # Defining labels for data\n",
        "  num_classes = 10\n",
        "  labels=[i for i in range(10)]\n",
        "\n",
        "  print(\"Number of data points in train data (initially) - \", len(train_x))\n",
        "  print(\"Number of data points in test data (initially) - \", len(test_x))\n",
        "\n",
        "  #performing the train-validation split\n",
        "  train_x, val_x, train_y, val_y = train_test_split(train_x, train_y, test_size=0.1, random_state=40)\n",
        "  print(\"Shape of each image - 28x28\" )\n",
        "  image_shape=train_x.shape[1]*train_x.shape[2]\n",
        "  print(\"shape of each image (1D) - \",image_shape)\n",
        "\n",
        "  #storing the number of points in each set\n",
        "  train_image_count=len(train_x)\n",
        "  val_image_count = len(val_x)\n",
        "  test_image_count=len(test_x)\n",
        "\n",
        "  # Creating a matrix of image data \n",
        "  # each image is represented as a row by flattening the matrix: converting (60000,28,28) tensor to (60000,784) matrix\n",
        "  X_train=np.zeros((train_image_count,image_shape))\n",
        "  X_val=np.zeros((val_image_count,image_shape))\n",
        "  X_test=np.zeros((test_image_count,image_shape))\n",
        "\n",
        "  # converting the images into grayscale by normalizing\n",
        "  for i in range(train_image_count):\n",
        "    X_train[i]=(copy.deepcopy(train_x[i].flatten()))/255.0 \n",
        "  for i in range(val_image_count):\n",
        "    X_val[i]=(copy.deepcopy(val_x[i].flatten()))/255.0\n",
        "  for i in range(test_image_count):\n",
        "    X_test[i]=(copy.deepcopy(test_x[i].flatten()))/255.0\n",
        "    \n",
        "  #One hot encoding the label vectors to represent a probability distribution\n",
        "  y_train = np.zeros((train_y.size, 10))\n",
        "  y_train[np.arange(train_y.size), train_y] = 1\n",
        "\n",
        "  y_val = np.zeros((val_y.size, 10))\n",
        "  y_val[np.arange(val_y.size), val_y] = 1\n",
        "\n",
        "  y_test = np.zeros((test_y.size, 10))\n",
        "  y_test[np.arange(test_y.size), test_y] = 1\n",
        "\n",
        "  \n",
        "\n",
        "  #returning all the datasets along with the labels\n",
        "  return X_train,X_val,X_test,y_train,y_val,y_test,labels\n",
        "  "
      ],
      "metadata": {
        "id": "80Ftpn6ojN_J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_val, X_test, y_train, y_val, y_test, lab = prepare_data_mnist()"
      ],
      "metadata": {
        "id": "7hNoGpvKgUkI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f3c57a36-2b48-4a8f-89f9-6692b666b213"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of data points in train data (initially) -  60000\n",
            "Number of data points in test data (initially) -  10000\n",
            "Shape of each image - 28x28\n",
            "shape of each image (1D) -  784\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def custom_config_mnist(mnist_config,wandb_log, config_number):\n",
        "\n",
        "  '''This function is used to train the neural network for the given hyperparameter\n",
        "  configuration and also evaluate the model finally logging the results onto \n",
        "  Wandb.\n",
        "\n",
        "  We have essentially used three hyperparameter configurations that were chosen\n",
        "  after thorough exploration from the Fashion MNIST dataset.'''\n",
        "\n",
        "\n",
        "  #Forming the network archtecture and storing it in the list.\n",
        "  num_layers=mnist_config[\"number_hidden_layers\"]\n",
        "  neurons=mnist_config[\"number_neurons\"]\n",
        "  layer_sizes = [784]\n",
        "  for i in range(num_layers):\n",
        "    layer_sizes = layer_sizes + [neurons]\n",
        "  layer_sizes  = layer_sizes + [10]\n",
        "  \n",
        "  # For local testing purposes, wandb_log is set to False. Then it doesnt log the\n",
        "  # results to the wandb workspace. Instead it prints here.\n",
        "  if (wandb_log== True):\n",
        "      #initializing the run\n",
        "      wandb.init(config=mnist_config, project = project_name, entity=entity_name)\n",
        "      config = wandb.config\n",
        "      #storing the hyperparameters in local variables\n",
        "      opt_fun = config.optimization_function\n",
        "\n",
        "      #Choosing the correct optimization function\n",
        "      if opt_fun == \"adam\":\n",
        "          optimization_function = adam\n",
        "      elif opt_fun == \"nadam\":\n",
        "          optimization_function = nadam\n",
        "      elif opt_fun == \"mini_batch_gd\":\n",
        "          optimization_function = mini_batch_gd\n",
        "      elif opt_fun == \"momentum_gd\":\n",
        "          optimization_function = momentum_gd\n",
        "      elif opt_fun == \"nesterov_gd\":\n",
        "          optimization_function = nesterov_gd\n",
        "      elif opt_fun == \"rmsprop\":\n",
        "          optimization_function = rmsprop\n",
        "      else:\n",
        "          print(\"Wrong optimization function\")\n",
        "          exit()\n",
        "\n",
        "      wandb.run.name = \"Mnist_Config_\"+str(config_number)\n",
        "\n",
        "      #calling the fit function to train the model using the best hyperparamters obtained from above\n",
        "      parameters = fit(X_train, y_train, layer_sizes,wandb_log, \n",
        "                     mnist_config[\"learning_rate\"],\n",
        "                     mnist_config[\"weight_type\"], \n",
        "                     mnist_config[\"activation_function\"], \n",
        "                     mnist_config[\"loss_function\"],\n",
        "                     mnist_config[\"mini_batch_size\"],\n",
        "                     mnist_config[\"max_epochs\"], \n",
        "                     mnist_config[\"lambd\"],\n",
        "                     optimization_function)\n",
        "      res = predict(X_test,y_test,parameters, mnist_config[\"activation_function\"], layer_sizes)\n",
        "        # Converting the one hot encoded vectors back to label_id's\n",
        "      y_t=[]\n",
        "      for k in range(len(y_test)):\n",
        "          y_t.append(y_test[k].argmax())\n",
        "\n",
        "      y_pred=[]\n",
        "      for k in range(len(res)):\n",
        "          y_pred.append(res[k].argmax())\n",
        "\n",
        "      #calculating the test accuracy using the test data\n",
        "      test_accuracy=calc_test_accuracy(y_pred,y_t)\n",
        "      print(\"Test accuracy:\",test_accuracy)\n",
        "\n",
        "      wandb.log({\"conf_mat\":wandb.plot.confusion_matrix(preds=y_pred,y_true=y_t,class_names=labels),\"Test Accuracy\": test_accuracy})\n",
        "\n",
        "      wandb.run.save()\n",
        "      wandb.run.finish()\n",
        "\n",
        "  else:\n",
        "    #Forming the layer_sizes i.e., the architecture of our neural network\n",
        "      optimizer=mnist_config[\"optimization_function\"]\n",
        "      \n",
        "      if optimizer == \"adam\":\n",
        "          opti = adam\n",
        "      elif optimizer == \"nadam\":\n",
        "          opti = nadam\n",
        "      elif optimizer == \"mini_batch_gd\":\n",
        "          opti = mini_batch_gd\n",
        "      elif optimizer == \"momentum_gd\":\n",
        "          opti = momentum_gd\n",
        "      elif optimizer == \"nesterov_gd\":\n",
        "          opti = nesterov_gd\n",
        "      elif optimizer == \"rmsprop\":\n",
        "          opti = rmsprop\n",
        "      else:\n",
        "          print(\"Wrong optimization function\")\n",
        "          exit()\n",
        "\n",
        "      parameters = fit(X_train, y_train, layer_sizes,wandb_log, \n",
        "                     mnist_config[\"learning_rate\"],\n",
        "                     mnist_config[\"weight_type\"], \n",
        "                     mnist_config[\"activation_function\"], \n",
        "                     mnist_config[\"loss_function\"],\n",
        "                     mnist_config[\"mini_batch_size\"],\n",
        "                     mnist_config[\"max_epochs\"], \n",
        "                     mnist_config[\"lambd\"],\n",
        "                     opti)\n",
        "      res = predict(X_test,y_test,parameters, mnist_config[\"activation_function\"], layer_sizes)\n",
        "\n",
        "      y_t=[]\n",
        "      for k in range(len(y_test)):\n",
        "          y_t.append(y_test[k].argmax())\n",
        "\n",
        "      y_pred=[]\n",
        "      for k in range(len(res)):\n",
        "          y_pred.append(res[k].argmax())\n",
        "\n",
        "      #calculating the test accuracy using the test data\n",
        "      test_accuracy=calc_test_accuracy(y_pred,y_t)\n",
        "      print(\"Test accuracy:\",test_accuracy)\n",
        "\n",
        "    "
      ],
      "metadata": {
        "id": "psktG5UNjQKf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4> Configuration 1</h4>"
      ],
      "metadata": {
        "id": "E2YVptJHpOl5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Change these values to get the required configuration's result\n",
        "\n",
        "config1 = {\n",
        "            'number_hidden_layers': 5,\n",
        "            'number_neurons': 128,\n",
        "            'learning_rate': 0.0001,\n",
        "            'weight_type': \"xavier\",\n",
        "            'activation_function':\"relu\",\n",
        "            'mini_batch_size' : 16,\n",
        "            'max_epochs': 20,\n",
        "            'lambd': 0.0005,\n",
        "            'optimization_function': \"nadam\",\n",
        "            'loss_function': \"cross_entropy\"\n",
        "              }"
      ],
      "metadata": {
        "id": "tXuA9vdb4b6P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "custom_config_mnist(config1, True, 1)"
      ],
      "metadata": {
        "id": "Na1QqZSq4eFO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 479,
          "referenced_widgets": [
            "2174ec287cf94766bdb9a31f09518eae",
            "6940ffb31b3b4b9f8f62238e7d15df72",
            "7237b35791904be7b4c395147ab0756f",
            "d0ba154ada2140099d67ae877344092a",
            "c70e5b0abd514e9b966dec3f68e13781",
            "aaba43fc77644fec9fc9e6a0900c91dc",
            "a63045e0e64643299d8dcb23deed14e8",
            "f547356504af48eb887c8e3ca132a183"
          ]
        },
        "outputId": "b8b08f8f-b957-4b1c-805e-492e234484b0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit: ··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                    Syncing run <strong><a href=\"https://wandb.ai/safikhan/mnist%20test/runs/jypy0cyq\" target=\"_blank\">ruby-rain-2</a></strong> to <a href=\"https://wandb.ai/safikhan/mnist%20test\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">docs</a>).<br/>\n",
              "\n",
              "                "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 20/20 [27:17<00:00, 81.87s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "99.92962962962963\n",
            "97.48333333333333\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Calling wandb.run.save without any arguments is deprecated.Changes to attributes are automatically persisted.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test accuracy: 97.48\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 186... <strong style=\"color:green\">(success).</strong>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2174ec287cf94766bdb9a31f09518eae",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; width: 100% }\n",
              "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
              "    </style>\n",
              "<div class=\"wandb-row\"><div class=\"wandb-col\">\n",
              "<h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>Train_Accuracy</td><td>▁▄▅▆▆▇▇▇▇▇▇█████████</td></tr><tr><td>Train_Loss</td><td>█▅▄▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁</td></tr><tr><td>Validation_Accuracy</td><td>▁▅▆▇▇▇▇▇████████████</td></tr><tr><td>Validation_loss</td><td>█▃▂▁▁▁▁▁▁▁▁▂▂▂▂▂▃▃▃▃</td></tr><tr><td>epoch</td><td>▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇██</td></tr></table><br/></div><div class=\"wandb-col\">\n",
              "<h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>Train_Accuracy</td><td>99.92963</td></tr><tr><td>Train_Loss</td><td>0.0053</td></tr><tr><td>Validation_Accuracy</td><td>97.48333</td></tr><tr><td>Validation_loss</td><td>0.12026</td></tr><tr><td>epoch</td><td>19</td></tr></table>\n",
              "</div></div>\n",
              "Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
              "<br/>Synced <strong style=\"color:#cdcd00\">ruby-rain-2</strong>: <a href=\"https://wandb.ai/safikhan/mnist%20test/runs/jypy0cyq\" target=\"_blank\">https://wandb.ai/safikhan/mnist%20test/runs/jypy0cyq</a><br/>\n",
              "Find logs at: <code>./wandb/run-20220223_174056-jypy0cyq/logs</code><br/>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4>Configuration 2</h4>"
      ],
      "metadata": {
        "id": "AL2HNo7_pwDR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "config2 = {\n",
        "            'number_hidden_layers': 5,\n",
        "            'number_neurons': 128,\n",
        "            'learning_rate': 0.0001,\n",
        "            'weight_type': \"xavier\",\n",
        "            'activation_function':\"relu\",\n",
        "            'mini_batch_size' : 64,\n",
        "            'max_epochs': 20,\n",
        "            'lambd': 0.0005,\n",
        "            'optimization_function': \"nesterov_gd\",\n",
        "            'loss_function': \"cross_entropy\"\n",
        "              }"
      ],
      "metadata": {
        "id": "iVlAA6Swpvg-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "custom_config_mnist(config2, True, 2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427,
          "referenced_widgets": [
            "0aa640b5c9e843b6b86370443e7de989",
            "510566ce597a42539c099fc4e2d80644",
            "7702c6812b6b4d9285fd00d4104430be",
            "d96c0570414545c3b812a0d90301edbc",
            "84c5bac8fe254dcb95d5526b50c496a1",
            "2c84c4f31d184131ae17610f9d75e2e9",
            "6c2458ee2e334fefb79d0d7ede73c0f5",
            "c6a586dcfa984e658e48dd7811d9a4ad"
          ]
        },
        "id": "kbYKXMv2qAHe",
        "outputId": "84391086-4fac-4c4f-f18d-787e75bec097"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33msafikhan\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                    Syncing run <strong><a href=\"https://wandb.ai/safikhan/mnist%20test/runs/41xuqt41\" target=\"_blank\">daily-rain-4</a></strong> to <a href=\"https://wandb.ai/safikhan/mnist%20test\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">docs</a>).<br/>\n",
              "\n",
              "                "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 20/20 [23:46<00:00, 71.34s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "99.38703703703705\n",
            "97.08333333333333\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Calling wandb.run.save without any arguments is deprecated.Changes to attributes are automatically persisted.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test accuracy: 97.00999999999999\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 477... <strong style=\"color:green\">(success).</strong>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0aa640b5c9e843b6b86370443e7de989",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; width: 100% }\n",
              "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
              "    </style>\n",
              "<div class=\"wandb-row\"><div class=\"wandb-col\">\n",
              "<h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>Train_Accuracy</td><td>▁▄▅▅▆▆▅▇▇▇▇▇▇███▇███</td></tr><tr><td>Train_Loss</td><td>█▅▄▄▃▃▄▂▂▂▂▁▁▁▁▁▂▁▁▁</td></tr><tr><td>Validation_Accuracy</td><td>▁▄▆▅▆▆▅▇▇▇▇▇████▇██▇</td></tr><tr><td>Validation_loss</td><td>█▃▁▂▂▁▄▁▁▁▃▃▃▂▃▃▆▄▅▅</td></tr><tr><td>epoch</td><td>▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇██</td></tr></table><br/></div><div class=\"wandb-col\">\n",
              "<h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>Train_Accuracy</td><td>99.38704</td></tr><tr><td>Train_Loss</td><td>0.01921</td></tr><tr><td>Validation_Accuracy</td><td>97.08333</td></tr><tr><td>Validation_loss</td><td>0.15636</td></tr><tr><td>epoch</td><td>19</td></tr></table>\n",
              "</div></div>\n",
              "Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
              "<br/>Synced <strong style=\"color:#cdcd00\">daily-rain-4</strong>: <a href=\"https://wandb.ai/safikhan/mnist%20test/runs/41xuqt41\" target=\"_blank\">https://wandb.ai/safikhan/mnist%20test/runs/41xuqt41</a><br/>\n",
              "Find logs at: <code>./wandb/run-20220223_181153-41xuqt41/logs</code><br/>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h4>Configuration 3</h4>"
      ],
      "metadata": {
        "id": "BcoZZ5fPqByM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "config3 = {\n",
        "            'number_hidden_layers': 4,\n",
        "            'number_neurons': 64,\n",
        "            'learning_rate': 0.0001,\n",
        "            'weight_type': \"xavier\",\n",
        "            'activation_function':\"relu\",\n",
        "            'mini_batch_size' : 32,\n",
        "            'max_epochs': 20,\n",
        "            'lambd': 0,\n",
        "            'optimization_function': \"momentum_gd\",\n",
        "            'loss_function': \"cross_entropy\"\n",
        "              }"
      ],
      "metadata": {
        "id": "EnRAEcKGqGR-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "custom_config_mnist(config3, True, 3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 392,
          "referenced_widgets": [
            "0804e5724c844c199b71f01b926033f4",
            "112fa9071e7f4643839af5675dde4b29",
            "7afce843532642d0bdc48b72d5286e15",
            "5e22b33609344bd6ad42cbf6895d5f63",
            "e37519bd05ea426391da7710a6f4e8fd",
            "dbe32aecff434ec8905e72c57888aa75",
            "e4767a5976e04fde957e72e41419b62b",
            "86a1b56c04c14c72b642fdbc8b0e54c9"
          ]
        },
        "id": "17TYj2FLqFFY",
        "outputId": "c69c1580-0185-4e40-cb9d-c6ab21a0805c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                    Syncing run <strong><a href=\"https://wandb.ai/safikhan/mnist%20test/runs/223l71mq\" target=\"_blank\">cool-universe-5</a></strong> to <a href=\"https://wandb.ai/safikhan/mnist%20test\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">docs</a>).<br/>\n",
              "\n",
              "                "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 20/20 [12:09<00:00, 36.47s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "99.01111111111112\n",
            "96.73333333333333\n",
            "Test accuracy: 96.67\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 619... <strong style=\"color:green\">(success).</strong>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0804e5724c844c199b71f01b926033f4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(Label(value=' 0.00MB of 0.00MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; width: 100% }\n",
              "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
              "    </style>\n",
              "<div class=\"wandb-row\"><div class=\"wandb-col\">\n",
              "<h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>Train_Accuracy</td><td>▁▃▄▅▆▆▇▇▇▇▇▇▇▇████▇█</td></tr><tr><td>Train_Loss</td><td>█▆▅▄▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁</td></tr><tr><td>Validation_Accuracy</td><td>▁▄▅▆▆▇▇▇█████▇████▇▇</td></tr><tr><td>Validation_loss</td><td>█▅▃▂▁▁▁▁▁▂▂▂▂▃▃▃▃▄▆▅</td></tr><tr><td>epoch</td><td>▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇██</td></tr></table><br/></div><div class=\"wandb-col\">\n",
              "<h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>Train_Accuracy</td><td>99.01111</td></tr><tr><td>Train_Loss</td><td>0.02866</td></tr><tr><td>Validation_Accuracy</td><td>96.73333</td></tr><tr><td>Validation_loss</td><td>0.15765</td></tr><tr><td>epoch</td><td>19</td></tr></table>\n",
              "</div></div>\n",
              "Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
              "<br/>Synced <strong style=\"color:#cdcd00\">cool-universe-5</strong>: <a href=\"https://wandb.ai/safikhan/mnist%20test/runs/223l71mq\" target=\"_blank\">https://wandb.ai/safikhan/mnist%20test/runs/223l71mq</a><br/>\n",
              "Find logs at: <code>./wandb/run-20220223_183552-223l71mq/logs</code><br/>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}