examples/Pipelines-with-Gordo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Build pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.preprocessing import MinMaxScaler, QuantileTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.ensemble import IsolationForest\n",
    "\n",
    "# Some pipeline we decide is 'the best' for our problem.\n",
    "pipe = Pipeline([\n",
    "    ('pca', PCA(n_components=10)),\n",
    "    ('feature_union', FeatureUnion([\n",
    "        ('feature_union_pipe1', Pipeline([\n",
    "            ('min_max', MinMaxScaler()),\n",
    "            ('pca', PCA(n_components=2))\n",
    "        ])),\n",
    "        ('feature_union_pipe2', Pipeline([\n",
    "            ('qt', QuantileTransformer(output_distribution='uniform')),\n",
    "        ]))\n",
    "    ])),\n",
    "    ('iso', IsolationForest(contamination='auto'))\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write this model architecture to a `Gordo Model Definition`\n",
    "\n",
    "### This a valid mapping to the Gordo config `model` key."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/skoro/Work/gordo/venv_gordo/lib/python3.7/site-packages/pandas/compat/__init__.py:120: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "sklearn.pipeline.Pipeline:\n",
      "  memory: null\n",
      "  steps:\n",
      "  - sklearn.decomposition._pca.PCA:\n",
      "      copy: true\n",
      "      iterated_power: auto\n",
      "      n_components: 10\n",
      "      random_state: null\n",
      "      svd_solver: auto\n",
      "      tol: 0.0\n",
      "      whiten: false\n",
      "  - sklearn.pipeline.FeatureUnion:\n",
      "      n_jobs: null\n",
      "      transformer_list:\n",
      "      - sklearn.pipeline.Pipeline:\n",
      "          memory: null\n",
      "          steps:\n",
      "          - sklearn.preprocessing._data.MinMaxScaler:\n",
      "              clip: false\n",
      "              copy: true\n",
      "              feature_range: !!python/tuple\n",
      "              - 0\n",
      "              - 1\n",
      "          - sklearn.decomposition._pca.PCA:\n",
      "              copy: true\n",
      "              iterated_power: auto\n",
      "              n_components: 2\n",
      "              random_state: null\n",
      "              svd_solver: auto\n",
      "              tol: 0.0\n",
      "              whiten: false\n",
      "          verbose: false\n",
      "      - sklearn.pipeline.Pipeline:\n",
      "          memory: null\n",
      "          steps:\n",
      "          - sklearn.preprocessing._data.QuantileTransformer:\n",
      "              copy: true\n",
      "              ignore_implicit_zeros: false\n",
      "              n_quantiles: 1000\n",
      "              output_distribution: uniform\n",
      "              random_state: null\n",
      "              subsample: 100000\n",
      "          verbose: false\n",
      "      transformer_weights: null\n",
      "      verbose: false\n",
      "  - sklearn.ensemble._iforest.IsolationForest:\n",
      "      bootstrap: false\n",
      "      contamination: auto\n",
      "      max_features: 1.0\n",
      "      max_samples: auto\n",
      "      n_estimators: 100\n",
      "      n_jobs: null\n",
      "      random_state: null\n",
      "      verbose: 0\n",
      "      warm_start: false\n",
      "  verbose: false\n"
     ]
    }
   ],
   "source": [
    "from gordo.serializer import into_definition\n",
    "\n",
    "yaml_definition_of_pipeline = yaml.dump(into_definition(pipe))\n",
    "print('-' * 30)\n",
    "print(yaml_definition_of_pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load definition back into a pipeline\n",
    "\n",
    "### You probably won't need to do this, but it's how we get a replica of your defined model in Gordo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('step_0', PCA(n_components=10)),\n",
      " ('step_1',\n",
      "  FeatureUnion(transformer_list=[('step_0',\n",
      "                                Pipeline(steps=[('step_0', MinMaxScaler()),\n",
      "                                                ('step_1',\n",
      "                                                 PCA(n_components=2))])),\n",
      "                               ('step_1',\n",
      "                                Pipeline(steps=[('step_0',\n",
      "                                                 QuantileTransformer())]))])),\n",
      " ('step_2', IsolationForest())]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/skoro/Work/gordo/venv_gordo/lib/python3.7/site-packages/ipykernel_launcher.py:4: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "from gordo.serializer import from_definition\n",
    "\n",
    "\n",
    "pipe = from_definition(yaml.load(yaml_definition_of_pipeline, Loader=yaml.FullLoader))\n",
    "pprint(pipe.steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let us train the pipelilne."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 20)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "X = np.random.random(int(1e5)).reshape(-1, 20)\n",
    "y = X.copy()\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('step_0', PCA(n_components=10)),\n",
       "                ('step_1',\n",
       "                 FeatureUnion(transformer_list=[('step_0',\n",
       "                                                 Pipeline(steps=[('step_0',\n",
       "                                                                  MinMaxScaler()),\n",
       "                                                                 ('step_1',\n",
       "                                                                  PCA(n_components=2))])),\n",
       "                                                ('step_1',\n",
       "                                                 Pipeline(steps=[('step_0',\n",
       "                                                                  QuantileTransformer())]))])),\n",
       "                ('step_2', IsolationForest())])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict as normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_anomolies = pipe.predict(X)\n",
    "predicted_anomolies.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now we want to serialize it, for some reason."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\x80\\x03csklearn.pipeline\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gordo.serializer import dumps, loads\n",
    "\n",
    "serialized_pipe_bytes = dumps(pipe)\n",
    "serialized_pipe_bytes[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load it back, ensuring the state is kept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe_clone = loads(serialized_pipe_bytes)\n",
    "predictions = pipe_clone.predict(X)\n",
    "assert np.allclose(predicted_anomolies, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optionally, you can save it to a directory "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tempfile import TemporaryDirectory\n",
    "from gordo.serializer import dump, load\n",
    "\n",
    "with TemporaryDirectory() as tmp:\n",
    "    \n",
    "    # Dump pipe to directory\n",
    "    dump(pipe, tmp)\n",
    "    \n",
    "    # Load it back\n",
    "    pipe_clone = load(tmp)\n",
    "    \n",
    "    assert np.allclose(pipe_clone.predict(X), predicted_anomolies)\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}