My_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6f4e3fcc-7b51-4723-a926-b70cca72b3c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "USER_FLAG = \"--user\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d9a6e78a-7fa4-4f81-b0f6-ee6231f2dd90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting google-cloud-aiplatform==1.7.0\n",
      "  Downloading google_cloud_aiplatform-1.7.0-py2.py3-none-any.whl (1.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: packaging>=14.3 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform==1.7.0) (21.3)\n",
      "Collecting google-cloud-storage<2.0.0dev,>=1.32.0\n",
      "  Downloading google_cloud_storage-1.44.0-py2.py3-none-any.whl (106 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.8/106.8 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: google-api-core[grpc]<3.0.0dev,>=1.26.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform==1.7.0) (2.8.1)\n",
      "Requirement already satisfied: google-cloud-bigquery<3.0.0dev,>=1.15.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform==1.7.0) (2.34.4)\n",
      "Requirement already satisfied: proto-plus>=1.10.1 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform==1.7.0) (1.20.6)\n",
      "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (2.28.1)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (1.56.3)\n",
      "Requirement already satisfied: google-auth<3.0dev,>=1.25.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (2.9.0)\n",
      "Requirement already satisfied: protobuf<4.0.0dev,>=3.15.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (3.19.4)\n",
      "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (1.47.0)\n",
      "Requirement already satisfied: grpcio-status<2.0dev,>=1.33.2 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (1.47.0)\n",
      "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform==1.7.0) (2.8.2)\n",
      "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.4.1 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform==1.7.0) (2.3.1)\n",
      "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform==1.7.0) (2.3.3)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from google-cloud-storage<2.0.0dev,>=1.32.0->google-cloud-aiplatform==1.7.0) (1.16.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=14.3->google-cloud-aiplatform==1.7.0) (3.0.9)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (0.2.7)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (4.8)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (5.0.0)\n",
      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /opt/conda/lib/python3.7/site-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform==1.7.0) (1.1.2)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (1.26.9)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (2022.6.15)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (2.1.0)\n",
      "Requirement already satisfied: cffi>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from google-crc32c<2.0dev,>=1.0->google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform==1.7.0) (1.15.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-api-core[grpc]<3.0.0dev,>=1.26.0->google-cloud-aiplatform==1.7.0) (0.4.8)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.7/site-packages (from cffi>=1.0.0->google-crc32c<2.0dev,>=1.0->google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform==1.7.0) (2.21)\n",
      "Installing collected packages: google-cloud-storage, google-cloud-aiplatform\n",
      "\u001b[33m  WARNING: The script tb-gcp-uploader is installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0mSuccessfully installed google-cloud-aiplatform-1.7.0 google-cloud-storage-1.44.0\n",
      "Collecting kfp==1.8.9\n",
      "  Downloading kfp-1.8.9.tar.gz (296 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m296.3/296.3 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting google-cloud-pipeline-components==0.2.0\n",
      "  Downloading google_cloud_pipeline_components-0.2.0-py3-none-any.whl (135 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.0/135.0 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting absl-py<=0.11,>=0.9\n",
      "  Downloading absl_py-0.11.0-py3-none-any.whl (127 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m127.8/127.8 kB\u001b[0m \u001b[31m21.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting PyYAML<6,>=5.3\n",
      "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m636.6/636.6 kB\u001b[0m \u001b[31m32.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: google-cloud-storage<2,>=1.20.0 in ./.local/lib/python3.7/site-packages (from kfp==1.8.9) (1.44.0)\n",
      "Collecting kubernetes<19,>=8.0.0\n",
      "  Downloading kubernetes-18.20.0-py2.py3-none-any.whl (1.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m49.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting google-api-python-client<2,>=1.7.8\n",
      "  Downloading google_api_python_client-1.12.11-py2.py3-none-any.whl (62 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.1/62.1 kB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting google-auth<2,>=1.6.1\n",
      "  Downloading google_auth-1.35.0-py2.py3-none-any.whl (152 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.9/152.9 kB\u001b[0m \u001b[31m25.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting requests-toolbelt<1,>=0.8.0\n",
      "  Downloading requests_toolbelt-0.9.1-py2.py3-none-any.whl (54 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.3/54.3 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: cloudpickle<3,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from kfp==1.8.9) (2.1.0)\n",
      "Collecting kfp-server-api<2.0.0,>=1.1.2\n",
      "  Downloading kfp-server-api-1.8.2.tar.gz (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.0/58.0 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting jsonschema<4,>=3.0.1\n",
      "  Downloading jsonschema-3.2.0-py2.py3-none-any.whl (56 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.3/56.3 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tabulate<1,>=0.8.6 in /opt/conda/lib/python3.7/site-packages (from kfp==1.8.9) (0.8.10)\n",
      "Requirement already satisfied: click<9,>=7.1.2 in /opt/conda/lib/python3.7/site-packages (from kfp==1.8.9) (8.1.3)\n",
      "Collecting Deprecated<2,>=1.2.7\n",
      "  Downloading Deprecated-1.2.13-py2.py3-none-any.whl (9.6 kB)\n",
      "Collecting strip-hints<1,>=0.1.8\n",
      "  Downloading strip-hints-0.1.10.tar.gz (29 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting docstring-parser<1,>=0.7.3\n",
      "  Downloading docstring_parser-0.14.1-py3-none-any.whl (33 kB)\n",
      "Collecting kfp-pipeline-spec<0.2.0,>=0.1.13\n",
      "  Downloading kfp_pipeline_spec-0.1.16-py3-none-any.whl (19 kB)\n",
      "Collecting fire<1,>=0.3.1\n",
      "  Downloading fire-0.4.0.tar.gz (87 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.7/87.7 kB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: protobuf<4,>=3.13.0 in /opt/conda/lib/python3.7/site-packages (from kfp==1.8.9) (3.19.4)\n",
      "Collecting uritemplate<4,>=3.0.1\n",
      "  Downloading uritemplate-3.0.1-py2.py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: pydantic<2,>=1.8.2 in /opt/conda/lib/python3.7/site-packages (from kfp==1.8.9) (1.9.1)\n",
      "Collecting typer<1.0,>=0.3.2\n",
      "  Downloading typer-0.5.0-py3-none-any.whl (28 kB)\n",
      "Collecting typing-extensions<4,>=3.7.4\n",
      "  Downloading typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)\n",
      "Requirement already satisfied: google-cloud-aiplatform>=1.4.3 in ./.local/lib/python3.7/site-packages (from google-cloud-pipeline-components==0.2.0) (1.7.0)\n",
      "Collecting google-api-core<2dev,>=1.26.0\n",
      "  Downloading google_api_core-1.31.6-py2.py3-none-any.whl (93 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m93.3/93.3 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting google-cloud-notebooks>=0.4.0\n",
      "  Downloading google_cloud_notebooks-1.4.0-py2.py3-none-any.whl (355 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m355.5/355.5 kB\u001b[0m \u001b[31m36.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from absl-py<=0.11,>=0.9->kfp==1.8.9) (1.16.0)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from click<9,>=7.1.2->kfp==1.8.9) (4.11.4)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /opt/conda/lib/python3.7/site-packages (from Deprecated<2,>=1.2.7->kfp==1.8.9) (1.14.1)\n",
      "Requirement already satisfied: termcolor in /opt/conda/lib/python3.7/site-packages (from fire<1,>=0.3.1->kfp==1.8.9) (1.1.0)\n",
      "Requirement already satisfied: packaging>=14.3 in /opt/conda/lib/python3.7/site-packages (from google-api-core<2dev,>=1.26.0->google-cloud-pipeline-components==0.2.0) (21.3)\n",
      "Requirement already satisfied: pytz in /opt/conda/lib/python3.7/site-packages (from google-api-core<2dev,>=1.26.0->google-cloud-pipeline-components==0.2.0) (2022.1)\n",
      "Requirement already satisfied: setuptools>=40.3.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core<2dev,>=1.26.0->google-cloud-pipeline-components==0.2.0) (59.8.0)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core<2dev,>=1.26.0->google-cloud-pipeline-components==0.2.0) (1.56.3)\n",
      "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core<2dev,>=1.26.0->google-cloud-pipeline-components==0.2.0) (2.28.1)\n",
      "Requirement already satisfied: httplib2<1dev,>=0.15.0 in /opt/conda/lib/python3.7/site-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.9) (0.20.4)\n",
      "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /opt/conda/lib/python3.7/site-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.9) (0.1.0)\n",
      "Collecting cachetools<5.0,>=2.0.0\n",
      "  Downloading cachetools-4.2.4-py3-none-any.whl (10 kB)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.9) (4.8)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.9) (0.2.7)\n",
      "Requirement already satisfied: google-api-core[grpc]<3.0.0dev,>=1.26.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform>=1.4.3->google-cloud-pipeline-components==0.2.0) (2.8.1)\n",
      "Requirement already satisfied: proto-plus>=1.10.1 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform>=1.4.3->google-cloud-pipeline-components==0.2.0) (1.20.6)\n",
      "Requirement already satisfied: google-cloud-bigquery<3.0.0dev,>=1.15.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform>=1.4.3->google-cloud-pipeline-components==0.2.0) (2.34.4)\n",
      "Requirement already satisfied: google-resumable-media<3.0dev,>=1.3.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.9) (2.3.3)\n",
      "Requirement already satisfied: google-cloud-core<3.0dev,>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.9) (2.3.1)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema<4,>=3.0.1->kfp==1.8.9) (21.4.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema<4,>=3.0.1->kfp==1.8.9) (0.18.1)\n",
      "Requirement already satisfied: urllib3>=1.15 in /opt/conda/lib/python3.7/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9) (1.26.9)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.7/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9) (2022.6.15)\n",
      "Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.7/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.9) (2.8.2)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.7/site-packages (from kubernetes<19,>=8.0.0->kfp==1.8.9) (1.3.3)\n",
      "Requirement already satisfied: requests-oauthlib in /opt/conda/lib/python3.7/site-packages (from kubernetes<19,>=8.0.0->kfp==1.8.9) (1.3.1)\n",
      "Requirement already satisfied: wheel in /opt/conda/lib/python3.7/site-packages (from strip-hints<1,>=0.1.8->kfp==1.8.9) (0.37.1)\n",
      "Collecting google-api-core[grpc]<3.0.0dev,>=1.26.0\n",
      "  Downloading google_api_core-2.8.2-py3-none-any.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.6/114.6 kB\u001b[0m \u001b[31m21.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.8.0-py3-none-any.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.7/114.7 kB\u001b[0m \u001b[31m17.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of certifi to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting certifi\n",
      "  Downloading certifi-2022.6.15-py3-none-any.whl (160 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m160.2/160.2 kB\u001b[0m \u001b[31m24.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of cachetools to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting cachetools<5.0,>=2.0.0\n",
      "  Downloading cachetools-4.2.2-py3-none-any.whl (11 kB)\n",
      "INFO: pip is looking at multiple versions of attrs to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting attrs>=17.4.0\n",
      "  Downloading attrs-21.4.0-py2.py3-none-any.whl (60 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of uritemplate to determine which version is compatible with other requirements. This could take a while.\n",
      "INFO: pip is looking at multiple versions of typing-extensions to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting typing-extensions<4,>=3.7.4\n",
      "  Downloading typing_extensions-3.10.0.1-py3-none-any.whl (26 kB)\n",
      "INFO: pip is looking at multiple versions of typer to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting typer<1.0,>=0.3.2\n",
      "  Downloading typer-0.4.2-py3-none-any.whl (27 kB)\n",
      "INFO: pip is looking at multiple versions of tabulate to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting tabulate<1,>=0.8.6\n",
      "  Downloading tabulate-0.8.10-py3-none-any.whl (29 kB)\n",
      "INFO: pip is looking at multiple versions of strip-hints to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting strip-hints<1,>=0.1.8\n",
      "  Downloading strip-hints-0.1.9.tar.gz (30 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of requests-toolbelt to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting requests-toolbelt<1,>=0.8.0\n",
      "  Downloading requests_toolbelt-0.9.0-py2.py3-none-any.whl (54 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.3/54.3 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of pyyaml to determine which version is compatible with other requirements. This could take a while.\n",
      "INFO: pip is looking at multiple versions of pydantic to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting pydantic<2,>=1.8.2\n",
      "  Downloading pydantic-1.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.1/11.1 MB\u001b[0m \u001b[31m88.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of protobuf to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting protobuf<4,>=3.13.0\n",
      "  Downloading protobuf-3.20.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m63.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of kubernetes to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting kubernetes<19,>=8.0.0\n",
      "  Downloading kubernetes-17.17.0-py3-none-any.whl (1.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m85.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of kfp-server-api to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting kfp-server-api<2.0.0,>=1.1.2\n",
      "  Downloading kfp-server-api-1.8.1.tar.gz (54 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.2/54.2 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of kfp-pipeline-spec to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting kfp-pipeline-spec<0.2.0,>=0.1.13\n",
      "  Downloading kfp_pipeline_spec-0.1.15-py3-none-any.whl (11 kB)\n",
      "INFO: pip is looking at multiple versions of jsonschema to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting jsonschema<4,>=3.0.1\n",
      "  Downloading jsonschema-3.1.1-py2.py3-none-any.whl (56 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.1/56.1 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of google-cloud-storage to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting google-cloud-storage<2,>=1.20.0\n",
      "  Using cached google_cloud_storage-1.44.0-py2.py3-none-any.whl (106 kB)\n",
      "  Downloading google_cloud_storage-1.43.0-py2.py3-none-any.whl (106 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.6/106.6 kB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_cloud_storage-1.42.3-py2.py3-none-any.whl (105 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.0/106.0 kB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_cloud_storage-1.42.2-py2.py3-none-any.whl (105 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.9/105.9 kB\u001b[0m \u001b[31m19.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_cloud_storage-1.42.1-py2.py3-none-any.whl (105 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.2/105.2 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_cloud_storage-1.42.0-py2.py3-none-any.whl (105 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.2/105.2 kB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_cloud_storage-1.41.1-py2.py3-none-any.whl (105 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.0/105.0 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of google-cloud-storage to determine which version is compatible with other requirements. This could take a while.\n",
      "  Downloading google_cloud_storage-1.41.0-py2.py3-none-any.whl (104 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.0/105.0 kB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of google-cloud-notebooks to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting google-cloud-notebooks>=0.4.0\n",
      "  Downloading google_cloud_notebooks-1.3.2-py2.py3-none-any.whl (354 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m354.1/354.1 kB\u001b[0m \u001b[31m41.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting google-api-core[grpc]<3.0.0dev,>=1.26.0\n",
      "  Downloading google_api_core-2.7.3-py3-none-any.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.6/114.6 kB\u001b[0m \u001b[31m20.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.7.2-py3-none-any.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.6/114.6 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.7.1-py3-none-any.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.7/114.7 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.7.0-py3-none-any.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.7/114.7 kB\u001b[0m \u001b[31m21.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.6.1-py3-none-any.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.6/114.6 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.6.0-py2.py3-none-any.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.6/114.6 kB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.5.0-py2.py3-none-any.whl (111 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.8/111.8 kB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.4.0-py2.py3-none-any.whl (111 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.7/111.7 kB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.3.2-py2.py3-none-any.whl (109 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.8/109.8 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.3.0-py2.py3-none-any.whl (109 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.8/109.8 kB\u001b[0m \u001b[31m19.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.2.2-py2.py3-none-any.whl (95 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.8/95.8 kB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.2.1-py2.py3-none-any.whl (95 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.6/95.6 kB\u001b[0m \u001b[31m18.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.2.0-py2.py3-none-any.whl (95 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.6/95.6 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.1.1-py2.py3-none-any.whl (95 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.1/95.1 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.1.0-py2.py3-none-any.whl (94 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.8/94.8 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.0.1-py2.py3-none-any.whl (92 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.5/92.5 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading google_api_core-2.0.0-py2.py3-none-any.whl (92 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.4/92.4 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: grpcio<2.0dev,>=1.29.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core<2dev,>=1.26.0->google-cloud-pipeline-components==0.2.0) (1.47.0)\n",
      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /opt/conda/lib/python3.7/site-packages (from google-resumable-media<3.0dev,>=1.3.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.9) (1.1.2)\n",
      "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /opt/conda/lib/python3.7/site-packages (from httplib2<1dev,>=0.15.0->google-api-python-client<2,>=1.7.8->kfp==1.8.9) (3.0.9)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.1->kfp==1.8.9) (0.4.8)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.26.0->google-cloud-pipeline-components==0.2.0) (2.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.26.0->google-cloud-pipeline-components==0.2.0) (3.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->click<9,>=7.1.2->kfp==1.8.9) (3.8.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib->kubernetes<19,>=8.0.0->kfp==1.8.9) (3.2.0)\n",
      "Requirement already satisfied: cffi>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from google-crc32c<2.0dev,>=1.0->google-resumable-media<3.0dev,>=1.3.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.9) (1.15.1)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.7/site-packages (from cffi>=1.0.0->google-crc32c<2.0dev,>=1.0->google-resumable-media<3.0dev,>=1.3.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.9) (2.21)\n",
      "Building wheels for collected packages: kfp, fire, kfp-server-api, strip-hints\n",
      "  Building wheel for kfp (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for kfp: filename=kfp-1.8.9-py3-none-any.whl size=409653 sha256=8dc6df67618185295e3f6a32342ff10a74344d760957d96939c27ec16dcba295\n",
      "  Stored in directory: /home/jupyter/.cache/pip/wheels/0e/20/7e/c2c43249eb0538c5aa2542bcc9b02affb0211ed5617fbd4abc\n",
      "  Building wheel for fire (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for fire: filename=fire-0.4.0-py2.py3-none-any.whl size=115942 sha256=70d79e23d43a045297c1ce8b10f231a3a46a4edc0359cb2010a17ba31f21ddc9\n",
      "  Stored in directory: /home/jupyter/.cache/pip/wheels/8a/67/fb/2e8a12fa16661b9d5af1f654bd199366799740a85c64981226\n",
      "  Building wheel for kfp-server-api (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for kfp-server-api: filename=kfp_server_api-1.8.2-py3-none-any.whl size=99716 sha256=48d82454ca57c8c041293b7788fbb3f409cf28762209c839f624577937a31c3a\n",
      "  Stored in directory: /home/jupyter/.cache/pip/wheels/77/36/d3/60e33cc9e15f269fe0e0f71cae6d077a5e43973d514b60b4ad\n",
      "  Building wheel for strip-hints (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for strip-hints: filename=strip_hints-0.1.10-py2.py3-none-any.whl size=22302 sha256=7a2bd2951664cd9c37e3849f1dac30545fbafab240eb9036a91ddb8258ecac8f\n",
      "  Stored in directory: /home/jupyter/.cache/pip/wheels/5e/14/c3/6e44e9b2545f2d570b03f5b6d38c00b7534aa8abb376978363\n",
      "Successfully built kfp fire kfp-server-api strip-hints\n",
      "Installing collected packages: typing-extensions, uritemplate, strip-hints, PyYAML, kfp-pipeline-spec, fire, docstring-parser, Deprecated, cachetools, absl-py, requests-toolbelt, kfp-server-api, jsonschema, google-auth, typer, kubernetes, google-api-core, google-api-python-client, google-cloud-notebooks, kfp, google-cloud-pipeline-components\n",
      "\u001b[33m  WARNING: The script strip-hints is installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  WARNING: The script jsonschema is installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  WARNING: The scripts dsl-compile, dsl-compile-v2 and kfp are installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "tfx 0.26.3 requires absl-py<0.11,>=0.9, but you have absl-py 0.11.0 which is incompatible.\n",
      "tfx 0.26.3 requires attrs<21,>=19.3.0, but you have attrs 21.4.0 which is incompatible.\n",
      "tfx 0.26.3 requires click<8,>=7, but you have click 8.1.3 which is incompatible.\n",
      "tfx 0.26.3 requires docker<5,>=4.1, but you have docker 5.0.3 which is incompatible.\n",
      "tfx 0.26.3 requires kubernetes<12,>=10.0.1, but you have kubernetes 18.20.0 which is incompatible.\n",
      "tfx 0.26.3 requires pyarrow<0.18,>=0.17, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tfx-bsl 0.26.1 requires absl-py<0.11,>=0.9, but you have absl-py 0.11.0 which is incompatible.\n",
      "tfx-bsl 0.26.1 requires pyarrow<0.18,>=0.17, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-transform 0.26.0 requires absl-py<0.11,>=0.9, but you have absl-py 0.11.0 which is incompatible.\n",
      "tensorflow-transform 0.26.0 requires pyarrow<0.18,>=0.17, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-model-analysis 0.26.1 requires absl-py<0.11,>=0.9, but you have absl-py 0.11.0 which is incompatible.\n",
      "tensorflow-model-analysis 0.26.1 requires pyarrow<0.18,>=0.17, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-metadata 0.26.0 requires absl-py<0.11,>=0.9, but you have absl-py 0.11.0 which is incompatible.\n",
      "tensorflow-data-validation 0.26.1 requires absl-py<0.11,>=0.9, but you have absl-py 0.11.0 which is incompatible.\n",
      "tensorflow-data-validation 0.26.1 requires joblib<0.15,>=0.12, but you have joblib 1.0.1 which is incompatible.\n",
      "tensorflow-data-validation 0.26.1 requires pyarrow<0.18,>=0.17, but you have pyarrow 8.0.0 which is incompatible.\n",
      "ml-pipelines-sdk 0.26.3 requires absl-py<0.11,>=0.9, but you have absl-py 0.11.0 which is incompatible.\n",
      "ml-pipelines-sdk 0.26.3 requires docker<5,>=4.1, but you have docker 5.0.3 which is incompatible.\n",
      "ml-metadata 0.26.0 requires absl-py<0.11,>=0.9, but you have absl-py 0.11.0 which is incompatible.\n",
      "ml-metadata 0.26.0 requires attrs<21,>=20.3, but you have attrs 21.4.0 which is incompatible.\n",
      "cloud-tpu-client 0.10 requires google-api-python-client==1.8.0, but you have google-api-python-client 1.12.11 which is incompatible.\n",
      "apache-beam 2.28.0 requires httplib2<0.18.0,>=0.8, but you have httplib2 0.20.4 which is incompatible.\n",
      "apache-beam 2.28.0 requires pyarrow<3.0.0,>=0.15.1, but you have pyarrow 8.0.0 which is incompatible.\n",
      "apache-beam 2.28.0 requires typing-extensions<3.8.0,>=3.7.0, but you have typing-extensions 3.10.0.2 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed Deprecated-1.2.13 PyYAML-5.4.1 absl-py-0.11.0 cachetools-4.2.4 docstring-parser-0.14.1 fire-0.4.0 google-api-core-1.31.6 google-api-python-client-1.12.11 google-auth-1.35.0 google-cloud-notebooks-1.3.2 google-cloud-pipeline-components-0.2.0 jsonschema-3.2.0 kfp-1.8.9 kfp-pipeline-spec-0.1.16 kfp-server-api-1.8.2 kubernetes-18.20.0 requests-toolbelt-0.9.1 strip-hints-0.1.10 typer-0.5.0 typing-extensions-3.10.0.2 uritemplate-3.0.1\n"
     ]
    }
   ],
   "source": [
    "!pip3 install {USER_FLAG} google-cloud-aiplatform==1.7.0 --upgrade\n",
    "!pip3 install {USER_FLAG} kfp==1.8.9 google-cloud-pipeline-components==0.2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c65f9fdf-8135-4d84-bd46-c3d6cfad17c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "if not os.getenv(\"IS_TESTING\"):\n",
    "    # Automatically restart kernel after installs\n",
    "    import IPython\n",
    "\n",
    "    app = IPython.Application.instance()\n",
    "    app.kernel.do_shutdown(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "95cda11d-4f1e-4e5e-845b-987afe6ab5dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KFP SDK version: 1.8.9\n",
      "google_cloud_pipeline_components version: 0.2.0\n"
     ]
    }
   ],
   "source": [
    "!python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\"\n",
    "!python3 -c \"import google_cloud_pipeline_components; print('google_cloud_pipeline_components version: {}'.format(google_cloud_pipeline_components.__version__))\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ff1ea02a-8602-4684-8cd7-bc0a4b3074ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Project ID:  gruppo-ethos-355909\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "PROJECT_ID = \"\"\n",
    "\n",
    "# Get your Google Cloud project ID from gcloud\n",
    "if not os.getenv(\"IS_TESTING\"):\n",
    "    shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
    "    PROJECT_ID = shell_output[0]\n",
    "    print(\"Project ID: \", PROJECT_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7ef25905-2d3b-415b-861b-c808100f4cd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "BUCKET_NAME=\"gs://\" + PROJECT_ID + \"-bucket-test_vertex_pipeline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "00c32c58-e17f-45c3-b1d7-ad8ef120b326",
   "metadata": {},
   "outputs": [],
   "source": [
    "import kfp\n",
    "\n",
    "from kfp.v2 import compiler, dsl\n",
    "from kfp.v2.dsl import component, pipeline, Artifact, ClassificationMetrics, Input, Output, Model, Metrics\n",
    "\n",
    "from google.cloud import aiplatform\n",
    "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
    "from typing import NamedTuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e66032b0-e363-48a4-a5d8-16fc7b96f99f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: PATH=/usr/local/cuda/bin:/opt/conda/bin:/opt/conda/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/home/jupyter/.local/bin:/home/jupyter/.local/bin\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'gs://gruppo-ethos-355909-bucket-test_vertex_pipeline/pipeline_root/'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PATH=%env PATH\n",
    "%env PATH={PATH}:/home/jupyter/.local/bin\n",
    "REGION=\"europe-west6-a\"\n",
    "\n",
    "PIPELINE_ROOT = f\"{BUCKET_NAME}/pipeline_root/\"\n",
    "PIPELINE_ROOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "08661056-7cc7-44dd-b2e4-7a7b287b837c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9e018641-b59f-43fb-9fc6-c80bdb3c4687",
   "metadata": {},
   "outputs": [],
   "source": [
    "@component(\n",
    "    base_image=\"gcr.io/deeplearning-platform-release/tf2-cpu.2-3:latest\",\n",
    "    output_component_file=\"tabular_eval_component.yaml\",\n",
    "    packages_to_install=[\"google-cloud-aiplatform\"],\n",
    ")\n",
    "def classification_model_eval_metrics(\n",
    "    project: str,\n",
    "    location: str,  # \"us-central1\",\n",
    "    api_endpoint: str,  # \"us-central1-aiplatform.googleapis.com\",\n",
    "    thresholds_dict_str: str,\n",
    "    model: Input[Artifact],\n",
    "    metrics: Output[Metrics],\n",
    "    metricsc: Output[ClassificationMetrics],\n",
    ") -> NamedTuple(\"Outputs\", [(\"dep_decision\", str)]):  # Return parameter.\n",
    "\n",
    "    import json\n",
    "    import logging\n",
    "\n",
    "    from google.cloud import aiplatform as aip\n",
    "\n",
    "    # Fetch model eval info\n",
    "    def get_eval_info(client, model_name):\n",
    "        from google.protobuf.json_format import MessageToDict\n",
    "\n",
    "        response = client.list_model_evaluations(parent=model_name)\n",
    "        metrics_list = []\n",
    "        metrics_string_list = []\n",
    "        for evaluation in response:\n",
    "            print(\"model_evaluation\")\n",
    "            print(\" name:\", evaluation.name)\n",
    "            print(\" metrics_schema_uri:\", evaluation.metrics_schema_uri)\n",
    "            metrics = MessageToDict(evaluation._pb.metrics)\n",
    "            for metric in metrics.keys():\n",
    "                logging.info(\"metric: %s, value: %s\", metric, metrics[metric])\n",
    "            metrics_str = json.dumps(metrics)\n",
    "            metrics_list.append(metrics)\n",
    "            metrics_string_list.append(metrics_str)\n",
    "\n",
    "        return (\n",
    "            evaluation.name,\n",
    "            metrics_list,\n",
    "            metrics_string_list,\n",
    "        )\n",
    "\n",
    "    # Use the given metrics threshold(s) to determine whether the model is\n",
    "    # accurate enough to deploy.\n",
    "    def classification_thresholds_check(metrics_dict, thresholds_dict):\n",
    "        for k, v in thresholds_dict.items():\n",
    "            logging.info(\"k {}, v {}\".format(k, v))\n",
    "            if k in [\"auRoc\", \"auPrc\"]:  # higher is better\n",
    "                if metrics_dict[k] < v:  # if under threshold, don't deploy\n",
    "                    logging.info(\"{} < {}; returning False\".format(metrics_dict[k], v))\n",
    "                    return False\n",
    "        logging.info(\"threshold checks passed.\")\n",
    "        return True\n",
    "\n",
    "    def log_metrics(metrics_list, metricsc):\n",
    "        test_confusion_matrix = metrics_list[0][\"confusionMatrix\"]\n",
    "        logging.info(\"rows: %s\", test_confusion_matrix[\"rows\"])\n",
    "\n",
    "        # log the ROC curve\n",
    "        fpr = []\n",
    "        tpr = []\n",
    "        thresholds = []\n",
    "        for item in metrics_list[0][\"confidenceMetrics\"]:\n",
    "            fpr.append(item.get(\"falsePositiveRate\", 0.0))\n",
    "            tpr.append(item.get(\"recall\", 0.0))\n",
    "            thresholds.append(item.get(\"confidenceThreshold\", 0.0))\n",
    "        print(f\"fpr: {fpr}\")\n",
    "        print(f\"tpr: {tpr}\")\n",
    "        print(f\"thresholds: {thresholds}\")\n",
    "        metricsc.log_roc_curve(fpr, tpr, thresholds)\n",
    "\n",
    "        # log the confusion matrix\n",
    "        annotations = []\n",
    "        for item in test_confusion_matrix[\"annotationSpecs\"]:\n",
    "            annotations.append(item[\"displayName\"])\n",
    "        logging.info(\"confusion matrix annotations: %s\", annotations)\n",
    "        metricsc.log_confusion_matrix(\n",
    "            annotations,\n",
    "            test_confusion_matrix[\"rows\"],\n",
    "        )\n",
    "\n",
    "        # log textual metrics info as well\n",
    "        for metric in metrics_list[0].keys():\n",
    "            if metric != \"confidenceMetrics\":\n",
    "                val_string = json.dumps(metrics_list[0][metric])\n",
    "                metrics.log_metric(metric, val_string)\n",
    "        # metrics.metadata[\"model_type\"] = \"AutoML Tabular classification\"\n",
    "\n",
    "    logging.getLogger().setLevel(logging.INFO)\n",
    "    aip.init(project=project)\n",
    "    # extract the model resource name from the input Model Artifact\n",
    "    model_resource_path = model.metadata[\"resourceName\"]\n",
    "    logging.info(\"model path: %s\", model_resource_path)\n",
    "\n",
    "    client_options = {\"api_endpoint\": api_endpoint}\n",
    "    # Initialize client that will be used to create and send requests.\n",
    "    client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
    "    eval_name, metrics_list, metrics_str_list = get_eval_info(\n",
    "        client, model_resource_path\n",
    "    )\n",
    "    logging.info(\"got evaluation name: %s\", eval_name)\n",
    "    logging.info(\"got metrics list: %s\", metrics_list)\n",
    "    log_metrics(metrics_list, metricsc)\n",
    "\n",
    "    thresholds_dict = json.loads(thresholds_dict_str)\n",
    "    deploy = classification_thresholds_check(metrics_list[0], thresholds_dict)\n",
    "    if deploy:\n",
    "        dep_decision = \"true\"\n",
    "    else:\n",
    "        dep_decision = \"false\"\n",
    "    logging.info(\"deployment decision is %s\", dep_decision)\n",
    "\n",
    "    return (dep_decision,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9b3ea4d6-ac63-4980-bf51-94f70a3dac30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "automl-beans1657533480\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "DISPLAY_NAME = 'automl-beans{}'.format(str(int(time.time())))\n",
    "print(DISPLAY_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1f5a333b-89d0-4142-a5f9-69bb9f00afe5",
   "metadata": {},
   "outputs": [],
   "source": [
    "@pipeline(name=\"automl-tab-beans-training-v2\",\n",
    "                  pipeline_root=PIPELINE_ROOT)\n",
    "def pipeline(\n",
    "    bq_source: str = \"bq://aju-dev-demos.beans.beans1\",\n",
    "    display_name: str = DISPLAY_NAME,\n",
    "    project: str = PROJECT_ID,\n",
    "    gcp_region: str = \"us-central1\",\n",
    "    api_endpoint: str = \"us-central1-aiplatform.googleapis.com\",\n",
    "    thresholds_dict_str: str = '{\"auRoc\": 0.95}',\n",
    "):\n",
    "    dataset_create_op = gcc_aip.TabularDatasetCreateOp(\n",
    "        project=project, display_name=display_name, bq_source=bq_source\n",
    "    )\n",
    "\n",
    "    training_op = gcc_aip.AutoMLTabularTrainingJobRunOp(\n",
    "        project=project,\n",
    "        display_name=display_name,\n",
    "        optimization_prediction_type=\"classification\",\n",
    "        budget_milli_node_hours=1000,\n",
    "        column_transformations=[\n",
    "            {\"numeric\": {\"column_name\": \"Area\"}},\n",
    "            {\"numeric\": {\"column_name\": \"Perimeter\"}},\n",
    "            {\"numeric\": {\"column_name\": \"MajorAxisLength\"}},\n",
    "            {\"numeric\": {\"column_name\": \"MinorAxisLength\"}},\n",
    "            {\"numeric\": {\"column_name\": \"AspectRation\"}},\n",
    "            {\"numeric\": {\"column_name\": \"Eccentricity\"}},\n",
    "            {\"numeric\": {\"column_name\": \"ConvexArea\"}},\n",
    "            {\"numeric\": {\"column_name\": \"EquivDiameter\"}},\n",
    "            {\"numeric\": {\"column_name\": \"Extent\"}},\n",
    "            {\"numeric\": {\"column_name\": \"Solidity\"}},\n",
    "            {\"numeric\": {\"column_name\": \"roundness\"}},\n",
    "            {\"numeric\": {\"column_name\": \"Compactness\"}},\n",
    "            {\"numeric\": {\"column_name\": \"ShapeFactor1\"}},\n",
    "            {\"numeric\": {\"column_name\": \"ShapeFactor2\"}},\n",
    "            {\"numeric\": {\"column_name\": \"ShapeFactor3\"}},\n",
    "            {\"numeric\": {\"column_name\": \"ShapeFactor4\"}},\n",
    "            {\"categorical\": {\"column_name\": \"Class\"}},\n",
    "        ],\n",
    "        dataset=dataset_create_op.outputs[\"dataset\"],\n",
    "        target_column=\"Class\",\n",
    "    )\n",
    "    model_eval_task = classification_model_eval_metrics(\n",
    "        project,\n",
    "        gcp_region,\n",
    "        api_endpoint,\n",
    "        thresholds_dict_str,\n",
    "        training_op.outputs[\"model\"],\n",
    "    )\n",
    "\n",
    "    with dsl.Condition(\n",
    "        model_eval_task.outputs[\"dep_decision\"] == \"true\",\n",
    "        name=\"deploy_decision\",\n",
    "    ):\n",
    "\n",
    "        endpoint_op = gcc_aip.EndpointCreateOp(\n",
    "            project=project,\n",
    "            location=gcp_region,\n",
    "            display_name=\"train-automl-beans\",\n",
    "        )\n",
    "\n",
    "        gcc_aip.ModelDeployOp(\n",
    "            model=training_op.outputs[\"model\"],\n",
    "            endpoint=endpoint_op.outputs[\"endpoint\"],\n",
    "            dedicated_resources_min_replica_count=1,\n",
    "            dedicated_resources_max_replica_count=1,\n",
    "            dedicated_resources_machine_type=\"n1-standard-4\",\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4b461df3-20e1-4359-9fff-3f83ad152d37",
   "metadata": {},
   "outputs": [],
   "source": [
    "compiler.Compiler().compile(\n",
    "    pipeline_func=pipeline, package_path=\"tab_classif_pipeline.json\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cac60b9f-eac0-4e47-9b12-776603dc85cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "ml_pipeline_job = aiplatform.PipelineJob(\n",
    "    display_name=\"automl-tab-beans-training\",\n",
    "    template_path=\"tab_classif_pipeline.json\",\n",
    "    pipeline_root=PIPELINE_ROOT,\n",
    "    parameter_values={\"project\": PROJECT_ID, \"display_name\": DISPLAY_NAME},\n",
    "    enable_caching=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b4ff2c29-c3b5-4eba-84b8-222adc0e372c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.pipeline_jobs:Creating PipelineJob\n",
      "INFO:google.cloud.aiplatform.pipeline_jobs:PipelineJob created. Resource name: projects/896262813704/locations/us-central1/pipelineJobs/automl-tab-beans-training-v2-20220711100048\n",
      "INFO:google.cloud.aiplatform.pipeline_jobs:To use this PipelineJob in another session:\n",
      "INFO:google.cloud.aiplatform.pipeline_jobs:pipeline_job = aiplatform.PipelineJob.get('projects/896262813704/locations/us-central1/pipelineJobs/automl-tab-beans-training-v2-20220711100048')\n",
      "INFO:google.cloud.aiplatform.pipeline_jobs:View Pipeline Job:\n",
      "https://console.cloud.google.com/vertex-ai/locations/us-central1/pipelines/runs/automl-tab-beans-training-v2-20220711100048?project=896262813704\n"
     ]
    }
   ],
   "source": [
    "ml_pipeline_job.submit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3c7fbb1-0c3d-4271-80ea-b19cee86c853",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-3.m94",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-3:m94"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}