Display Markdown Files from GitHub Enterprise (#368)

* Display Markdown Files from GitHub Enterprise * Enable upload of assets from GitHub Enterprise * Enable displaying README files from GitHub Enterprise Signed-off-by: Christian Kadner <ckadner@us.ibm.com> * Pin node version to 16.13 Signed-off-by: Christian Kadner <ckadner@us.ibm.com> Signed-off-by: Christian Kadner <ckadner@us.ibm.com>
machine-learning-exchange · Nov 11, 2022 · a969ff9 · a969ff9
1 parent e08c853
commit a969ff9
Show file tree

Hide file tree

Showing 24 changed files with 216 additions and 75 deletions.
diff --git a/api/README.md b/api/README.md
@@ -105,25 +105,33 @@ desired.
 
 ## Build the Docker Image
 
+    DOCKER_USER=<your_docker_user_id>
+    IMAGE_TAG=<tag_or_version>
+
     cd server
-    docker build -t <your_docker_user_id>/mlx-api-server:0.1 .
-    docker login
-    docker push <your_docker_user_id>/mlx-api-server:0.1
+    docker build -t ${DOCKER_USER}/mlx-api:${IMAGE_TAG} .
+    docker push ${DOCKER_USER}/mlx-api:${IMAGE_TAG}
     cd ..
 
+
 ## (Re-)Deploy to Kubernetes Cluster
 
 Change the Docker image tag in the deployment spec `server/mlx-api.yml` 
-from `image: docker.io/aipipeline/mlx-api:nightly-master` 
+from `image: docker.io/mlexchange/mlx-api:nightly-main` 
 to `image: docker.io/<your_docker_user_id>/mlx-api-server:0.1`
 and then run:
 
     ./deploy.sh
 
 or:
 
-    kubectl delete -f ./server/mlx-api.yml
-    kubectl apply -f ./server/mlx-api.yml
+    kubectl -n kubeflow delete -f ./server/mlx-api.yml
+    kubectl -n kubeflow apply -f ./server/mlx-api.yml
+
+or:
+
+    IMG="${DOCKER_USER}/mlx-api:${IMAGE_TAG}"
+    kubectl -n kubeflow set image deployment/mlx-api mlx-api-server=${IMG}
 
 
 ## Testing API Code Changes Locally with Docker Compose

diff --git a/api/client/README.md b/api/client/README.md
@@ -4,8 +4,8 @@ MLX API Client
 
 This Python package is automatically generated by the [Swagger Codegen](https://github.com/swagger-api/swagger-codegen) project:
 
-- API version: 0.1.30-upload-catalog-from-url
-- Package version: 1.0.0
+- API version: 0.2.0-ghe-readme
+- Package version: 0.2.0
 - Build package: io.swagger.codegen.languages.PythonClientCodegen
 
 ## Requirements.

diff --git a/api/client/setup.py b/api/client/setup.py
@@ -18,7 +18,7 @@
 from setuptools import setup, find_packages  # noqa: H301
 
 NAME = "mlx-client"
-VERSION = "0.1.30-upload-catalog-from-url"
+VERSION = "0.2.0-ghe-readme"
 # To install the library, run the following
 #
 # python setup.py install

diff --git a/api/client/swagger_client/configuration.py b/api/client/swagger_client/configuration.py
@@ -235,6 +235,6 @@ def to_debug_report(self):
         return "Python SDK Debug Report:\n"\
                "OS: {env}\n"\
                "Python Version: {pyversion}\n"\
-               "Version of the API: 0.1.30-upload-catalog-from-url\n"\
-               "SDK Package Version: 0.1.0".\
+               "Version of the API: 0.2.0-ghe-readme\n"\
+               "SDK Package Version: 0.2.0".\
                format(env=sys.platform, pyversion=sys.version)
diff --git a/api/server/mlx-api.yml b/api/server/mlx-api.yml
@@ -48,12 +48,12 @@ spec:
       labels:
         service: mlx-api
         environment: dev
-        version: v0.1.29
+        version: v0.2.0
     spec:
       serviceAccountName: mlx-api
       containers:
       - name: mlx-api-server
-        image: docker.io/mlexchange/mlx-api:nightly-main
+        image: docker.io/mlexchange/mlx-api:v0.2.0
         imagePullPolicy: Always
         ports:
         - containerPort: 8080

diff --git a/api/server/setup.py b/api/server/setup.py
@@ -6,7 +6,7 @@
 from setuptools import setup, find_packages
 
 NAME = "mlx-api"
-VERSION = "0.1.30-upload-catalog-from-url"
+VERSION = "0.2.0-ghe-readme"
 
 # To install the library, run the following
 #

diff --git a/api/server/swagger_server/__init__.py b/api/server/swagger_server/__init__.py
@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-VERSION = "0.1.30-upload-catalog-from-url"
+VERSION = "0.2.0-ghe-readme"
diff --git a/api/server/swagger_server/controllers_impl/__init__.py b/api/server/swagger_server/controllers_impl/__init__.py
@@ -4,9 +4,10 @@
 
 import requests
 
+from os import environ as env
 from werkzeug.datastructures import FileStorage
 
-from kfp_tekton.compiler._k8s_helper import sanitize_k8s_name;
+from kfp_tekton.compiler._k8s_helper import sanitize_k8s_name
 from swagger_server.data_access.minio_client import extract_yaml_from_tarfile
 from swagger_server.models.api_parameter import ApiParameter
 from swagger_server.util import ApiError
@@ -17,6 +18,9 @@
 # TODO: move into controllers_impl/util.py
 ###############################################################################
 
+ghe_api_token = env.get("GHE_API_TOKEN")
+
+
 def get_yaml_file_content_from_uploadfile(uploadfile: FileStorage):
 
     file_name = uploadfile.filename
@@ -69,6 +73,12 @@ def download_file_content_from_url(url: str, bearer_token: str = None) -> bytes:
     if bearer_token and "?token=" not in url:
         request_headers.update({"Authorization": f"Bearer {bearer_token}"})
 
+    if "github.ibm.com" in url and "?token=" not in url:
+        if not bearer_token and not ghe_api_token:
+            raise ApiError(f"Must provide API token to access files on GitHub Enterprise: {url}", 422)
+        else:
+            request_headers.update({'Authorization': f'token {bearer_token or ghe_api_token}'})
+
     try:
         raw_url = url.replace("/blob/", "/") \
             .replace("/github.ibm.com/", "/raw.github.ibm.com/") \

diff --git a/api/server/swagger_server/controllers_impl/notebook_service_controller_impl.py b/api/server/swagger_server/controllers_impl/notebook_service_controller_impl.py
@@ -10,13 +10,12 @@
 
 from datetime import datetime
 from io import BytesIO
-from os import environ as env
 from typing import AnyStr
 from urllib.parse import urlparse
 from werkzeug.datastructures import FileStorage
 
 from swagger_server.controllers_impl import download_file_content_from_url, \
-    get_yaml_file_content_from_uploadfile, validate_parameters, validate_id
+    get_yaml_file_content_from_uploadfile, validate_id, ghe_api_token
 from swagger_server.data_access.minio_client import store_file, delete_objects, \
     get_file_content_and_url, enable_anonymous_read_access, NoSuchKey, \
     create_tarfile, get_object_url
@@ -34,9 +33,6 @@
 from swagger_server.util import ApiError
 
 
-ghe_api_token = env.get("IBM_GHE_API_TOKEN")
-
-
 def approve_notebooks_for_publishing(notebook_ids):  # noqa: E501
     """approve_notebooks_for_publishing
 
@@ -470,7 +466,8 @@ def _download_notebook(url: str, enterprise_github_api_token: str) -> dict:
 
     request_headers = dict()
 
-    if "ibm.com" in url and "?token=" not in url:
+    # TODO: re-use ./init.py#download_file_content_from_url
+    if "github.ibm.com" in url and "?token=" not in url:
         if not enterprise_github_api_token and not ghe_api_token:
             raise ApiError(f"Must provide API token to access notebooks on Enterprise GitHub: {url}", 422)
         else:

diff --git a/api/server/swagger_server/swagger/swagger.yaml b/api/server/swagger_server/swagger/swagger.yaml
@@ -5,7 +5,7 @@
 swagger: "2.0"
 info:
   description: "MLX API Extension for Kubeflow Pipelines"
-  version: "0.1.30-upload-catalog-from-url"
+  version: "0.2.0-ghe-readme"
   title: "MLX API"
 basePath: "/apis/v1alpha1"
 schemes:

diff --git a/api/swagger/swagger.yaml b/api/swagger/swagger.yaml
@@ -5,7 +5,7 @@
 swagger: "2.0"
 
 info:
-  version: "0.1.30-upload-catalog-from-url"
+  version: "0.2.0-ghe-readme"
   title: "MLX API"
   description: "MLX API Extension for Kubeflow Pipelines"
 

diff --git a/dashboard/origin-mlx/Dockerfile b/dashboard/origin-mlx/Dockerfile
@@ -1,5 +1,5 @@
 # base image
-FROM node:16-slim
+FROM node:16.13-slim
 
 # create app directory
 RUN mkdir -p /workspace

diff --git a/dashboard/origin-mlx/README.md b/dashboard/origin-mlx/README.md
@@ -55,9 +55,12 @@ http://localhost:3000/settings
 ## Build a Docker Image for MLX UI
 
 ```Bash
+export DOCKER_USER=<your-docker-user-id>
+export IMAGE_TAG=<tag-or-version>
+
 cd dashboard/origin-mlx
-docker build -t <your docker user-id>/mlx-ui:<tag name> -f Dockerfile .
-docker push <your docker user-id>/<repo name>:<tag name>
+docker build -t ${DOCKER_USER}/mlx-ui:${IMAGE_TAG} -f Dockerfile .
+docker push ${DOCKER_USER}/mlx-ui:${IMAGE_TAG}
 ```
 
 ## (Re-)Deploy to Kubernetes Cluster
@@ -208,7 +211,12 @@ There are a few environment variables that can be defined that dictate how MLX i
 * `REACT_APP_DISABLE_LOGIN` - A switch to turn off login mechanism
 * `REACT_APP_KFP_STANDALONE` - The KFP is standalone deployment or not
 * `REACT_APP_TTL` - The amount of seconds a cached entry remains valid for (24 hours by default)
-* `REACT_APP_CACHE_INTERVAL` - The minimum amount of time in seconds between two checks on the validity of the cache's contents (24 hours by default)
+* `REACT_APP_CACHE_INTERVAL` - The minimum amount of time in seconds between two checks on the validity of the cache's 
+  contents (24 hours by default)
+* `REACT_APP_GHE_API_TOKEN` - Enterprise GitHub API Token to "read" Markdown files from GitHub Enterprise. Only use when
+  MLX deployment is behind corporate firewall. The minimal set of permission required for the token are `repo` and 
+  `admin:org/read:org` (on a private repository).
+
 
 # Caching Details
 

diff --git a/dashboard/origin-mlx/mlx-ui.yaml b/dashboard/origin-mlx/mlx-ui.yaml
@@ -28,7 +28,7 @@ spec:
       containers:
       - name: mlx-ui
         # You can use your own webapp image below
-        image: mlexchange/mlx-ui:nightly-origin-main
+        image: mlexchange/mlx-ui:v0.2.0
         imagePullPolicy: Always
         env:
         - name: REACT_APP_BRAND
@@ -38,7 +38,7 @@ spec:
         - name: REACT_APP_UPLOAD
           value: "true"
         - name: REACT_APP_BASE_PATH
-          value: /mlx
+          value: ""
         ports:
         - containerPort: 3000
       serviceAccountName: mlx-ui

diff --git a/dashboard/origin-mlx/package-lock.json b/dashboard/origin-mlx/package-lock.json
diff --git a/dashboard/origin-mlx/package.json b/dashboard/origin-mlx/package.json
@@ -39,7 +39,7 @@
     "remark-gfm": "^3.0.0",
     "styled-components": "^4.3.1",
     "title-case": "^3.0.3",
-    "typescript": "^3.5.2",
+    "typescript": "^4.8.4",
     "typestyle": "^2.0.1",
     "yamljs": "^0.3.0"
   },

diff --git a/dashboard/origin-mlx/server/package-lock.json b/dashboard/origin-mlx/server/package-lock.json
diff --git a/dashboard/origin-mlx/server/package.json b/dashboard/origin-mlx/server/package.json
@@ -11,7 +11,7 @@
     "passport-http": "^0.3.0",
     "session-file-store": "^1.5.0",
     "set-value": "^4.1.0",
-    "typescript": "^3.5.2"
+    "typescript": "^4.8.4"
   },
   "devDependencies": {
     "@types/cookie-parser": "^1.4.2",