diff --git a/api/examples/catalog_api.py b/api/examples/catalog_api.py index 85593982..b1a80a08 100644 --- a/api/examples/catalog_api.py +++ b/api/examples/catalog_api.py @@ -25,7 +25,8 @@ catalog_upload_file = "./../../bootstrapper/catalog_upload.json" -IBM_GHE_API_TOKEN = env.get("IBM_GHE_API_TOKEN") +GHE_API_TOKEN = env.get("GHE_API_TOKEN") +GHE_WEB_URL = env.get("GHE_WEB_URL", "github.ibm.com") def get_swagger_client(): @@ -59,7 +60,7 @@ def upload_catalog_assets(upload_file=catalog_upload_file) -> ApiCatalogUploadRe upload_items = json.load(f) upload_body = ApiCatalogUpload( - api_access_tokens=[ApiAccessToken(api_token=IBM_GHE_API_TOKEN, url_host="github.ibm.com")], + api_access_tokens=[ApiAccessToken(api_token=GHE_API_TOKEN, url_host=GHE_WEB_URL)], components=upload_items.get("components"), datasets=upload_items.get("datasets"), models=upload_items.get("models"), diff --git a/api/examples/components_api.py b/api/examples/components_api.py index 84c31f6b..242dad6b 100644 --- a/api/examples/components_api.py +++ b/api/examples/components_api.py @@ -364,7 +364,7 @@ def main(): component = list_components(filter_dict={"name": 'Create Secret - Kubernetes Cluster'})[0] generate_code(component.id) args = { - 'token': env.get("IBM_GHE_API_TOKEN"), + 'token': env.get("GHE_API_TOKEN"), 'url': 'https://raw.github.ibm.com/user/repo/master/secret.yml', 'name': 'my-test-credential' } diff --git a/api/examples/notebooks_api.py b/api/examples/notebooks_api.py index 83e99ccd..d69bd6c9 100644 --- a/api/examples/notebooks_api.py +++ b/api/examples/notebooks_api.py @@ -35,7 +35,9 @@ yaml_files = sorted(filter(lambda f: "template" not in f, glob("./../../../katalog/notebook-samples/*.yaml", recursive=True))) -IBM_GHE_API_TOKEN = env.get("IBM_GHE_API_TOKEN") +GHE_API_TOKEN = env.get("GHE_API_TOKEN") +GHE_WEB_URL = env.get("GHE_WEB_URL", "github.ibm.com") +GHE_RAW_URL = env.get("GHE_RAW_URL", "raw.github.ibm.com") def get_swagger_client(): @@ -103,8 +105,8 @@ def upload_notebook_templates(yaml_files: [str] = yaml_files) -> [str]: with open(yaml_file, "rb") as f: yaml_dict = yaml.load(f, Loader=yaml.SafeLoader) - if "github.ibm.com" in yaml_dict["implementation"]["github"]["source"]: - api_token = IBM_GHE_API_TOKEN + if GHE_WEB_URL in yaml_dict["implementation"]["github"]["source"]: + api_token = GHE_API_TOKEN else: api_token = None @@ -365,9 +367,9 @@ def download_notebooks_from_github(): download_url = url.replace("/blob", "")\ .replace("github.com", "raw.githubusercontent.com")\ - .replace("github.ibm.com", "raw.github.ibm.com") + .replace(GHE_WEB_URL, GHE_RAW_URL) - if "github.ibm.com" in url: + if GHE_WEB_URL in url: headers = {'Authorization': 'token %s' % env.get("IBM_GHE_API_TOKEN")} else: headers = {} diff --git a/api/server/swagger_server/controllers_impl/__init__.py b/api/server/swagger_server/controllers_impl/__init__.py index dd1c3649..dfc51eab 100644 --- a/api/server/swagger_server/controllers_impl/__init__.py +++ b/api/server/swagger_server/controllers_impl/__init__.py @@ -18,7 +18,9 @@ # TODO: move into controllers_impl/util.py ############################################################################### -ghe_api_token = env.get("GHE_API_TOKEN") +GHE_API_TOKEN = env.get("GHE_API_TOKEN") +GHE_WEB_URL = env.get("GHE_WEB_URL", "github.ibm.com") +GHE_RAW_URL = env.get("GHE_RAW_URL", "raw.github.ibm.com") def get_yaml_file_content_from_uploadfile(uploadfile: FileStorage): @@ -73,15 +75,15 @@ def download_file_content_from_url(url: str, bearer_token: str = None) -> bytes: if bearer_token and "?token=" not in url: request_headers.update({"Authorization": f"Bearer {bearer_token}"}) - if "github.ibm.com" in url and "?token=" not in url: - if not bearer_token and not ghe_api_token: + if GHE_WEB_URL in url and "?token=" not in url: + if not bearer_token and not GHE_API_TOKEN: raise ApiError(f"Must provide API token to access files on GitHub Enterprise: {url}", 422) else: - request_headers.update({'Authorization': f'token {bearer_token or ghe_api_token}'}) + request_headers.update({'Authorization': f'token {bearer_token or GHE_API_TOKEN}'}) try: raw_url = url.replace("/blob/", "/") \ - .replace("/github.ibm.com/", "/raw.github.ibm.com/") \ + .replace(GHE_WEB_URL, GHE_RAW_URL) \ .replace("/github.com/", "/raw.githubusercontent.com/") response = requests.get(raw_url, allow_redirects=True, headers=request_headers) diff --git a/api/server/swagger_server/controllers_impl/notebook_service_controller_impl.py b/api/server/swagger_server/controllers_impl/notebook_service_controller_impl.py index 2f87c69b..83ced770 100644 --- a/api/server/swagger_server/controllers_impl/notebook_service_controller_impl.py +++ b/api/server/swagger_server/controllers_impl/notebook_service_controller_impl.py @@ -15,7 +15,8 @@ from werkzeug.datastructures import FileStorage from swagger_server.controllers_impl import download_file_content_from_url, \ - get_yaml_file_content_from_uploadfile, validate_id, ghe_api_token + get_yaml_file_content_from_uploadfile, validate_id, GHE_API_TOKEN, \ + GHE_WEB_URL, GHE_RAW_URL from swagger_server.data_access.minio_client import store_file, delete_objects, \ get_file_content_and_url, enable_anonymous_read_access, NoSuchKey, \ create_tarfile, get_object_url @@ -454,7 +455,7 @@ def _upload_notebook_yaml(yaml_file_content: AnyStr, name=None, access_token=Non file_name="requirements.txt", file_content=requirements_all.encode()) # if the url included an access token, replace the original url with the s3 url - if "?token=" in url or "github.ibm.com" in url: + if "?token=" in url or GHE_WEB_URL in url: api_notebook.url = s3_url update_multiple(ApiNotebook, [notebook_id], "url", s3_url) enable_anonymous_read_access(bucket_name="mlpipeline", prefix="notebooks/*") @@ -467,14 +468,14 @@ def _download_notebook(url: str, enterprise_github_api_token: str) -> dict: request_headers = dict() # TODO: re-use ./init.py#download_file_content_from_url - if "github.ibm.com" in url and "?token=" not in url: - if not enterprise_github_api_token and not ghe_api_token: + if GHE_WEB_URL in url and "?token=" not in url: + if not enterprise_github_api_token and not GHE_API_TOKEN: raise ApiError(f"Must provide API token to access notebooks on Enterprise GitHub: {url}", 422) else: - request_headers.update({'Authorization': f'token {enterprise_github_api_token or ghe_api_token}'}) + request_headers.update({'Authorization': f'token {enterprise_github_api_token or GHE_API_TOKEN}'}) try: - raw_url = url.replace("/github.ibm.com/", "/raw.github.ibm.com/")\ + raw_url = url.replace(GHE_WEB_URL, GHE_RAW_URL)\ .replace("/github.com/", "/raw.githubusercontent.com/")\ .replace("/blob/", "/") response = requests.get(raw_url, allow_redirects=True, headers=request_headers) diff --git a/dashboard/origin-mlx/Dockerfile b/dashboard/origin-mlx/Dockerfile index c29d3bf1..8cb898d0 100644 --- a/dashboard/origin-mlx/Dockerfile +++ b/dashboard/origin-mlx/Dockerfile @@ -28,7 +28,8 @@ USER node # mark as production build ENV NODE_ENV=production -# run build on container startup in order to build in environment variables +# run `build` at container startup time to render the REACT_APP environment +# variables into the JavaScript bundle that will run on the client Web browser # - https://create-react-app.dev/docs/adding-custom-environment-variables/ # TODO: find a better solution, i.e. # - https://www.tutorialworks.com/openshift-deploy-react-app/ diff --git a/dashboard/origin-mlx/README.md b/dashboard/origin-mlx/README.md index 453a0f3f..005a8101 100644 --- a/dashboard/origin-mlx/README.md +++ b/dashboard/origin-mlx/README.md @@ -213,7 +213,7 @@ There are a few environment variables that can be defined that dictate how MLX i * `REACT_APP_TTL` - The amount of seconds a cached entry remains valid for (24 hours by default) * `REACT_APP_CACHE_INTERVAL` - The minimum amount of time in seconds between two checks on the validity of the cache's contents (24 hours by default) -* `REACT_APP_GHE_API_TOKEN` - Enterprise GitHub API Token to "read" Markdown files from GitHub Enterprise. Only use when +* `GHE_API_TOKEN` - Enterprise GitHub API Token to "read" Markdown files from GitHub Enterprise. Only use when MLX deployment is behind corporate firewall. The minimal set of permission required for the token are `repo` and `admin:org/read:org` (on a private repository). diff --git a/dashboard/origin-mlx/server/package-lock.json b/dashboard/origin-mlx/server/package-lock.json index 75e7949e..63e50e32 100644 --- a/dashboard/origin-mlx/server/package-lock.json +++ b/dashboard/origin-mlx/server/package-lock.json @@ -10,6 +10,7 @@ "express-rate-limit": "^5.2.6", "express-session": "^1.17.1", "http-proxy-middleware": "^0.18.0", + "node-fetch": "^2.6.6", "passport": "^0.4.1", "passport-http": "^0.3.0", "session-file-store": "^1.5.0", @@ -53,9 +54,9 @@ } }, "node_modules/@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "version": "4.17.14", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.14.tgz", + "integrity": "sha512-TEbt+vaPFQ+xpxFLFssxUDXj5cWCxZJjIcB7Yg0k0GMHGtgtQgpvx/MUQUeAkNbA9AAGrwkAsoeItdTgS7FMyg==", "dev": true, "dependencies": { "@types/body-parser": "*", @@ -65,9 +66,9 @@ } }, "node_modules/@types/express-serve-static-core": { - "version": "4.17.29", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.29.tgz", - "integrity": "sha512-uMd++6dMKS32EOuw1Uli3e3BPgdLIXmezcfHv7N4c1s3gkhikBplORPpMq3fuWkxncZN1reb16d5n8yhQ80x7Q==", + "version": "4.17.31", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz", + "integrity": "sha512-DxMhY+NAsTwMMFHBTtJFNp5qiHKJ7TeqOo23zVEM9alT1Ml27Q3xcTH0xwxn7Q0BbMcVEJOs/7aQtUWupUQN3Q==", "dev": true, "dependencies": { "@types/node": "*", @@ -76,24 +77,24 @@ } }, "node_modules/@types/express-session": { - "version": "1.17.4", - "resolved": "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.4.tgz", - "integrity": "sha512-7cNlSI8+oOBUHTfPXMwDxF/Lchx5aJ3ho7+p9jJZYVg9dVDJFh3qdMXmJtRsysnvS+C6x46k9DRYmrmCkE+MVg==", + "version": "1.17.5", + "resolved": "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.5.tgz", + "integrity": "sha512-l0DhkvNVfyUPEEis8fcwbd46VptfA/jmMwHfob2TfDMf3HyPLiB9mKD71LXhz5TMUobODXPD27zXSwtFQLHm+w==", "dev": true, "dependencies": { "@types/express": "*" } }, "node_modules/@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", + "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==", "dev": true }, "node_modules/@types/node": { - "version": "14.18.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.21.tgz", - "integrity": "sha512-x5W9s+8P4XteaxT/jKF0PSb7XEvo5VmqEWgsMlyeY4ZlLK8I6aH6g5TPPyDlLAep+GYf4kefb7HFyc7PAO3m+Q==", + "version": "14.18.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.33.tgz", + "integrity": "sha512-qelS/Ra6sacc4loe/3MSjXNL1dNQ/GjxNHVzuChwMfmk7HuycRLVQN2qNY3XahK+fZc5E2szqQSKUyAF0E+2bg==", "dev": true }, "node_modules/@types/passport": { @@ -118,12 +119,12 @@ "dev": true }, "node_modules/@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==", "dev": true, "dependencies": { - "@types/mime": "^1", + "@types/mime": "*", "@types/node": "*" } }, @@ -245,9 +246,9 @@ "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" }, "node_modules/body-parser": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", - "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.4", @@ -257,7 +258,7 @@ "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.10.3", + "qs": "6.11.0", "raw-body": "2.5.1", "type-is": "~1.6.18", "unpipe": "1.0.0" @@ -726,13 +727,13 @@ } }, "node_modules/express": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", - "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.0", + "body-parser": "1.20.1", "content-disposition": "0.5.4", "content-type": "~1.0.4", "cookie": "0.5.0", @@ -751,7 +752,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.10.3", + "qs": "6.11.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.18.0", @@ -916,9 +917,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", - "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==", + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", "funding": [ { "type": "individual", @@ -988,9 +989,9 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/get-intrinsic": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", - "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -1458,6 +1459,25 @@ "node": ">= 0.6" } }, + "node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -1671,9 +1691,9 @@ } }, "node_modules/qs": { - "version": "6.10.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", - "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "dependencies": { "side-channel": "^1.0.4" }, @@ -2248,6 +2268,11 @@ "node": ">=0.6" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -2428,6 +2453,20 @@ "node": ">= 0.8" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/write-file-atomic": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", @@ -2470,9 +2509,9 @@ } }, "@types/express": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", - "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", + "version": "4.17.14", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.14.tgz", + "integrity": "sha512-TEbt+vaPFQ+xpxFLFssxUDXj5cWCxZJjIcB7Yg0k0GMHGtgtQgpvx/MUQUeAkNbA9AAGrwkAsoeItdTgS7FMyg==", "dev": true, "requires": { "@types/body-parser": "*", @@ -2482,9 +2521,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.29", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.29.tgz", - "integrity": "sha512-uMd++6dMKS32EOuw1Uli3e3BPgdLIXmezcfHv7N4c1s3gkhikBplORPpMq3fuWkxncZN1reb16d5n8yhQ80x7Q==", + "version": "4.17.31", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz", + "integrity": "sha512-DxMhY+NAsTwMMFHBTtJFNp5qiHKJ7TeqOo23zVEM9alT1Ml27Q3xcTH0xwxn7Q0BbMcVEJOs/7aQtUWupUQN3Q==", "dev": true, "requires": { "@types/node": "*", @@ -2493,24 +2532,24 @@ } }, "@types/express-session": { - "version": "1.17.4", - "resolved": "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.4.tgz", - "integrity": "sha512-7cNlSI8+oOBUHTfPXMwDxF/Lchx5aJ3ho7+p9jJZYVg9dVDJFh3qdMXmJtRsysnvS+C6x46k9DRYmrmCkE+MVg==", + "version": "1.17.5", + "resolved": "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.5.tgz", + "integrity": "sha512-l0DhkvNVfyUPEEis8fcwbd46VptfA/jmMwHfob2TfDMf3HyPLiB9mKD71LXhz5TMUobODXPD27zXSwtFQLHm+w==", "dev": true, "requires": { "@types/express": "*" } }, "@types/mime": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", - "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz", + "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==", "dev": true }, "@types/node": { - "version": "14.18.21", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.21.tgz", - "integrity": "sha512-x5W9s+8P4XteaxT/jKF0PSb7XEvo5VmqEWgsMlyeY4ZlLK8I6aH6g5TPPyDlLAep+GYf4kefb7HFyc7PAO3m+Q==", + "version": "14.18.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.33.tgz", + "integrity": "sha512-qelS/Ra6sacc4loe/3MSjXNL1dNQ/GjxNHVzuChwMfmk7HuycRLVQN2qNY3XahK+fZc5E2szqQSKUyAF0E+2bg==", "dev": true }, "@types/passport": { @@ -2535,12 +2574,12 @@ "dev": true }, "@types/serve-static": { - "version": "1.13.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", - "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==", "dev": true, "requires": { - "@types/mime": "^1", + "@types/mime": "*", "@types/node": "*" } }, @@ -2634,9 +2673,9 @@ "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" }, "body-parser": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", - "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", "requires": { "bytes": "3.1.2", "content-type": "~1.0.4", @@ -2646,7 +2685,7 @@ "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.10.3", + "qs": "6.11.0", "raw-body": "2.5.1", "type-is": "~1.6.18", "unpipe": "1.0.0" @@ -3009,13 +3048,13 @@ } }, "express": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.1.tgz", - "integrity": "sha512-zZBcOX9TfehHQhtupq57OF8lFZ3UZi08Y97dwFCkD8p9d/d2Y3M+ykKcwaMDEL+4qyUolgBDX6AblpR3fL212Q==", + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.0", + "body-parser": "1.20.1", "content-disposition": "0.5.4", "content-type": "~1.0.4", "cookie": "0.5.0", @@ -3034,7 +3073,7 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", "proxy-addr": "~2.0.7", - "qs": "6.10.3", + "qs": "6.11.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.18.0", @@ -3168,9 +3207,9 @@ } }, "follow-redirects": { - "version": "1.15.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", - "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==" + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" }, "for-in": { "version": "1.0.2", @@ -3211,9 +3250,9 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "get-intrinsic": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", - "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -3565,6 +3604,14 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -3722,9 +3769,9 @@ } }, "qs": { - "version": "6.10.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", - "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "requires": { "side-channel": "^1.0.4" } @@ -4162,6 +4209,11 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -4295,6 +4347,20 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "write-file-atomic": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", diff --git a/dashboard/origin-mlx/server/package.json b/dashboard/origin-mlx/server/package.json index 42e928ba..92a55b44 100644 --- a/dashboard/origin-mlx/server/package.json +++ b/dashboard/origin-mlx/server/package.json @@ -7,6 +7,7 @@ "express-rate-limit": "^5.2.6", "express-session": "^1.17.1", "http-proxy-middleware": "^0.18.0", + "node-fetch": "^2.6.6", "passport": "^0.4.1", "passport-http": "^0.3.0", "session-file-store": "^1.5.0", diff --git a/dashboard/origin-mlx/server/server.ts b/dashboard/origin-mlx/server/server.ts index dd81bc4b..9bdb7041 100644 --- a/dashboard/origin-mlx/server/server.ts +++ b/dashboard/origin-mlx/server/server.ts @@ -15,6 +15,8 @@ import * as proxy from 'http-proxy-middleware'; import * as path from 'path'; import * as process from 'process'; import { ClientRequest } from 'http'; +import assert = require("assert"); +import fetch from 'node-fetch'; const { MLX_API_ENDPOINT = 'mlx-api', @@ -23,6 +25,9 @@ const { KUBEFLOW_USERID_HEADER = 'kubeflow-userid', REACT_APP_DISABLE_LOGIN = 'false', REACT_APP_RATE_LIMIT = 100, + GHE_WEB_URL = 'github.ibm.com', + GHE_RAW_URL = 'raw.github.ibm.com', + GHE_API_TOKEN = '', } = process.env; const app = express() as Application; @@ -50,6 +55,39 @@ if (!disableLogin) { proxyCheckingMiddleware.push(checkPermissionMiddleware); } +app.get('/readme', (req, res) => { + let url: string = req.query.url.toString(); + + let headers: {[index: string]: any} = {} + + if (url.includes('github') && !url.includes('raw')) { + // convert plain GitHub url to "raw" url + url = url.replace('/blob/', '/') + .replace(GHE_WEB_URL, GHE_RAW_URL) + .replace('/github.com/', '/raw.githubusercontent.com/') + } + + if (url.includes(GHE_RAW_URL)) { + // for Enterprise GitHub we use a (read-only) API token, with the minimal + // set of permission required: 'repo' and 'admin:org/read:org' + if (!GHE_API_TOKEN) { + console.log('Enterprise GitHub API Token must be provided via env var GHE_API_TOKEN.') + } + headers['Authorization'] = `token ${GHE_API_TOKEN}` + } + + const options = { + method: 'GET', + headers: headers + }; + + res.setHeader('Content-Type', 'text/plain; charset=UTF-8'); + fetch(url, options) + .then(response => response.text()) + .then(text => res.send(text)) + .catch(error => res.send(error)); +}); + if (REACT_APP_BASE_PATH.length !== 0) { app.all('/' + apiPrefix + '/*', [...proxyCheckingMiddleware, getForwardProxyMiddleware(!disableLogin)]); @@ -73,9 +111,9 @@ app.use(REACT_APP_BASE_PATH, (req, res, next) => { StaticHandler(staticDir)(req, res, next); }); -// TODO: This may or may not be needed anymore. Originally there were routing issues that -// caused routes to incorrectly fail when refreshing the react page. -// These should be fixed now, but should be tested to ensure they are. +// TODO: This may or may not be needed anymore. Originally there were routing +// issues that caused routes to incorrectly fail when refreshing the react page. +// These should be fixed now, but should be tested to ensure they are. if (REACT_APP_BASE_PATH.length !== 0) { app.use('/', staticHandler); app.use('/pipelines/', staticHandler); @@ -97,12 +135,13 @@ var limiter = new ratelimit({ }); app.get('*', limiter, (req, res) => { + console.log('app.get("*")'); // TODO: look into caching this file to speed up multiple requests. res.sendFile(path.resolve(staticDir, 'index.html')); }); app.listen(port, () => { - console.log('Server listening at http://localhost:' + port); + console.log('Server is listening at http://localhost:' + port); }); function initLogin(app: express.Application) { diff --git a/dashboard/origin-mlx/src/components/Detail/KFServingDetail.tsx b/dashboard/origin-mlx/src/components/Detail/KFServingDetail.tsx index 0a82fcd4..7d69ed3e 100644 --- a/dashboard/origin-mlx/src/components/Detail/KFServingDetail.tsx +++ b/dashboard/origin-mlx/src/components/Detail/KFServingDetail.tsx @@ -105,7 +105,7 @@ export default class KFServingDetail extends React.Component { constructor(props: any) { @@ -17,51 +16,15 @@ export default class MarkdownViewer extends React.Component<{url: string}, {term } UNSAFE_componentWillMount() { - let url = this.props.url - let headers: {[index: string]: any} = {} + // use a UI server API endpoint `/readme` to request README.md files from + // GitHub Enterprise, to not expose API tokens on the client web browser + fetch(`/readme?url=${encodeURIComponent(this.props.url)}`) + .then(response => response.text()) + .then((text) => { + this.setState({terms: text}) + }) + }; - if (url.includes("github") && !url.includes("raw")) { - // convert plain GitHub url to "raw" url - url = this.props.url.replace("/blob/", "/") - .replace("/github.ibm.com/", "/raw.github.ibm.com/") - .replace("/github.com/", "/raw.githubusercontent.com/") - } - - if (url.includes("github.ibm.com")) { - // for Enterprise GitHub we use the GitHub API (v3) and a (read-only) API token - // https://docs.github.com/en/enterprise-server@3.1/rest/repos/contents#get-contents - // when using a personal access token the minimal set of permission required - // are 'repo' and 'admin:org/read:org' (on a private repository) - // - // i.e.: - // readme_url: https://github.ibm.com/CODAIT/MAX-Age-Estimation/blob/master/README.md - // - // curl -H 'Authorization: token ${REACT_APP_GHE_API_TOKEN}' \ - // -H 'Accept: application/vnd.github.v3.raw' \ - // -L https://github.ibm.com/api/v3/repos/CODAIT/MAX-Age-Estimation/contents/README.md - - const url_segments = url.split(/[#?]+/)[0].split("/") - assert(url_segments[0] === "https:") - let api_base = "https://github.ibm.com/api/v3" - let owner = url_segments[3] - let repo = url_segments[4] - let ref = url_segments[5] - let path = url_segments.slice(6).join("/") - url = `${api_base}/repos/${owner}/${repo}/contents/${path}?ref=${ref}` - - const ghe_api_token = process.env.REACT_APP_GHE_API_TOKEN - if (!ghe_api_token) { - console.log("Enterprise GitHub API Token must be provided via env var REACT_APP_GHE_API_TOKEN.") - } - headers["Authorization"] = `token ${ghe_api_token}` - headers["Accept"] = "application/vnd.github.v3.raw" - } - fetch(url, { - headers: headers - }).then((response) => response.text()).then((text) => { - this.setState({ terms: text }) - }) - } render() { return (
diff --git a/dashboard/origin-mlx/src/components/UploadButton.tsx b/dashboard/origin-mlx/src/components/UploadButton.tsx index 4b55de47..257ddd2c 100644 --- a/dashboard/origin-mlx/src/components/UploadButton.tsx +++ b/dashboard/origin-mlx/src/components/UploadButton.tsx @@ -73,7 +73,7 @@ function UploadButton(props: RouteComponentProps) { const path = `${serviceName}` console.log("path") console.log(path) - setUploadStatus({fullStatus: ("Deployment Suceeded: predictor starting now."), link: path}) + setUploadStatus({fullStatus: ("Deployment Succeeded: predictor starting now."), link: path}) // TODO: Publishable and featured properties deprecated for kfservices - will revisit this // publishable.add(path) // featured.add(path) diff --git a/dashboard/origin-mlx/src/pages/KFServingUploadPage.tsx b/dashboard/origin-mlx/src/pages/KFServingUploadPage.tsx index 985e48ec..1fb7eadb 100644 --- a/dashboard/origin-mlx/src/pages/KFServingUploadPage.tsx +++ b/dashboard/origin-mlx/src/pages/KFServingUploadPage.tsx @@ -54,7 +54,7 @@ function UploadPage() { else{ // Makes the newly uploaded file publishable and featured const path = `${serviceName}` - setUploadStatus({fullStatus: ("Deployment Suceeded: predictor starting now."), link: path}) + setUploadStatus({fullStatus: ("Deployment Succeeded: predictor starting now."), link: path}) } setFile(null) } diff --git a/dashboard/origin-mlx/src/pages/UploadPage.tsx b/dashboard/origin-mlx/src/pages/UploadPage.tsx index fb2898c3..89b8d477 100644 --- a/dashboard/origin-mlx/src/pages/UploadPage.tsx +++ b/dashboard/origin-mlx/src/pages/UploadPage.tsx @@ -86,7 +86,7 @@ function UploadPage(props: RouteComponentProps) { // Makes the newly uploaded file publishable and featured const path = response_json.id - setUploadStatus({fullStatus: ("Upload Suceeded. Click to view "), link: path}) + setUploadStatus({fullStatus: ("Upload Succeeded. Click to view "), link: path}) publishable.add(path) featured.add(path) diff --git a/docs/mlx-setup.md b/docs/mlx-setup.md index 0d6c8f38..bacd3282 100644 --- a/docs/mlx-setup.md +++ b/docs/mlx-setup.md @@ -116,7 +116,7 @@ be accessed. GHE_API_TOKEN= kubectl -n kubeflow set env deployment mlx-api GHE_API_TOKEN=${GHE_API_TOKEN} -kubectl -n kubeflow set env deployment mlx-ui REACT_APP_GHE_API_TOKEN=${GHE_API_TOKEN} +kubectl -n kubeflow set env deployment mlx-ui GHE_API_TOKEN=${GHE_API_TOKEN} ``` You can also use a [Kubernetes Secret](https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables)