diff --git a/.dockerignore b/.dockerignore index cf80bd00..4c153be5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,7 +17,6 @@ build .env.development.local .env.test.local .env.production.local -.env npm-debug.log* yarn-debug.log* @@ -62,3 +61,4 @@ venv .git .github +openml.db diff --git a/.env b/.env new file mode 100644 index 00000000..bf62b586 --- /dev/null +++ b/.env @@ -0,0 +1,23 @@ +FLASK_APP=autoapp.py +FLASK_ENV=development +SMTP_SERVER=smtp.mailtrap.io +SMTP_PORT=2525 +DATABASE_URI=sqlite:///openml.db +EMAIL_SENDER=m@mailtrao.io +SMTP_LOGIN= +SMTP_PASS= +APP_SECRET_KEY=abcd +JWT_SECRET_KEY=abcd +TESTING=True + +URL_API=https://www.openml.org/ + +# REACT +# React env variables are fixed when building the app. +# React env variables need to be prefixed with "REACT_APP", otherwise the variables are ignored for +# security reasons. +REACT_APP_URL_SITE_BACKEND=https://localhost:5000/ +REACT_APP_URL_API=https://www.openml.org/ +REACT_APP_URL_ELASTICSEARCH=https://www.openml.org/es/ +REACT_APP_ELASTICSEARCH_VERSION_MAYOR=6 +REACT_APP_URL_MINIO=https://openml1.win.tue.nl/ \ No newline at end of file diff --git a/.env.k8s b/.env.k8s new file mode 100644 index 00000000..4851ba50 --- /dev/null +++ b/.env.k8s @@ -0,0 +1,23 @@ +FLASK_APP=autoapp.py +FLASK_ENV=development +SMTP_SERVER=smtp.mailtrap.io +SMTP_PORT=2525 +DATABASE_URI=sqlite:///openml.db +EMAIL_SENDER=m@mailtrao.io +SMTP_LOGIN= +SMTP_PASS= +APP_SECRET_KEY=abcd +JWT_SECRET_KEY=abcd +TESTING=True + +URL_API=https://k8s.openml.org/ + +# REACT +# React env variables are fixed when building the app. +# React env variables need to be prefixed with "REACT_APP", otherwise the variables are ignored for +# security reasons. +REACT_APP_URL_SITE_BACKEND=https://localhost:5000/ +REACT_APP_URL_API=https://k8sapi.openml.org/ +REACT_APP_URL_ELASTICSEARCH=https://k8s.openml.org/es/ +REACT_APP_ELASTICSEARCH_VERSION_MAYOR=8 +REACT_APP_URL_MINIO=https://openml1.win.tue.nl/ \ No newline at end of file diff --git a/.flaskenv b/.flaskenv deleted file mode 100644 index 48d9643d..00000000 --- a/.flaskenv +++ /dev/null @@ -1,15 +0,0 @@ -FLASK_APP=autoapp.py -ELASTICSEARCH_SERVER=https://www.openml.org/es -OPENML_SERVER=https://www.openml.org -FLASK_ENV=development -SMTP_SERVER=smtp.mailtrap.io -SMTP_PORT=2525 -DATABASE_URI=sqlite:///openml.db -EMAIL_SENDER=m@mailtrao.io -SMTP_LOGIN= -SMTP_PASS= -APP_SECRET_KEY=abcd -JWT_SECRET_KEY=abcd -TESTING=True -SERVER_URL=https://localhost:5000/ -REDIRECT_URL=https://localhost:5000 diff --git a/.flaskenv_TEMPLATE b/.flaskenv_TEMPLATE deleted file mode 100644 index 59c6c542..00000000 --- a/.flaskenv_TEMPLATE +++ /dev/null @@ -1,12 +0,0 @@ -FLASK_APP=autoapp.py -ELASTICSEARCH_SERVER=https://www.openml.org/es -OPENML_SERVER=https://www.openml.org -FLASK_ENV=development -DATABASE_URI=mysql+pymysql://root:@localhost/openml -SMTP_SERVER=smtp.mailtrap.io -SMTP_PORT=2525 -SMTP_LOGIN= -SMTP_PASS= -APP_SECRET_KEY= -EMAIL_SERVER=localhost:5000 -REDIRECT_URL=https://localhost:5000 diff --git a/.gitignore b/.gitignore index 8fd7b44a..45647ad8 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ build .env.development.local .env.test.local .env.production.local -.env npm-debug.log* yarn-debug.log* @@ -57,3 +56,6 @@ users.sql node_modules node_modules.nosync venv + + +openml.db diff --git a/docker/Dockerfile b/docker/Dockerfile index 18664be4..6ab146a0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-bookworm +FROM python:3.11-slim-bookworm RUN apt update && apt upgrade -y RUN python -m pip install --upgrade pip diff --git a/docker/README.md b/docker/README.md index ae8a6597..904c4460 100644 --- a/docker/README.md +++ b/docker/README.md @@ -3,18 +3,34 @@ ## Usage ```bash -docker run --rm -d -p 5000:5000 --name openml-frontend openml/frontend +docker run --rm -d -p 5000:5000 --name openml-frontend --env-file .env openml/frontend:current docker kill openml-frontend # the container doesn't seem to respond to a friendly stop request ``` -To configure, you can add environment variables. See .flaskenv for the envirnoment variables. To add -the environment variables to docker, you can use `--env-file`: +For the k8s deployment, use: ```bash -docker run --rm -d -p 5000:5000 --name openml-frontend --env-file .env openml/frontend +docker run --rm -d -p 5000:5000 --name openml-frontend --env-file .env.k8s openml/frontend:k8s +docker kill openml-frontend # the container doesn't seem to respond to a friendly stop request ``` ## Build and publish + +Currently, you need to manually run the `npm run build` before a docker build. We should put +that step inside the docker build step. I didn't do that yet, because this is a minor +inconvenience, and I think we might want to refactor the build/deployment process a bit anyway +(do we want to separate the frontend and this backend into separate containers? Do we want to +merge the backend and the api-backend-server?). + +For current production: ```bash +./server/src/client/app/node_modules/.bin/env-cmd -f ./.env npm run build --prefix server/src/client/app/ docker build -f docker/Dockerfile --tag openml/frontend:latest . docker push openml/frontend:latest ``` + +For k8s: +```bash +./server/src/client/app/node_modules/.bin/env-cmd -f ./.env.k8s npm run build --prefix server/src/client/app/ +docker build -f docker/Dockerfile --tag openml/frontend:k8s . +docker push openml/frontend:k8s +``` \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index dd02df27..e587ed0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,105 +1,41 @@ -alembic==1.3.0 -argon2-cffi==19.2.0 -arrow==0.15.4 -attrs==19.3.0 -Babel==2.9.1 -bcrypt==3.1.7 -binaryornot==0.4.4 -blinker==1.4 -category-encoders==2.0.0 -certifi==2022.12.7 -cffi -chardet==3.0.4 -Click==7.0 -configparser==4.0.2 -coverage -cryptography==3.3.2 -cycler==0.10.0 -dash==1.3.0 -dash-core-components==1.2.0 -dash-html-components==1.0.1 -dash-renderer==1.1.0 -dash-table==4.3.0 -environs==7.1.0 -Flask==1.0.2 -Flask-Argon2==0.1.5.1 -Flask-BabelEx==0.9.3 -Flask-Bcrypt==0.7.1 -Flask-Caching==1.8.0 -Flask-Compress==1.4.0 -Flask-Cors==3.0.9 -Flask-Dance==3.0.0 -Flask-DebugToolbar==0.10.1 -Flask-JWT-Extended==3.24.1 -Flask-Login==0.4.1 +category-encoders==2.6.3 +dash==2.14.2 +dash-core-components==2.0.0 +dash-html-components==2.0.0 +dash-renderer==1.9.1 +dash-table==5.0.0 +environs==10.0.0 +Flask==3.0.0 +Flask-Argon2==0.3.0.0 +Flask-BabelEx==0.9.4 +Flask-Bcrypt==1.0.1 +Flask-Caching==2.1.0 +Flask-Compress==1.14 +Flask-Cors==4.0.0 +Flask-Dance==7.0.0 +Flask-DebugToolbar==0.14.1 +Flask-JWT-Extended==4.6.0 +Flask-Login==0.6.3 Flask-Mail==0.9.1 -Flask-Migrate==2.5.2 +Flask-Migrate==4.0.5 Flask-Principal==0.4.0 Flask-Security==3.0.0 -Flask-SQLAlchemy==2.4.1 -Flask-User==1.0.2.1 -Flask-WTF==0.14.2 -future==0.17.1 -gunicorn==20.0.4 -idna==2.8 -importlib-metadata -isort==4.3.21 -itsdangerous==1.1.0 -Jinja2==2.11.3 -jinja2-time==0.2.0 -joblib==1.2.0 -kiwisolver==1.1.0 -liac-arff==2.4.0 -lxml==4.9.1 -Mako==1.2.2 -MarkupSafe==1.1.1 -marshmallow==3.3.0 -matplotlib -more-itertools==8.0.2 +Flask-SQLAlchemy==3.1.1 +Flask-User==1.0.2.2 +Flask-WTF==1.2.1 +gunicorn==21.2.0 names==0.3.0 -numpy -oauthlib==3.1.0 -packaging==20.1 -pandas -passlib==1.7.2 -patsy==0.5.1 -percy==2.0.2 -plotly==4.1.1 -pluggy==0.13.1 -poyo==0.5.0 -py==1.10.0 -pycparser==2.19 -PyJWT==2.4.0 -PyMySQL==0.9.3 -pyOpenSSL==19.0.0 -pyparsing==2.4.6 -pytest==5.3.4 -pytest-flask==0.15.1 -pytest-mock==2.0.0 -pytest-sugar==0.9.2 -python-dateutil==2.8.0 -python-dotenv==0.10.3 -python-editor==1.0.4 -pytz==2019.2 -requests>=2.22.0 -requests-oauthlib==1.3.0 -retrying==1.3.3 -scikit-learn -scipy -seaborn==0.9.0 -selenium==3.141.0 -six==1.14.0 -SQLAlchemy==1.3.8 -SQLAlchemy-Utils==0.36.1 -termcolor==1.1.0 -urllib3==1.26.5 -URLObject==2.4.3 -wcwidth==0.1.7 -Werkzeug==0.16.0 -whichcraft==0.6.1 -wincertstore==0.2 -WTForms==2.2.1 -xmltodict==0.12.0 -zipp -Pillow -openml +numpy==1.26.2 +pandas==2.1.4 +pillow==10.2.0 +plotly==5.18.0 +pymysql==1.1.0 +pytest==7.4.4 +pytest-flask==1.3.0 +pytest-mock==3.12.0 +pytest-sugar==0.9.7 +python-dotenv==1.0.0 +scikit-learn==1.3.2 +scipy==1.11.4 +Werkzeug==3.0.1 +openml==0.14.1 \ No newline at end of file diff --git a/server/app.py b/server/app.py index b3b24c70..68c3e916 100644 --- a/server/app.py +++ b/server/app.py @@ -4,9 +4,6 @@ from .extensions import Base, argon2, bcrypt, db, engine, jwt from .src.dashboard.dashapp import create_dash_app -# from flask_cors import CORS -# from flask_dance.contrib.github import make_github_blueprint, github - def register_extensions(app): """Registering extensions for flask app @@ -19,10 +16,8 @@ def register_extensions(app): jwt.init_app(app) bcrypt.init_app(app) - # Database initialisation - db_session = scoped_session( - sessionmaker(autocommit=True, autoflush=False, bind=engine) - ) + # Initialization, see Flask Security + db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) Base.query = db_session.query_property() db.init_app(app) with app.app_context(): diff --git a/server/collection/views.py b/server/collection/views.py index 7db13dcb..64c9827d 100644 --- a/server/collection/views.py +++ b/server/collection/views.py @@ -1,10 +1,11 @@ +import uuid + +import openml from flask import Blueprint, jsonify, request from flask_cors import CORS -from flask_jwt_extended import get_jwt_identity, jwt_required -from server.user.models import User -import openml -import uuid -import os +from flask_jwt_extended import jwt_required + +from server.setup import setup_openml_config collection_bp = Blueprint( "collection", __name__, static_folder="server/src/client/app/build" @@ -13,21 +14,18 @@ CORS(collection_bp) +@collection_bp.before_request +def setup(): + setup_openml_config() + + @collection_bp.route("/upload-collection-runs", methods=["POST"]) -@jwt_required +@jwt_required() def upload_collection_runs(): """ Function to upload the collection_runs returns: JSON response """ - current_user = get_jwt_identity() - user = User.query.filter_by(email=current_user).first() - user_api_key = user.session_hash - openml.config.apikey = user_api_key - # TODO change line below in production - testing = os.environ.get("TESTING") - if testing: - openml.config.start_using_configuration_for_example() data = request.get_json() collection_name = data["collectionname"] description = data["description"] @@ -48,20 +46,12 @@ def upload_collection_runs(): @collection_bp.route("/upload-collection-tasks", methods=["POST"]) -@jwt_required +@jwt_required() def upload_collection_task(): """ Function to upload the collection_tasks returns: JSON response """ - current_user = get_jwt_identity() - user = User.query.filter_by(email=current_user).first() - user_api_key = user.session_hash - openml.config.apikey = user_api_key - # change line below in testing - testing = os.environ.get("TESTING") - if testing: - openml.config.start_using_configuration_for_example() data = request.get_json() collection_name = data["collectionname"] description = data["description"] diff --git a/server/config.py b/server/config.py index 473c43c8..996e538a 100644 --- a/server/config.py +++ b/server/config.py @@ -1,5 +1,6 @@ import datetime import os + from dotenv import load_dotenv @@ -8,7 +9,7 @@ class Config(object): Config object for flask app """ - load_dotenv(".flaskenv") + load_dotenv(".env") SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URI") # 'sqlite:///' + os.path.join(basedir, 'app.db') SQLALCHEMY_TRACK_MODIFICATIONS = False @@ -24,3 +25,5 @@ class Config(object): GITHUB_OAUTH_CLIENT_ID = "" GITHUB_OAUTH_CLIENT_SECRET = "" OAUTHLIB_INSECURE_TRANSPORT = True # True only for dev not for production + + diff --git a/server/data/views.py b/server/data/views.py index 94fe29b7..213a22f5 100644 --- a/server/data/views.py +++ b/server/data/views.py @@ -1,15 +1,18 @@ -from flask import Blueprint, jsonify, request -from flask_cors import CORS -from flask_jwt_extended import get_jwt_identity, jwt_required -from server.user.models import User -from werkzeug.utils import secure_filename -from pathlib import Path +import json import os +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import arff import openml import pandas as pd -import json -import arff -from urllib.parse import parse_qs, urlparse +from flask import Blueprint, jsonify, request +from flask_cors import CORS +from flask_jwt_extended import jwt_required +from werkzeug.utils import secure_filename + +from server.setup import setup_openml_config +from server.utils import current_user data_blueprint = Blueprint( "data", __name__, static_folder="server/src/client/app/build" @@ -18,15 +21,16 @@ CORS(data_blueprint) +@data_blueprint.before_request +def setup(): + setup_openml_config() + + @data_blueprint.route("/data-edit", methods=["GET", "POST"]) -@jwt_required +@jwt_required() def data_edit(): - current_user = get_jwt_identity() - user = User.query.filter_by(email=current_user).first() - openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") - if testing: - openml.config.start_using_configuration_for_example() + user = current_user() + url = request.args.get("url") parsed = urlparse(url) dataset_id = parse_qs(parsed.query)["id"] @@ -84,7 +88,7 @@ def data_edit(): ) elif request.method == "POST": j_obj = request.get_json() - # dataset = openml.datasets.get_dataset(int(dataset_id)) + owner = j_obj["owner"] description = j_obj["description"] creator = j_obj["creator"] @@ -146,19 +150,14 @@ def data_edit(): @data_blueprint.route("/data-upload", methods=["POST"]) -@jwt_required +@jwt_required() def data_upload(): """ Function to upload dataset """ - current_user = get_jwt_identity() - user = User.query.filter_by(email=current_user).first() + + user = current_user() user_api_key = user.session_hash - openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") - if testing: - openml.config.start_using_configuration_for_example() - # openml.config.start_using_configuration_for_example() print(request) data_file = request.files["dataset"] @@ -195,7 +194,6 @@ def data_upload(): arff_dict = arff.load(arff_file) attribute_names, dtypes = zip(*arff_dict["attributes"]) data = pd.DataFrame(arff_dict["data"], columns=attribute_names) - data = pd.DataFrame(arff_dict["data"], columns=attribute_names) for attribute_name, dtype in arff_dict["attributes"]: # 'real' and 'numeric' are probably interpreted correctly. # Date support needs to be added. @@ -237,16 +235,10 @@ def data_upload(): @data_blueprint.route("/data-tag", methods=["POST"]) -@jwt_required +@jwt_required() def data_tag(): j_obj = request.get_json() tag = j_obj['tag'] - current_user = get_jwt_identity() - user = User.query.filter_by(email=current_user).first() - openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") - if testing: - openml.config.start_using_configuration_for_example() url = request.args.get("url") parsed = urlparse(url) dataset_id = parse_qs(parsed.query)["id"] @@ -254,4 +246,4 @@ def data_tag(): dataset = openml.datasets.get_dataset(dataset_id) dataset.push_tag(tag) - pass \ No newline at end of file + diff --git a/server/extensions.py b/server/extensions.py index dbc2c8e0..0f5306a6 100644 --- a/server/extensions.py +++ b/server/extensions.py @@ -1,4 +1,7 @@ +import logging import os +from distutils.util import strtobool + from dotenv import load_dotenv # import sqlalchemy @@ -17,11 +20,12 @@ Declares extension for Flask App, connects with already existing database """ # specifying engine according to existing db -load_dotenv(".flaskenv") -try: +load_dotenv(".env") + + +if not strtobool(os.environ.get("TESTING", "True")): engine = create_engine( os.environ.get("DATABASE_URI"), - convert_unicode=True, echo=False, pool_size=20, max_overflow=0, @@ -29,11 +33,11 @@ ) Base = declarative_base() Base.metadata.reflect(engine) -except TypeError: +else: + logging.warning("Testing mode, using local sqlite db.") engine = create_engine( "sqlite:///" + os.path.join(basedir, "openml.db"), echo=False, - convert_unicode=True, ) Config.SQLALCHEMY_DATABASE_URI = "sqlite:///" + os.path.join(basedir, "openml.db") Base = declarative_base() diff --git a/server/public/views.py b/server/public/views.py index 3a4604f9..8cb2133c 100644 --- a/server/public/views.py +++ b/server/public/views.py @@ -1,5 +1,7 @@ import datetime import hashlib +import os +from distutils.util import strtobool from flask_cors import CORS from server.extensions import db @@ -15,9 +17,13 @@ blueprint = Blueprint("public", __name__) + CORS(blueprint) +DO_SEND_EMAIL = strtobool(os.environ.get("SEND_EMAIL", "True")) + + @blueprint.route("/signup", methods=["POST"]) def signupfunc(): """Registering user and checking for already existing user""" @@ -37,7 +43,7 @@ def signupfunc(): user.remember_code = "0000" user.created_on = "0000" user.last_login = "0000" - user.active = "0" + user.active = "0" if DO_SEND_EMAIL else "1" user.first_name = register_obj["first_name"] user.last_name = register_obj["last_name"] user.company = "0000" @@ -53,7 +59,8 @@ def signupfunc(): timestamp = timestamp.strftime("%d %H:%M:%S") md5_digest = hashlib.md5(timestamp.encode()).hexdigest() user.update_activation_code(md5_digest) - confirmation_email(user.email, md5_digest) + if DO_SEND_EMAIL: + confirmation_email(user.email, md5_digest) db.session.add(user) # db.session.commit() # user_ = User.query.filter_by(email=register_obj["email"]).first() @@ -74,7 +81,8 @@ def password(): user = User.query.filter_by(email=jobj["email"]).first() user.update_forgotten_code(md5_digest) # user.update_forgotten_time(timestamp) - forgot_password_email(user.email, md5_digest) + if DO_SEND_EMAIL: + forgot_password_email(user.email, md5_digest) db.session.merge(user) db.session.commit() return jsonify({"msg": "Token sent"}), 200 @@ -89,7 +97,8 @@ def confirmation_token(): md5_digest = hashlib.md5(timestamp.encode()).hexdigest() user = User.query.filter_by(email=jobj["email"]).first() user.update_activation_code(md5_digest) - confirmation_email(user.email, md5_digest) + if DO_SEND_EMAIL: + confirmation_email(user.email, md5_digest) # updating user groups here user_ = UserGroups(user_id=user.id, group_id=2) db.session.merge(user) diff --git a/server/setup.py b/server/setup.py new file mode 100644 index 00000000..3b1daa02 --- /dev/null +++ b/server/setup.py @@ -0,0 +1,21 @@ +import os +from distutils.util import strtobool + +import openml + +from server.utils import current_user + + +SERVER_BASE_URL = os.getenv("URL_API", "https://www.openml.org/") + + +def setup_openml_config(): + """ + This setup should be run before each request that interacts with the openml package, + because the configuration depends on the user. + """ + openml.config.server = SERVER_BASE_URL + "api/v1/xml" + if strtobool(os.environ.get("TESTING", "True")): + openml.config.start_using_configuration_for_example() + user = current_user() + openml.config.apikey = user.session_hash if user else '' diff --git a/server/src/client/app/TEMPLATE.env b/server/src/client/app/TEMPLATE.env deleted file mode 100644 index 9b03eb07..00000000 --- a/server/src/client/app/TEMPLATE.env +++ /dev/null @@ -1,3 +0,0 @@ -# URL that tells React where to find the Flask app. -# When running on a remote server, change it to the server URL -REACT_APP_SERVER_URL=http://localhost:5000/ \ No newline at end of file diff --git a/server/src/client/app/package-lock.json b/server/src/client/app/package-lock.json index 318236dd..c489cbbf 100644 --- a/server/src/client/app/package-lock.json +++ b/server/src/client/app/package-lock.json @@ -63,6 +63,9 @@ "typescript": "^4.5.4", "webfontloader": "^1.6.28" }, + "devDependencies": { + "env-cmd": "^10.1.0" + }, "optionalDependencies": { "fsevents": "^2.3.2", "lodash": "^4.17.21" @@ -8762,6 +8765,31 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/env-cmd": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/env-cmd/-/env-cmd-10.1.0.tgz", + "integrity": "sha512-mMdWTT9XKN7yNth/6N6g2GuKuJTsKMDHlQFUDacb/heQRRWOTIZ42t1rMHnQu4jYxU1ajdTeJM+9eEETlqToMA==", + "dev": true, + "dependencies": { + "commander": "^4.0.0", + "cross-spawn": "^7.0.0" + }, + "bin": { + "env-cmd": "bin/env-cmd.js" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/env-cmd/node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", diff --git a/server/src/client/app/package.json b/server/src/client/app/package.json index 79387f2f..0caff38d 100644 --- a/server/src/client/app/package.json +++ b/server/src/client/app/package.json @@ -77,5 +77,8 @@ "optionalDependencies": { "fsevents": "^2.3.2", "lodash": "^4.17.21" + }, + "devDependencies": { + "env-cmd": "^10.1.0" } } diff --git a/server/src/client/app/src/App.js b/server/src/client/app/src/App.js index 127e67a8..767bd360 100755 --- a/server/src/client/app/src/App.js +++ b/server/src/client/app/src/App.js @@ -68,7 +68,7 @@ class App extends React.Component { } }; axios - .get(process.env.REACT_APP_SERVER_URL + "verifytoken", yourConfig) + .get(process.env.REACT_APP_URL_SITE_BACKEND + "verifytoken", yourConfig) .then(response => { if ( response.statusText !== undefined && @@ -76,7 +76,7 @@ class App extends React.Component { !this.state.loggedIn ) { axios - .get(process.env.REACT_APP_SERVER_URL + "profile", yourConfig) + .get(process.env.REACT_APP_URL_SITE_BACKEND + "profile", yourConfig) .then(response => { let img = undefined; if (response.data.image.includes(path.sep)) { diff --git a/server/src/client/app/src/components/Header.js b/server/src/client/app/src/components/Header.js index e63711af..9df4398b 100755 --- a/server/src/client/app/src/components/Header.js +++ b/server/src/client/app/src/components/Header.js @@ -372,7 +372,7 @@ class UserMenu extends Component { }; axios .post( - process.env.REACT_APP_SERVER_URL + "logout", + process.env.REACT_APP_URL_SITE_BACKEND + "logout", { logout: "true" }, diff --git a/server/src/client/app/src/components/Sidebar.js b/server/src/client/app/src/components/Sidebar.js index b7d74c07..46465760 100755 --- a/server/src/client/app/src/components/Sidebar.js +++ b/server/src/client/app/src/components/Sidebar.js @@ -170,6 +170,10 @@ const SidebarFooter = styled.div` overflow: hidden; `; + +const ELASTICSEARCH_SERVER = process.env.REACT_APP_URL_ELASTICSEARCH || "https://www.openml.org/es/"; +const ELASTICSEARCH_VERSION_MAYOR = process.env.REACT_APP_ELASTICSEARCH_VERSION_MAYOR || 6 + function SidebarCategory({ name, icon, @@ -290,13 +294,19 @@ class Sidebar extends React.Component { axiosCancelToken = axios.CancelToken.source(); countUpdate = async () => { - const ELASTICSEARCH_SERVER = "https://www.openml.org/es/"; const data = { size: 0, query: { bool: { should: [ { term: { status: "active" } }, { bool: { must_not: { exists: { field: "status" } } } } ] } }, - aggs: { count_by_type: { terms: { field: "_type", size: 100 } } } + aggs: { + count_by_type: { + terms: { + field: ELASTICSEARCH_VERSION_MAYOR >= 8 ? "_index" : "_type", + size: 100 + } + } + } }; const headers = { @@ -320,15 +330,16 @@ class Sidebar extends React.Component { }); // second query for benchmark counts + const study_url = ELASTICSEARCH_VERSION_MAYOR >= 8 ? "study/_search" : "study/study/_search" const bench_data = { size: 0, query: { bool : { filter : { bool: { should: [ {"wildcard": { "name": "*benchmark*" }}, {"wildcard": { "name": "*suite*" }}] } }}} }; axios - .post(ELASTICSEARCH_SERVER + "study/study/_search", bench_data, headers) + .post(ELASTICSEARCH_SERVER + study_url, bench_data, headers) .then(response => { let counts = this.state.counts; - counts["benchmark"] = response.data.hits.total; + counts["benchmark"] = ELASTICSEARCH_VERSION_MAYOR >= 8 ? response.data.hits.total.value : response.data.hits.total; this.setState({ counts: counts }); }) .catch(error => { diff --git a/server/src/client/app/src/pages/auth/APIKey.js b/server/src/client/app/src/pages/auth/APIKey.js index 2de181df..68868277 100644 --- a/server/src/client/app/src/pages/auth/APIKey.js +++ b/server/src/client/app/src/pages/auth/APIKey.js @@ -31,7 +31,7 @@ const yourConfig = { function APIKey() { const [apikey, setApikey] = useState(false); - axios.get(process.env.REACT_APP_SERVER_URL + "api-key", yourConfig) + axios.get(process.env.REACT_APP_URL_SITE_BACKEND + "api-key", yourConfig) .then(function (response) { console.log(response); setApikey(response.data.apikey); @@ -43,7 +43,7 @@ function APIKey() { function apiFlask(event) { event.preventDefault(); axios - .post(process.env.REACT_APP_SERVER_URL + "api-key", { + .post(process.env.REACT_APP_URL_SITE_BACKEND + "api-key", { resetapikey:true }, yourConfig) .then(function (response) { diff --git a/server/src/client/app/src/pages/auth/CollectionRunsUpload.js b/server/src/client/app/src/pages/auth/CollectionRunsUpload.js index 4ad97ea9..e40687b3 100644 --- a/server/src/client/app/src/pages/auth/CollectionRunsUpload.js +++ b/server/src/client/app/src/pages/auth/CollectionRunsUpload.js @@ -42,7 +42,7 @@ function Public() { axios .post( - process.env.REACT_APP_SERVER_URL + "upload-collection-runs", + process.env.REACT_APP_URL_SITE_BACKEND + "upload-collection-runs", { description: event.target.description.value, collectionname: event.target.collectionname.value, diff --git a/server/src/client/app/src/pages/auth/CollectionTasksUpload.js b/server/src/client/app/src/pages/auth/CollectionTasksUpload.js index ce3c841a..25b462cb 100644 --- a/server/src/client/app/src/pages/auth/CollectionTasksUpload.js +++ b/server/src/client/app/src/pages/auth/CollectionTasksUpload.js @@ -44,7 +44,7 @@ function Public() { axios .post( - process.env.REACT_APP_SERVER_URL + "upload-collection-tasks", + process.env.REACT_APP_URL_SITE_BACKEND + "upload-collection-tasks", { description: event.target.description.value, collectionname: event.target.collectionname.value, diff --git a/server/src/client/app/src/pages/auth/ConfirmPage.js b/server/src/client/app/src/pages/auth/ConfirmPage.js index 48a56f64..d659b7ca 100644 --- a/server/src/client/app/src/pages/auth/ConfirmPage.js +++ b/server/src/client/app/src/pages/auth/ConfirmPage.js @@ -21,7 +21,7 @@ const Wrapper = styled(Paper)` function Confirm() { const [verifToken, setverifToken] = useState(false); console.log(window.location.href) - axios.post(process.env.REACT_APP_SERVER_URL+"confirmation",{ + axios.post(process.env.REACT_APP_URL_SITE_BACKEND+"confirmation",{ url:window.location.href, }).then(function(response) { console.log(response.data); diff --git a/server/src/client/app/src/pages/auth/ConfirmationToken.js b/server/src/client/app/src/pages/auth/ConfirmationToken.js index 3a8487b2..b8c54af0 100644 --- a/server/src/client/app/src/pages/auth/ConfirmationToken.js +++ b/server/src/client/app/src/pages/auth/ConfirmationToken.js @@ -28,7 +28,7 @@ function ConfirmationToken() { function sendflask(event){ event.preventDefault(); console.log('executed'); - axios.post(process.env.REACT_APP_SERVER_URL+"send-confirmation-token",{ + axios.post(process.env.REACT_APP_URL_SITE_BACKEND+"send-confirmation-token",{ email: event.target.email.value }).then(function(response) { console.log(response.data); diff --git a/server/src/client/app/src/pages/auth/DataEdit.js b/server/src/client/app/src/pages/auth/DataEdit.js index 83a8fadf..3e043063 100644 --- a/server/src/client/app/src/pages/auth/DataEdit.js +++ b/server/src/client/app/src/pages/auth/DataEdit.js @@ -88,7 +88,7 @@ function DataEdit() { url: window.location.href, }, }; - const response = await axios(process.env.REACT_APP_SERVER_URL + "data-edit", yourConfig); + const response = await axios(process.env.REACT_APP_URL_SITE_BACKEND + "data-edit", yourConfig); //setUserId(response.data.user_id); setName(response.data.name); @@ -114,7 +114,7 @@ function DataEdit() { if (owner === true) { axios .post( - process.env.REACT_APP_SERVER_URL + "data-edit", + process.env.REACT_APP_URL_SITE_BACKEND + "data-edit", { owner: "true", description: event.target.description.value, @@ -141,7 +141,7 @@ function DataEdit() { } else if (owner === false) { axios .post( - process.env.REACT_APP_SERVER_URL + "data-edit", + process.env.REACT_APP_URL_SITE_BACKEND + "data-edit", { owner: "false", description: event.target.description.value, diff --git a/server/src/client/app/src/pages/auth/DataUpload.js b/server/src/client/app/src/pages/auth/DataUpload.js index 46cd9298..9f99cdc9 100644 --- a/server/src/client/app/src/pages/auth/DataUpload.js +++ b/server/src/client/app/src/pages/auth/DataUpload.js @@ -125,7 +125,7 @@ function Public() { setError(false); axios .post( - process.env.REACT_APP_SERVER_URL + "data-upload", + process.env.REACT_APP_URL_SITE_BACKEND + "data-upload", data, yourConfig ) @@ -144,7 +144,7 @@ function Public() { if (editdata === true) { axios .post( - process.env.REACT_APP_SERVER_URL + "data-edit-upload", + process.env.REACT_APP_URL_SITE_BACKEND + "data-edit-upload", data, yourConfig ) diff --git a/server/src/client/app/src/pages/auth/Feedback.js b/server/src/client/app/src/pages/auth/Feedback.js index 4ea5769e..620f8b8b 100644 --- a/server/src/client/app/src/pages/auth/Feedback.js +++ b/server/src/client/app/src/pages/auth/Feedback.js @@ -27,7 +27,7 @@ function Public() { function feedbacktoflask(event) { event.preventDefault(); axios - .post(process.env.REACT_APP_SERVER_URL + "feedback", { + .post(process.env.REACT_APP_URL_SITE_BACKEND + "feedback", { email: event.target.email.value, feedback: event.target.feedback.value, }) diff --git a/server/src/client/app/src/pages/auth/Profile.js b/server/src/client/app/src/pages/auth/Profile.js index f6be5f98..e9a758b1 100755 --- a/server/src/client/app/src/pages/auth/Profile.js +++ b/server/src/client/app/src/pages/auth/Profile.js @@ -52,7 +52,7 @@ function Public() { } }; axios - .get(process.env.REACT_APP_SERVER_URL + "profile", yourConfig) + .get(process.env.REACT_APP_URL_SITE_BACKEND + "profile", yourConfig) .then(function(response) { console.log(response); @@ -80,7 +80,7 @@ function Public() { setError(false); axios .post( - process.env.REACT_APP_SERVER_URL + "profile", + process.env.REACT_APP_URL_SITE_BACKEND + "profile", { bio: event.target.biography.value, first_name: event.target.firstname.value, @@ -108,7 +108,7 @@ function Public() { // setImage(images[0]) axios .post( - process.env.REACT_APP_SERVER_URL + "image", + process.env.REACT_APP_URL_SITE_BACKEND + "image", formData, yourConfig ) diff --git a/server/src/client/app/src/pages/auth/ProfilePage.js b/server/src/client/app/src/pages/auth/ProfilePage.js index cc6c842e..d19db97a 100755 --- a/server/src/client/app/src/pages/auth/ProfilePage.js +++ b/server/src/client/app/src/pages/auth/ProfilePage.js @@ -46,6 +46,8 @@ const RedMenuIcon = styled(FontAwesomeIcon)({ color: red[400] }); +const ELASTICSEARCH_SERVER = process.env.REACT_APP_URL_ELASTICSEARCH || "https://www.openml.org/es/"; + function Public() { const [email, setEmail] = useState(""); const [bio, setBio] = useState(""); @@ -66,7 +68,7 @@ function Public() { }; axios - .get(process.env.REACT_APP_SERVER_URL + "profile", yourConfig) + .get(process.env.REACT_APP_URL_SITE_BACKEND + "profile", yourConfig) .then(function(response) { console.log(response); setImage(response.data.image); @@ -76,7 +78,7 @@ function Public() { setLname(response.data.last_name); setId(response.data.id); if (id !== false) { - fetch("https://openml.org/es/user/user/" + id.toString()) + fetch(`${ELASTICSEARCH_SERVER}user/user/` + id.toString()) .then(response => response.json()) .then(data => { setDataset(data._source.datasets_uploaded); diff --git a/server/src/client/app/src/pages/auth/ResetPage.js b/server/src/client/app/src/pages/auth/ResetPage.js index e4a14373..f2da7b0d 100644 --- a/server/src/client/app/src/pages/auth/ResetPage.js +++ b/server/src/client/app/src/pages/auth/ResetPage.js @@ -27,7 +27,7 @@ function ResetPage() { console.log(window.location.href); const [redirect, setRedirect] = useState(false); axios - .post(process.env.REACT_APP_SERVER_URL + "forgot-token", { + .post(process.env.REACT_APP_URL_SITE_BACKEND + "forgot-token", { url: window.location.href }) .then(function(response) { @@ -40,7 +40,7 @@ function ResetPage() { event.preventDefault(); console.log("executed"); axios - .post(process.env.REACT_APP_SERVER_URL + "resetpassword", { + .post(process.env.REACT_APP_URL_SITE_BACKEND + "resetpassword", { url: window.location.href, password: event.target.password.value }) diff --git a/server/src/client/app/src/pages/auth/ResetPassword.js b/server/src/client/app/src/pages/auth/ResetPassword.js index b49c6217..7eae70f7 100755 --- a/server/src/client/app/src/pages/auth/ResetPassword.js +++ b/server/src/client/app/src/pages/auth/ResetPassword.js @@ -31,7 +31,7 @@ function ResetPassword() { event.preventDefault(); console.log("executed"); axios - .post(process.env.REACT_APP_SERVER_URL + "forgotpassword", { + .post(process.env.REACT_APP_URL_SITE_BACKEND + "forgotpassword", { email: event.target.email.value }) .then(function (response) { diff --git a/server/src/client/app/src/pages/auth/SignIn.js b/server/src/client/app/src/pages/auth/SignIn.js index 014ed869..7f5f17ea 100755 --- a/server/src/client/app/src/pages/auth/SignIn.js +++ b/server/src/client/app/src/pages/auth/SignIn.js @@ -40,7 +40,7 @@ function SignIn() { function sendtoflask(event) { event.preventDefault(); axios - .post(process.env.REACT_APP_SERVER_URL + "login", { + .post(process.env.REACT_APP_URL_SITE_BACKEND + "login", { email: event.target.email.value, password: event.target.password.value }) diff --git a/server/src/client/app/src/pages/auth/SignUp.js b/server/src/client/app/src/pages/auth/SignUp.js index 2d10f104..60b5b0cb 100755 --- a/server/src/client/app/src/pages/auth/SignUp.js +++ b/server/src/client/app/src/pages/auth/SignUp.js @@ -51,7 +51,7 @@ function SignUp() { setErrorMessage("Please enter valid email"); } else { axios - .post(process.env.REACT_APP_SERVER_URL + "signup", { + .post(process.env.REACT_APP_URL_SITE_BACKEND + "signup", { first_name: event.target.fname.value, last_name: event.target.lname.value, email: event.target.email.value, diff --git a/server/src/client/app/src/pages/auth/TaskUpload.js b/server/src/client/app/src/pages/auth/TaskUpload.js index 54bcb913..ca05ff91 100644 --- a/server/src/client/app/src/pages/auth/TaskUpload.js +++ b/server/src/client/app/src/pages/auth/TaskUpload.js @@ -53,7 +53,7 @@ function Public() { event.preventDefault(); axios - .post(process.env.REACT_APP_SERVER_URL + "upload-task", { + .post(process.env.REACT_APP_URL_SITE_BACKEND + "upload-task", { dataset_id: event.target.datasetid.value, task_type: event.target.tasktype.value, target_name: event.target.targetname.value, diff --git a/server/src/client/app/src/pages/search/Dataset.js b/server/src/client/app/src/pages/search/Dataset.js index fc34a1c5..d2749212 100644 --- a/server/src/client/app/src/pages/search/Dataset.js +++ b/server/src/client/app/src/pages/search/Dataset.js @@ -36,6 +36,9 @@ const Action = styled.div` justify-content: center; `; +const SERVER_URL = process.env.REACT_APP_URL_API || "https://www.openml.org/"; +const MINIO_URL = process.env.REACT_APP_URL_MINIO || "https://openml1.win.tue.nl/"; + const CroissantComponent = ({ url }) => { const [jsonData, setJsonData] = useState({}); @@ -91,7 +94,7 @@ export class DatasetItem extends React.Component { const qualityTableColumns = ["", "Quality Name", "Value"]; const did = this.props.object.data_id; const did_padded = did.toString().padStart(4, "0"); - const bucket_url = "https://openml1.win.tue.nl/datasets/"; + const bucket_url = `${MINIO_URL}datasets/`; const bucket_bracket = Math.floor(did / 10000).toString().padStart(4, "0"); const croissant_url = bucket_url + bucket_bracket + "/" + did_padded + "/dataset_" + did + "_croissant.json"; return ( @@ -111,7 +114,7 @@ export class DatasetItem extends React.Component { - + xml @@ -119,7 +122,7 @@ export class DatasetItem extends React.Component { - + json diff --git a/server/src/client/app/src/pages/search/api.js b/server/src/client/app/src/pages/search/api.js index ed7c5718..ab73c7f8 100644 --- a/server/src/client/app/src/pages/search/api.js +++ b/server/src/client/app/src/pages/search/api.js @@ -58,7 +58,8 @@ export function getProperty(obj, param) { } } -const ELASTICSEARCH_SERVER = "https://www.openml.org/es/"; +const ELASTICSEARCH_SERVER = process.env.REACT_APP_URL_ELASTICSEARCH || "https://www.openml.org/es/"; +const ELASTICSEARCH_VERSION_MAYOR = process.env.REACT_APP_ELASTICSEARCH_VERSION_MAYOR || 6 // general search export function search( @@ -95,6 +96,7 @@ export function search( } }; } + let params = { from: from, size: size, @@ -114,7 +116,7 @@ export function search( }, aggs: { type: { - terms: { field: "_type" } + terms: { field: ELASTICSEARCH_VERSION_MAYOR >= 8 ? "_index" : "_type" } } }, _source: fields.filter(l => !!l) @@ -128,9 +130,10 @@ export function search( } // uncomment for debugging the search //console.log("Search: " + JSON.stringify(params)); - //return fetch(process.env.ELASTICSEARCH_SERVER + '/' + type + '/'+ type + '/_search?type=' + type, + //return fetch(process.env.REACT_APP_URL_ELASTICSEARCH + '/' + type + '/'+ type + '/_search?type=' + type, + const search_url = ELASTICSEARCH_VERSION_MAYOR >= 8 ? type + "/_search" : type + "/" + type + "/_search?type=" + type return fetch( - ELASTICSEARCH_SERVER + type + "/" + type + "/_search?type=" + type, + ELASTICSEARCH_SERVER + search_url, { headers: { Accept: "application/json", @@ -145,7 +148,7 @@ export function search( .then(request => request.json()) .then(data => { return { - counts: data["hits"]["total"], + counts: ELASTICSEARCH_VERSION_MAYOR >= 8 ? data["hits"]["total"]["value"] : data["hits"]["total"], results: data["hits"]["hits"].map(x => { let source = x["_source"]; let res = {}; @@ -160,7 +163,8 @@ export function search( //get specific item export function getItem(type, itemId) { - return fetch(ELASTICSEARCH_SERVER + "/" + type + "/" + type + "/" + itemId, { + const search_url = ELASTICSEARCH_VERSION_MAYOR >= 8 ? type + "/_doc/" + itemId : type + "/" + type + "/" + itemId + return fetch(ELASTICSEARCH_SERVER + search_url, { headers: { Accept: "application/json", "Content-Type": "application/json" @@ -183,7 +187,7 @@ export function getItem(type, itemId) { // Not used? export function getList(itemId) { - return fetch(ELASTICSEARCH_SERVER + "/data/data/list/tag/" + itemId, { + return fetch(ELASTICSEARCH_SERVER + "data/data/list/tag/" + itemId, { headers: { Accept: "application/json", "Content-Type": "application/json" diff --git a/server/src/dashboard/callbacks.py b/server/src/dashboard/callbacks.py index 6ecce398..b7c4e3c9 100644 --- a/server/src/dashboard/callbacks.py +++ b/server/src/dashboard/callbacks.py @@ -1,6 +1,6 @@ import re -import dash_html_components as html +from dash import html from dash.dependencies import Input, Output from .data_callbacks import register_data_callbacks diff --git a/server/src/dashboard/dashapp.py b/server/src/dashboard/dashapp.py index 9e7a1e08..2c0a74dd 100644 --- a/server/src/dashboard/dashapp.py +++ b/server/src/dashboard/dashapp.py @@ -1,10 +1,8 @@ import shutil -from pathlib import Path import dash -import dash_core_components as dcc -import dash_html_components as html import openml +from dash import dcc, html from flask_caching import Cache from .caching import CACHE_DIR_ROOT, CACHE_DIR_FLASK, CACHE_DIR_DASHBOARD diff --git a/server/src/dashboard/data_callbacks.py b/server/src/dashboard/data_callbacks.py index 0cf26e6e..17f8a226 100644 --- a/server/src/dashboard/data_callbacks.py +++ b/server/src/dashboard/data_callbacks.py @@ -1,15 +1,10 @@ import re -import dash_core_components as dcc -import dash_html_components as html import numpy as np import pandas as pd import plotly.express as px - -# from plotly.subplots import make_subplots - import plotly.graph_objs as go - +from dash import dcc, html from dash.dependencies import Input, Output, State from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor diff --git a/server/src/dashboard/flow_callbacks.py b/server/src/dashboard/flow_callbacks.py index dbf34c8e..92157ce3 100644 --- a/server/src/dashboard/flow_callbacks.py +++ b/server/src/dashboard/flow_callbacks.py @@ -4,9 +4,10 @@ import plotly.graph_objs as go from dash.dependencies import Input, Output from openml import evaluations, tasks +from openml.tasks import TaskType from .dash_config import DASH_CACHING -from openml.tasks import TaskType +from ...setup import SERVER_BASE_URL TIMEOUT = 5 * 60 if DASH_CACHING else 0 @@ -84,11 +85,11 @@ def update_flow_plots(pathname, metric, tasktype, parameter): tick_text = [] # Set clickable labels for run_id in df["run_id"].values: - link = ' ' + link = f' ' run_link.append(link) for data_id in df["data_id"].values: - link = '' + link = f'' tick_text.append(link) hover_text = [] if parameter == "None": diff --git a/server/src/dashboard/helpers.py b/server/src/dashboard/helpers.py index f796d7b4..ade5e109 100644 --- a/server/src/dashboard/helpers.py +++ b/server/src/dashboard/helpers.py @@ -36,8 +36,7 @@ def get_run_df(run_id: int): {"task_type": run.task_type}.items(), columns=["evaluations", "results"] ) df2["values"] = "" - df = df.append(df2) - df = df.append(df3) + df = pd.concat([df2, df3], ignore_index=True) df.to_pickle(CACHE_DIR_DASHBOARD / f"run{run_id}.pkl") return run, df diff --git a/server/src/dashboard/layouts.py b/server/src/dashboard/layouts.py index 1a969e55..2fc183db 100644 --- a/server/src/dashboard/layouts.py +++ b/server/src/dashboard/layouts.py @@ -1,9 +1,7 @@ -import os from typing import List, Tuple -import dash_core_components as dcc -import dash_html_components as html -import dash_table as dt +from dash import dash_table as dt +from dash import dcc, html from openml import datasets, evaluations, runs, setups, study from .caching import CACHE_DIR_DASHBOARD @@ -285,7 +283,6 @@ def get_layout_from_task(task_id): className="container", # style={'overflowY': 'hidden'} ) - return layout @@ -382,7 +379,6 @@ def get_layout_from_flow(flow_id): ], className="container", ) - return layout diff --git a/server/src/dashboard/overviews.py b/server/src/dashboard/overviews.py index 169691dd..24f1a807 100644 --- a/server/src/dashboard/overviews.py +++ b/server/src/dashboard/overviews.py @@ -1,7 +1,6 @@ -import dash_core_components as dcc -import dash_html_components as html import pandas as pd import plotly.graph_objs as go +from dash import dcc, html from dash.dependencies import Input, Output from openml import datasets, flows, runs, tasks from openml.extensions.sklearn import SklearnExtension diff --git a/server/src/dashboard/run_callbacks.py b/server/src/dashboard/run_callbacks.py index 672b623c..5bd7acc0 100644 --- a/server/src/dashboard/run_callbacks.py +++ b/server/src/dashboard/run_callbacks.py @@ -1,15 +1,12 @@ import io import re - -# import arff import urllib.request -import dash_core_components as dcc -import dash_html_components as html import numpy as np import pandas as pd import plotly import plotly.graph_objs as go +from dash import dcc, html from dash.dependencies import Input, Output from scipy.io import arff from sklearn.metrics import precision_recall_curve, roc_curve @@ -17,6 +14,7 @@ from .caching import CACHE_DIR_DASHBOARD from .dash_config import DASH_CACHING +from ...setup import SERVER_BASE_URL TIMEOUT = 5 * 60 if DASH_CACHING else 0 @@ -95,10 +93,7 @@ def pr_chart(pathname, rows): if "Classification" not in task_type: return "Only classification supported", "Only classification supported" pred_id = df[df["evaluations"] == "predictions"]["results"].values[0] - url = ( - "https://www.openml.org/data/download/{}".format(pred_id) - + "/predictions.arff" - ) + url = f"{SERVER_BASE_URL}/data/download/{pred_id}/predictions.arff" ftp_stream = urllib.request.urlopen(url) data, meta = arff.loadarff(io.StringIO(ftp_stream.read().decode("utf-8"))) df = pd.DataFrame(data) diff --git a/server/src/dashboard/study_callbacks.py b/server/src/dashboard/study_callbacks.py index 8e20e035..7cd2eedb 100644 --- a/server/src/dashboard/study_callbacks.py +++ b/server/src/dashboard/study_callbacks.py @@ -1,11 +1,10 @@ import re -import dash_core_components as dcc -import dash_html_components as html import numpy as np import openml import plotly.express as px import plotly.graph_objs as go +from dash import dcc, html from dash.dependencies import Input, Output from dash.exceptions import PreventUpdate diff --git a/server/src/dashboard/suite_callbacks.py b/server/src/dashboard/suite_callbacks.py index 11d00b58..f215c97b 100644 --- a/server/src/dashboard/suite_callbacks.py +++ b/server/src/dashboard/suite_callbacks.py @@ -1,9 +1,8 @@ import re -import dash_core_components as dcc -import dash_html_components as html import openml import plotly.graph_objs as go +from dash import dcc, html from dash.dependencies import Input, Output diff --git a/server/src/dashboard/task_callbacks.py b/server/src/dashboard/task_callbacks.py index ce51d979..d8095773 100644 --- a/server/src/dashboard/task_callbacks.py +++ b/server/src/dashboard/task_callbacks.py @@ -1,19 +1,17 @@ import re -import dash_core_components as dcc -import dash_html_components as html -import dash_table as dt import pandas as pd import plotly.graph_objs as go - +from dash import dash_table as dt +from dash import dcc, html from dash.dependencies import Input, Output - from openml import evaluations from openml.extensions.sklearn import SklearnExtension from .caching import CACHE_DIR_DASHBOARD -from .helpers import get_highest_rank from .dash_config import DASH_CACHING +from .helpers import get_highest_rank +from ...setup import SERVER_BASE_URL font = [ "Nunito Sans", @@ -65,7 +63,7 @@ def update_task_plots(pathname, metric, n_clicks): if pathname is not None and "/dashboard/task" in pathname: task_id = int(re.search(r"task/(\d+)", pathname).group(1)) else: - return html.Div(), html.Div() + return html.Div(), html.Div(), html.Div() if n_clicks is None: n_clicks = 0 @@ -86,9 +84,9 @@ def update_task_plots(pathname, metric, n_clicks): ) if df_new.empty and df_old.empty: - return html.Div(), html.Div() - else: - df = df_old.append(df_new) + return html.Div(), html.Div(), html.Div() + + df = pd.concat([df_old, df_new], ignore_index=True) df.to_pickle(filename_cache) run_link = [] @@ -96,11 +94,11 @@ def update_task_plots(pathname, metric, n_clicks): truncated = [] # Plotly hack to add href to each data point for run_id in df["run_id"].values: - link = ' ' + link = f'' run_link.append(link) # Plotly hack to link flow names for flow_id in df["flow_id"].values: - link = '' + link = f'' tick_text.append(link) # Truncate flow names (50 chars) for flow in df["flow_name"].values: @@ -244,6 +242,7 @@ def update_task_plots(pathname, metric, n_clicks): ) dummy_fig = html.Div(dcc.Graph(figure=fig), style={"display": "none"}) eval_div = html.Div(dcc.Graph(figure=fig)) + return ( dummy_fig, eval_div, diff --git a/server/task/views.py b/server/task/views.py index 761a5229..3fb36a30 100644 --- a/server/task/views.py +++ b/server/task/views.py @@ -1,9 +1,9 @@ +import openml from flask import Blueprint, jsonify, request from flask_cors import CORS -from flask_jwt_extended import get_jwt_identity, jwt_required -from server.user.models import User -import openml -import os +from flask_jwt_extended import jwt_required + +from server.setup import setup_openml_config task_blueprint = Blueprint( "task", __name__, static_folder="server/src/client/app/build" @@ -12,20 +12,17 @@ CORS(task_blueprint) +@task_blueprint.before_request +def setup(): + setup_openml_config() + + @task_blueprint.route("/upload-task", methods=["POST"]) -@jwt_required +@jwt_required() def upload_task(): """ Function to upload task """ - current_user = get_jwt_identity() - user = User.query.filter_by(email=current_user).first() - user_api_key = user.session_hash - openml.config.apikey = user_api_key - # change line below in testing - testing = os.environ.get("TESTING") - if testing: - openml.config.start_using_configuration_for_example() data = request.get_json() tasktypes = openml.tasks.TaskTypeEnum t_type = data["task_type"] diff --git a/server/user/models.py b/server/user/models.py index f70c9c71..8de25367 100644 --- a/server/user/models.py +++ b/server/user/models.py @@ -10,35 +10,6 @@ class User(Base): __table__ = Base.metadata.tables["users"] __table_args__ = {"autoload": True} - # Attribute names to help out with functions - # id = Column(Integer, primary_key=True, unique=True) - # ip_address = Column(String(64)) - # username = Column(String(64), index=True, unique=True) - # email = Column(String(120), index=True, unique=True) - # password = Column(String(240)) - # activation_selector = Column(String(120))#Unique - # activation_code = Column(String(120)) - # forgotten_password_selector = Column(String(120))#Unique - # forgotten_password_code = Column(String(120)) - # forgotten_password_time = Column(String(120)) - # remember_selector = Column(String(120))#Unique - # remember_code = Column(String(120)) - # created_on = Column(String(120)) - # last_login = Column(String(120)) - # active = Column(String(120)) - # first_name = Column(String(120)) - # last_name = Column(String(120)) - # company = Column(String(120)) - # phone = Column(String(120)) - # country = Column(String(120)) - # image = Column(String(120)) - # bio = Column(String(240)) - # core = Column(String(240)) - # external_source = Column(String(120)) - # external_id = Column(String(120)) - # session_hash = Column(String(120))# session hash is API key - # password_hash = Column(String(120)) - def set_password(self, password): self.password = argon2.generate_password_hash(password) diff --git a/server/user/views.py b/server/user/views.py index d9309add..d0a77049 100644 --- a/server/user/views.py +++ b/server/user/views.py @@ -1,6 +1,7 @@ import datetime import hashlib import os +from distutils.util import strtobool from urllib.parse import parse_qs, urlparse from flask import Blueprint, jsonify, request, send_from_directory, abort, Response @@ -8,7 +9,7 @@ from flask_jwt_extended import ( create_access_token, get_jwt_identity, - get_raw_jwt, + get_jwt, jwt_required, ) from pathlib import Path @@ -23,16 +24,17 @@ "user", __name__, static_folder="server/src/client/app/build" ) + CORS(user_blueprint) -blacklist = set() +blocklist = set() -@jwt.token_in_blacklist_loader -def check_if_token_in_blacklist(decrypted_token): - """Checking if token is in blacklist token""" +@jwt.token_in_blocklist_loader +def check_if_token_in_blocklist(jwt_header, decrypted_token): + """Checking if token is in blocklist token""" jti = decrypted_token["jti"] - return jti in blacklist + return jti in blocklist @user_blueprint.route("/login", methods=["POST"]) @@ -68,7 +70,7 @@ def login(): db.session.add(user_) db.session.commit() access_token = create_access_token(identity=user.email) - testing = os.environ.get("TESTING") + testing = strtobool(os.environ.get("TESTING", "True")) print(testing) if testing: print("executed") @@ -80,7 +82,7 @@ def login(): @user_blueprint.route("/profile", methods=["GET", "POST"]) -@jwt_required +@jwt_required() def profile(): """ Function to edit and retrieve user profile information @@ -124,15 +126,15 @@ def profile(): return jsonify({"msg": "profile OK"}), 200 +@jwt_required() @user_blueprint.route("/verifytoken", methods=["GET"]) -@jwt_required def verifytoken(): return "token-valid" # TODO Change Address before production @user_blueprint.route("/image", methods=["POST"]) -@jwt_required +@jwt_required() def image(): """Function to receive and set user image""" current_user = get_jwt_identity() @@ -165,7 +167,7 @@ def images(path): # @user_blueprint.route('/send-image', methods=['GET']) -# @jwt_required +# @jwt_required() # def send_image(): # current_user = get_jwt_identity() # user = User.query.filter_by(email=current_user).first() @@ -174,16 +176,16 @@ def images(path): @user_blueprint.route("/logout", methods=["POST"]) -@jwt_required +@jwt_required() def logout(): """Function to logout user""" - jti = get_raw_jwt()["jti"] - blacklist.add(jti) + jti = get_jwt()["jti"] + blocklist.add(jti) return jsonify({"msg": "Successfully logged out"}), 200 @user_blueprint.route("/api-key", methods=["POST", "GET"]) -@jwt_required +@jwt_required() def apikey(): """Change and retrieve API-Key""" current_user = get_jwt_identity() @@ -199,7 +201,7 @@ def apikey(): @user_blueprint.route("/delete", methods=["GET", "POST"]) -@jwt_required +@jwt_required() def delete_user(): """Delete current user: Frontend and functionality not decided yet""" # current_user = get_jwt_identity() diff --git a/server/utils.py b/server/utils.py index cf2bf15a..b9d58fae 100644 --- a/server/utils.py +++ b/server/utils.py @@ -2,6 +2,10 @@ import ssl import os +from flask_jwt_extended import get_jwt_identity, verify_jwt_in_request + +from server.user.models import User + context = ssl.create_default_context() @@ -68,3 +72,9 @@ def send_feedback(email, feedback): server.sendmail(sender, receiver, message) print("Email sent") server.quit() + + +def current_user() -> User | None: + if verify_jwt_in_request(): + jwt_identity = get_jwt_identity() + return User.query.filter_by(email=jwt_identity).first() diff --git a/tests/test_collection_routes.py b/tests/test_collection_routes.py index 9b638886..900632fe 100644 --- a/tests/test_collection_routes.py +++ b/tests/test_collection_routes.py @@ -2,7 +2,7 @@ def test_upload_collection_runs(test_client, init_database): - response = test_client.post( + test_client.post( "/login", json={"email": "ff@ff.com", "password": "ff"}, follow_redirects=True ) access_token = str(os.environ.get("TEST_ACCESS_TOKEN"))