Updated python dependencies & configurable docker builds

.dockerignore

@@ -17,7 +17,6 @@ build
 .env.development.local
 .env.test.local
 .env.production.local
-.env
 
 npm-debug.log*
 yarn-debug.log*
@@ -62,3 +61,4 @@ venv
 .git
 .github
 
+openml.db

.env

@@ -0,0 +1,23 @@
+FLASK_APP=autoapp.py
+FLASK_ENV=development
+SMTP_SERVER=smtp.mailtrap.io
+SMTP_PORT=2525
+DATABASE_URI=sqlite:///openml.db
+EMAIL_SENDER=m@mailtrao.io
+SMTP_LOGIN=
+SMTP_PASS=
+APP_SECRET_KEY=abcd
+JWT_SECRET_KEY=abcd
+TESTING=True
+
+URL_API=https://www.openml.org/
+
+# REACT
+# React env variables are fixed when building the app.
+# React env variables need to be prefixed with "REACT_APP", otherwise the variables are ignored for
+# security reasons.
+REACT_APP_URL_SITE_BACKEND=https://localhost:5000/
+REACT_APP_URL_API=https://www.openml.org/
+REACT_APP_URL_ELASTICSEARCH=https://www.openml.org/es/
+REACT_APP_ELASTICSEARCH_VERSION_MAYOR=6
+REACT_APP_URL_MINIO=https://openml1.win.tue.nl/

.env.k8s

@@ -0,0 +1,23 @@
+FLASK_APP=autoapp.py
+FLASK_ENV=development
+SMTP_SERVER=smtp.mailtrap.io
+SMTP_PORT=2525
+DATABASE_URI=sqlite:///openml.db
+EMAIL_SENDER=m@mailtrao.io
+SMTP_LOGIN=
+SMTP_PASS=
+APP_SECRET_KEY=abcd
+JWT_SECRET_KEY=abcd
+TESTING=True
+
+URL_API=https://k8s.openml.org/
+
+# REACT
+# React env variables are fixed when building the app.
+# React env variables need to be prefixed with "REACT_APP", otherwise the variables are ignored for
+# security reasons.
+REACT_APP_URL_SITE_BACKEND=https://localhost:5000/
+REACT_APP_URL_API=https://k8sapi.openml.org/
+REACT_APP_URL_ELASTICSEARCH=https://k8s.openml.org/es/
+REACT_APP_ELASTICSEARCH_VERSION_MAYOR=8
+REACT_APP_URL_MINIO=https://openml1.win.tue.nl/

.gitignore

@@ -17,7 +17,6 @@ build
 .env.development.local
 .env.test.local
 .env.production.local
-.env
 
 npm-debug.log*
 yarn-debug.log*
@@ -57,3 +56,6 @@ users.sql
 node_modules
 node_modules.nosync
 venv
+
+
+openml.db

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim-bookworm
+FROM python:3.11-slim-bookworm
 
 RUN apt update && apt upgrade -y
 RUN python -m pip install --upgrade pip

docker/README.md

@@ -3,18 +3,34 @@
 ## Usage
 
 ```bash
-docker run --rm -d -p 5000:5000 --name openml-frontend openml/frontend
+docker run --rm -d -p 5000:5000 --name openml-frontend --env-file .env openml/frontend:current
 docker kill openml-frontend  # the container doesn't seem to respond to a friendly stop request
 ```
 
-To configure, you can add environment variables. See .flaskenv for the envirnoment variables. To add 
-the environment variables to docker, you can use `--env-file`:
+For the k8s deployment, use:
 ```bash
-docker run --rm -d -p 5000:5000 --name openml-frontend --env-file .env openml/frontend
+docker run --rm -d -p 5000:5000 --name openml-frontend --env-file .env.k8s openml/frontend:k8s
+docker kill openml-frontend  # the container doesn't seem to respond to a friendly stop request
 ```
 
 ## Build and publish
+
+Currently, you need to manually run the `npm run build` before a docker build. We should put 
+that step inside the docker build step. I didn't do that yet, because this is a minor 
+inconvenience, and I think we might want to refactor the build/deployment process a bit anyway 
+(do we want to separate the frontend and this backend into separate containers? Do we want to 
+merge the backend and the api-backend-server?).
+
+For current production:
 ```bash
+./server/src/client/app/node_modules/.bin/env-cmd -f ./.env npm run build --prefix server/src/client/app/
 docker build -f docker/Dockerfile --tag openml/frontend:latest .
 docker push openml/frontend:latest
 ```
+
+For k8s:
+```bash
+./server/src/client/app/node_modules/.bin/env-cmd -f ./.env.k8s npm run build --prefix server/src/client/app/
+docker build -f docker/Dockerfile --tag openml/frontend:k8s .
+docker push openml/frontend:k8s
+```

requirements.txt

@@ -1,105 +1,41 @@
-alembic==1.3.0
-argon2-cffi==19.2.0
-arrow==0.15.4
-attrs==19.3.0
-Babel==2.9.1
-bcrypt==3.1.7
-binaryornot==0.4.4
-blinker==1.4
-category-encoders==2.0.0
-certifi==2022.12.7
-cffi
-chardet==3.0.4
-Click==7.0
-configparser==4.0.2
-coverage
-cryptography==3.3.2
-cycler==0.10.0
-dash==1.3.0
-dash-core-components==1.2.0
-dash-html-components==1.0.1
-dash-renderer==1.1.0
-dash-table==4.3.0
-environs==7.1.0
-Flask==1.0.2
-Flask-Argon2==0.1.5.1
-Flask-BabelEx==0.9.3
-Flask-Bcrypt==0.7.1
-Flask-Caching==1.8.0
-Flask-Compress==1.4.0
-Flask-Cors==3.0.9
-Flask-Dance==3.0.0
-Flask-DebugToolbar==0.10.1
-Flask-JWT-Extended==3.24.1
-Flask-Login==0.4.1
+category-encoders==2.6.3
+dash==2.14.2
+dash-core-components==2.0.0
+dash-html-components==2.0.0
+dash-renderer==1.9.1
+dash-table==5.0.0
+environs==10.0.0
+Flask==3.0.0
+Flask-Argon2==0.3.0.0
+Flask-BabelEx==0.9.4
+Flask-Bcrypt==1.0.1
+Flask-Caching==2.1.0
+Flask-Compress==1.14
+Flask-Cors==4.0.0
+Flask-Dance==7.0.0
+Flask-DebugToolbar==0.14.1
+Flask-JWT-Extended==4.6.0
+Flask-Login==0.6.3
 Flask-Mail==0.9.1
-Flask-Migrate==2.5.2
+Flask-Migrate==4.0.5
 Flask-Principal==0.4.0
 Flask-Security==3.0.0
-Flask-SQLAlchemy==2.4.1
-Flask-User==1.0.2.1
-Flask-WTF==0.14.2
-future==0.17.1
-gunicorn==20.0.4
-idna==2.8
-importlib-metadata
-isort==4.3.21
-itsdangerous==1.1.0
-Jinja2==2.11.3
-jinja2-time==0.2.0
-joblib==1.2.0
-kiwisolver==1.1.0
-liac-arff==2.4.0
-lxml==4.9.1
-Mako==1.2.2
-MarkupSafe==1.1.1
-marshmallow==3.3.0
-matplotlib
-more-itertools==8.0.2
+Flask-SQLAlchemy==3.1.1
+Flask-User==1.0.2.2
+Flask-WTF==1.2.1
+gunicorn==21.2.0
 names==0.3.0
-numpy
-oauthlib==3.1.0
-packaging==20.1
-pandas
-passlib==1.7.2
-patsy==0.5.1
-percy==2.0.2
-plotly==4.1.1
-pluggy==0.13.1
-poyo==0.5.0
-py==1.10.0
-pycparser==2.19
-PyJWT==2.4.0
-PyMySQL==0.9.3
-pyOpenSSL==19.0.0
-pyparsing==2.4.6
-pytest==5.3.4
-pytest-flask==0.15.1
-pytest-mock==2.0.0
-pytest-sugar==0.9.2
-python-dateutil==2.8.0
-python-dotenv==0.10.3
-python-editor==1.0.4
-pytz==2019.2
-requests>=2.22.0
-requests-oauthlib==1.3.0
-retrying==1.3.3
-scikit-learn
-scipy
-seaborn==0.9.0
-selenium==3.141.0
-six==1.14.0
-SQLAlchemy==1.3.8
-SQLAlchemy-Utils==0.36.1
-termcolor==1.1.0
-urllib3==1.26.5
-URLObject==2.4.3
-wcwidth==0.1.7
-Werkzeug==0.16.0
-whichcraft==0.6.1
-wincertstore==0.2
-WTForms==2.2.1
-xmltodict==0.12.0
-zipp
-Pillow
-openml
+numpy==1.26.2
+pandas==2.1.4
+pillow==10.2.0
+plotly==5.18.0
+pymysql==1.1.0
+pytest==7.4.4
+pytest-flask==1.3.0
+pytest-mock==3.12.0
+pytest-sugar==0.9.7
+python-dotenv==1.0.0
+scikit-learn==1.3.2
+scipy==1.11.4
+Werkzeug==3.0.1
+openml==0.14.1

server/app.py

@@ -4,9 +4,6 @@
 from .extensions import Base, argon2, bcrypt, db, engine, jwt
 from .src.dashboard.dashapp import create_dash_app
 
-# from flask_cors import CORS
-# from flask_dance.contrib.github import make_github_blueprint, github
-
 
 def register_extensions(app):
     """Registering extensions for flask app
@@ -19,10 +16,8 @@ def register_extensions(app):
     jwt.init_app(app)
     bcrypt.init_app(app)
 
-    # Database initialisation
-    db_session = scoped_session(
-        sessionmaker(autocommit=True, autoflush=False, bind=engine)
-    )
+    # Initialization, see Flask Security
+    db_session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine))
     Base.query = db_session.query_property()
     db.init_app(app)
     with app.app_context():

server/collection/views.py

@@ -1,10 +1,11 @@
+import uuid
+
+import openml
 from flask import Blueprint, jsonify, request
 from flask_cors import CORS
-from flask_jwt_extended import get_jwt_identity, jwt_required
-from server.user.models import User
-import openml
-import uuid
-import os
+from flask_jwt_extended import jwt_required
+
+from server.setup import setup_openml_config
 
 collection_bp = Blueprint(
     "collection", __name__, static_folder="server/src/client/app/build"
@@ -13,21 +14,18 @@
 CORS(collection_bp)
 
 
+@collection_bp.before_request
+def setup():
+    setup_openml_config()
+
+
 @collection_bp.route("/upload-collection-runs", methods=["POST"])
-@jwt_required
+@jwt_required()
 def upload_collection_runs():
     """
     Function to upload the collection_runs
     returns: JSON response
     """
-    current_user = get_jwt_identity()
-    user = User.query.filter_by(email=current_user).first()
-    user_api_key = user.session_hash
-    openml.config.apikey = user_api_key
-    # TODO change line below in production
-    testing = os.environ.get("TESTING")
-    if testing:
-        openml.config.start_using_configuration_for_example()
     data = request.get_json()
     collection_name = data["collectionname"]
     description = data["description"]
@@ -48,20 +46,12 @@ def upload_collection_runs():
 
 
 @collection_bp.route("/upload-collection-tasks", methods=["POST"])
-@jwt_required
+@jwt_required()
 def upload_collection_task():
     """
     Function to upload the collection_tasks
     returns: JSON response
     """
-    current_user = get_jwt_identity()
-    user = User.query.filter_by(email=current_user).first()
-    user_api_key = user.session_hash
-    openml.config.apikey = user_api_key
-    # change line below in testing
-    testing = os.environ.get("TESTING")
-    if testing:
-        openml.config.start_using_configuration_for_example()
     data = request.get_json()
     collection_name = data["collectionname"]
     description = data["description"]