From 0845d0ed0b0bb866990cf82e2c8af6c073d2de21 Mon Sep 17 00:00:00 2001 From: Joost van der Gaag Date: Wed, 13 Jul 2022 13:36:32 +0200 Subject: [PATCH 01/10] Updated environment files and added template for react env file (for docker purposes) --- .flaskenv | 15 --------------- .gitignore | 5 ++++- TEMPLATE.reactenv | 7 +++++++ package-lock.json | 6 ++++++ 4 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 .flaskenv create mode 100644 TEMPLATE.reactenv create mode 100644 package-lock.json diff --git a/.flaskenv b/.flaskenv deleted file mode 100644 index 48d9643d..00000000 --- a/.flaskenv +++ /dev/null @@ -1,15 +0,0 @@ -FLASK_APP=autoapp.py -ELASTICSEARCH_SERVER=https://www.openml.org/es -OPENML_SERVER=https://www.openml.org -FLASK_ENV=development -SMTP_SERVER=smtp.mailtrap.io -SMTP_PORT=2525 -DATABASE_URI=sqlite:///openml.db -EMAIL_SENDER=m@mailtrao.io -SMTP_LOGIN= -SMTP_PASS= -APP_SECRET_KEY=abcd -JWT_SECRET_KEY=abcd -TESTING=True -SERVER_URL=https://localhost:5000/ -REDIRECT_URL=https://localhost:5000 diff --git a/.gitignore b/.gitignore index 0feddd1e..4bf9d95a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ .env.test.local .env.production.local .env +.reactenv +.reactenv_aws +.flaskenv npm-debug.log* yarn-debug.log* @@ -55,4 +58,4 @@ temp_data/ users.sql node_modules -node_modules.nosync +node_modules.nosync \ No newline at end of file diff --git a/TEMPLATE.reactenv b/TEMPLATE.reactenv new file mode 100644 index 00000000..c28bf33d --- /dev/null +++ b/TEMPLATE.reactenv @@ -0,0 +1,7 @@ +# URL that tells React where to find the Flask app. +# When running on a remote server, change it to the server URL +REACT_APP_SERVER_URL=http://localhost:5000/ +# URL used to connect to the PHP website for downloading data +REACT_APP_OLD_SERVER_URL=http://localhost/ +# Elastic Search URL +REACT_APP_ES_URL=http://localhost:9200/ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..c994d361 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "openml.org", + "lockfileVersion": 2, + "requires": true, + "packages": {} +} From 835a9ae95ea3123ea8c8e656a6daecf7979b2259 Mon Sep 17 00:00:00 2001 From: Joost van der Gaag Date: Wed, 13 Jul 2022 15:31:12 +0200 Subject: [PATCH 02/10] Env updates and docker fixes Update baseurls (backend + ES) to use environment variables instead of hardcoded strings (defaults to OpenML configs) Added SEND_EMAIL Flask env variable to enable/disable sending emails for development puproses. Fixed signin button not working Updated flas_jwt_extended version to fix following bug: https://github.com/vimalloc/flask-jwt-extended/issues/377 --- .flaskenv_TEMPLATE | 1 + requirements.txt | 2 +- server/public/views.py | 17 ++++-- .../src/client/app/src/components/Sidebar.js | 56 ++++++++++--------- .../client/app/src/pages/auth/ProfilePage.js | 14 +++-- .../src/client/app/src/pages/auth/SignIn.js | 3 +- .../client/app/src/pages/search/Dataset.js | 6 +- server/src/client/app/src/pages/search/api.js | 16 +++--- 8 files changed, 66 insertions(+), 49 deletions(-) diff --git a/.flaskenv_TEMPLATE b/.flaskenv_TEMPLATE index 59c6c542..60188476 100644 --- a/.flaskenv_TEMPLATE +++ b/.flaskenv_TEMPLATE @@ -10,3 +10,4 @@ SMTP_PASS= APP_SECRET_KEY= EMAIL_SERVER=localhost:5000 REDIRECT_URL=https://localhost:5000 +SEND_EMAIL=False \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 46f37779..e00df734 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ Flask-Compress==1.4.0 Flask-Cors==3.0.9 Flask-Dance==3.0.0 Flask-DebugToolbar==0.10.1 -Flask-JWT-Extended==3.24.1 +Flask-JWT-Extended==3.35.0 Flask-Login==0.4.1 Flask-Mail==0.9.1 Flask-Migrate==2.5.2 diff --git a/server/public/views.py b/server/public/views.py index 3a4604f9..e98f22c6 100644 --- a/server/public/views.py +++ b/server/public/views.py @@ -1,5 +1,6 @@ import datetime import hashlib +import os from flask_cors import CORS from server.extensions import db @@ -16,7 +17,7 @@ blueprint = Blueprint("public", __name__) CORS(blueprint) - +do_send_email = os.environ.get("SEND_EMAIL", "True") == "True" @blueprint.route("/signup", methods=["POST"]) def signupfunc(): @@ -37,7 +38,10 @@ def signupfunc(): user.remember_code = "0000" user.created_on = "0000" user.last_login = "0000" - user.active = "0" + if do_send_email: + user.active = "0" + else: + user.active = "1" user.first_name = register_obj["first_name"] user.last_name = register_obj["last_name"] user.company = "0000" @@ -53,7 +57,8 @@ def signupfunc(): timestamp = timestamp.strftime("%d %H:%M:%S") md5_digest = hashlib.md5(timestamp.encode()).hexdigest() user.update_activation_code(md5_digest) - confirmation_email(user.email, md5_digest) + if do_send_email: + confirmation_email(user.email, md5_digest) db.session.add(user) # db.session.commit() # user_ = User.query.filter_by(email=register_obj["email"]).first() @@ -74,7 +79,8 @@ def password(): user = User.query.filter_by(email=jobj["email"]).first() user.update_forgotten_code(md5_digest) # user.update_forgotten_time(timestamp) - forgot_password_email(user.email, md5_digest) + if do_send_email: + forgot_password_email(user.email, md5_digest) db.session.merge(user) db.session.commit() return jsonify({"msg": "Token sent"}), 200 @@ -89,7 +95,8 @@ def confirmation_token(): md5_digest = hashlib.md5(timestamp.encode()).hexdigest() user = User.query.filter_by(email=jobj["email"]).first() user.update_activation_code(md5_digest) - confirmation_email(user.email, md5_digest) + if do_send_email: + confirmation_email(user.email, md5_digest) # updating user groups here user_ = UserGroups(user_id=user.id, group_id=2) db.session.merge(user) diff --git a/server/src/client/app/src/components/Sidebar.js b/server/src/client/app/src/components/Sidebar.js index b7d74c07..754720a5 100755 --- a/server/src/client/app/src/components/Sidebar.js +++ b/server/src/client/app/src/components/Sidebar.js @@ -79,8 +79,8 @@ const Brand = styled(ListItem)` props.searchcolor && props.currenttheme === 1 ? props.searchcolor : props.searchcolor - ? props.theme.sidebar.background - : props.theme.sidebar.header.background}; + ? props.theme.sidebar.background + : props.theme.sidebar.header.background}; padding-left: ${props => props.theme.spacing(3)}; font-size: 13pt; height: 56px; @@ -290,12 +290,16 @@ class Sidebar extends React.Component { axiosCancelToken = axios.CancelToken.source(); countUpdate = async () => { - const ELASTICSEARCH_SERVER = "https://www.openml.org/es/"; + const ELASTICSEARCH_SERVER = process.env.REACT_APP_ES_URL || "https://www.openml.org/es/"; const data = { size: 0, - query: { bool: { should: [ { term: { status: "active" } }, - { bool: { must_not: { exists: { field: "status" } } } } ] } }, + query: { + bool: { + should: [{ term: { status: "active" } }, + { bool: { must_not: { exists: { field: "status" } } } }] + } + }, aggs: { count_by_type: { terms: { field: "_type", size: 100 } } } }; @@ -318,11 +322,11 @@ class Sidebar extends React.Component { .catch(error => { console.log(error); }); - + // second query for benchmark counts const bench_data = { size: 0, - query: { bool : { filter : { bool: { should: [ {"wildcard": { "name": "*benchmark*" }}, {"wildcard": { "name": "*suite*" }}] } }}} + query: { bool: { filter: { bool: { should: [{ "wildcard": { "name": "*benchmark*" } }, { "wildcard": { "name": "*suite*" } }] } } } } }; axios .post(ELASTICSEARCH_SERVER + "study/study/_search", bench_data, headers) @@ -428,7 +432,7 @@ class Sidebar extends React.Component { currentcolor={context.getColor()} badge={ context.type === undefined && - this.state.counts[category.entity_type] + this.state.counts[category.entity_type] ? this.state.counts[category.entity_type] : 0 } @@ -467,24 +471,24 @@ class Sidebar extends React.Component { component={NavLink} searchExpand={ category.entity_type === context.type && - context.searchCollapsed + context.searchCollapsed ? () => context.collapseSearch(false) : undefined } badge={ category.entity_type === context.type ? (context.filters.measure_type && - route.subtype.split("_")[1] === - context.filters.measure_type - .value) || + route.subtype.split("_")[1] === + context.filters.measure_type + .value) || (context.filters.study_type && route.subtype === - context.filters.study_type.value) // Only show subtype counts if a subtype is selected + context.filters.study_type.value) // Only show subtype counts if a subtype is selected ? context.counts : 0 - : this.state.counts[category.entity_type] - ? this.state.counts[category.entity_type] - : 0 + : this.state.counts[category.entity_type] + ? this.state.counts[category.entity_type] + : 0 } /> ))} @@ -513,20 +517,20 @@ class Sidebar extends React.Component { : this.state.counts[category.entity_type] : context.type === undefined && this.state.counts[category.entity_type] - ? this.state.counts[category.entity_type] - : 0 + ? this.state.counts[category.entity_type] + : 0 } activecategory={ (location.pathname !== "/search" && location.pathname === category.path) || - (category.entity_type === context.type && - context.type !== undefined) + (category.entity_type === context.type && + context.type !== undefined) ? "true" : "false" } searchExpand={ category.entity_type === context.type && - context.searchCollapsed + context.searchCollapsed ? () => context.collapseSearch(false) : undefined } @@ -557,7 +561,7 @@ class Sidebar extends React.Component { } searchExpand={ category.entity_type === context.type && - context.searchCollapsed + context.searchCollapsed ? context.collapseSearch : undefined } @@ -579,7 +583,7 @@ class Sidebar extends React.Component {     @@ -156,18 +156,22 @@ function Public() { function Settings() { return ( - - Profile - - - - - - - - {/**/} +
+ + Profile + + + + + + + + {/**/} + - +
); } diff --git a/server/src/client/app/src/pages/search/Dataset.js b/server/src/client/app/src/pages/search/Dataset.js index e7335f3a..77cadc53 100644 --- a/server/src/client/app/src/pages/search/Dataset.js +++ b/server/src/client/app/src/pages/search/Dataset.js @@ -34,7 +34,7 @@ const Action = styled.div` justify-content: center; `; -const OLD_SERVER_BASE_URL = process.env.REACT_APP_OLD_SERVER_URL || "https://www.openml.org/"; +const SERVER_URL = process.env.REACT_APP_OLD_SERVER_URL || "https://www.openml.org/"; export class DatasetItem extends React.Component { constructor() { @@ -58,7 +58,7 @@ export class DatasetItem extends React.Component { {context => ( - + xml @@ -66,7 +66,7 @@ export class DatasetItem extends React.Component { - + json From d9f7fcd31be65a0b94e6e3f14f63976176fd1de0 Mon Sep 17 00:00:00 2001 From: Joost van der Gaag Date: Fri, 15 Jul 2022 11:38:37 +0200 Subject: [PATCH 05/10] Fix data upload not working (pushing to test server) --- .flaskenv_TEMPLATE | 9 +++- server/data/views.py | 13 ++++-- .../client/app/src/pages/auth/DataUpload.js | 41 +++++++++---------- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/.flaskenv_TEMPLATE b/.flaskenv_TEMPLATE index 60188476..7fd5487e 100644 --- a/.flaskenv_TEMPLATE +++ b/.flaskenv_TEMPLATE @@ -5,9 +5,14 @@ FLASK_ENV=development DATABASE_URI=mysql+pymysql://root:@localhost/openml SMTP_SERVER=smtp.mailtrap.io SMTP_PORT=2525 +EMAIL_SENDER= SMTP_LOGIN= SMTP_PASS= -APP_SECRET_KEY= +APP_SECRET_KEY=abcd +JWT_SECRET_KEY=abcd EMAIL_SERVER=localhost:5000 +SERVER_URL=https://localhost:5000 REDIRECT_URL=https://localhost:5000 -SEND_EMAIL=False \ No newline at end of file +PYTHON_SERVER=http://website/api/v1/xml/ +SEND_EMAIL=False +TESTING=True diff --git a/server/data/views.py b/server/data/views.py index 94fe29b7..238b8f49 100644 --- a/server/data/views.py +++ b/server/data/views.py @@ -24,7 +24,8 @@ def data_edit(): current_user = get_jwt_identity() user = User.query.filter_by(email=current_user).first() openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") + openml.config.server = os.getenv('PYTHON_SERVER') + testing = os.environ.get("TESTING", "False") == "True" if testing: openml.config.start_using_configuration_for_example() url = request.args.get("url") @@ -155,10 +156,15 @@ def data_upload(): user = User.query.filter_by(email=current_user).first() user_api_key = user.session_hash openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") + openml.config.server = os.getenv('PYTHON_SERVER') + testing = os.environ.get("TESTING", "False") == "True" + print(testing) if testing: + print("Set testing server") openml.config.start_using_configuration_for_example() # openml.config.start_using_configuration_for_example() + # else: + # openml.config.stop_using_configuration_for_example() print(request) data_file = request.files["dataset"] @@ -244,7 +250,8 @@ def data_tag(): current_user = get_jwt_identity() user = User.query.filter_by(email=current_user).first() openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") + openml.config.server = os.getenv('PYTHON_SERVER') + testing = os.environ.get("TESTING", "False") == "True" if testing: openml.config.start_using_configuration_for_example() url = request.args.get("url") diff --git a/server/src/client/app/src/pages/auth/DataUpload.js b/server/src/client/app/src/pages/auth/DataUpload.js index 46cd9298..0abe7dc7 100644 --- a/server/src/client/app/src/pages/auth/DataUpload.js +++ b/server/src/client/app/src/pages/auth/DataUpload.js @@ -64,14 +64,14 @@ function Public() { const onDrop = useCallback(acceptedFiles => { console.log(acceptedFiles); }, []); - + const maxSize = 1000000000; const { getRootProps, getInputProps, acceptedFiles } = useDropzone({ onDrop, minSize: 0, maxSize: maxSize, }); - + const handleChange = event => { setLicence(event.target.value); }; @@ -96,7 +96,6 @@ function Public() { function datatoflask(event) { event.preventDefault(); - setError(false); const obj = { dataset_name: event.target.datasetname.value, @@ -104,7 +103,7 @@ function Public() { creator: event.target.creator.value, contributor: event.target.contributor.value, collection_date: event.target.collection_date.value, - licence: event.target.licence.value, + licence: licence, language: event.target.language.value, // attribute: event.target.attribute.value, ignore_attribute: event.target.ignore_attribute.value, @@ -129,7 +128,7 @@ function Public() { data, yourConfig ) - .then(function(response) { + .then(function (response) { if (response.data.msg === "dataset uploaded") { setSuccess(true); } @@ -148,7 +147,7 @@ function Public() { data, yourConfig ) - .then(function(response) { + .then(function (response) { setEditPath(response.data.msg); console.log(response.data.msg); setEditSuccess(true); @@ -188,21 +187,21 @@ function Public() { )} - - -
- - - - - - Drag 'n' drop some files here, or click to select files. - - - Currently we only support text based formats like csv and json - -
- + + +
+ + + + + + Drag 'n' drop some files here, or click to select files. + + + Currently we only support text based formats like csv and json + +
+ {acceptedFiles.length > 0 && acceptedFiles.map(acceptedFile => ( ))} From 1f15d03116f8c5676d1694c6704af8305ee9b855 Mon Sep 17 00:00:00 2001 From: Joost van der Gaag Date: Tue, 19 Jul 2022 10:27:58 +0200 Subject: [PATCH 06/10] Update naming of environment variables to make them less ambiguous --- .flaskenv_TEMPLATE | 2 +- server/data/views.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.flaskenv_TEMPLATE b/.flaskenv_TEMPLATE index 7fd5487e..3adf155c 100644 --- a/.flaskenv_TEMPLATE +++ b/.flaskenv_TEMPLATE @@ -13,6 +13,6 @@ JWT_SECRET_KEY=abcd EMAIL_SERVER=localhost:5000 SERVER_URL=https://localhost:5000 REDIRECT_URL=https://localhost:5000 -PYTHON_SERVER=http://website/api/v1/xml/ +BACKEND_SERVER=http://website/api/v1/xml/ SEND_EMAIL=False TESTING=True diff --git a/server/data/views.py b/server/data/views.py index 238b8f49..18d96844 100644 --- a/server/data/views.py +++ b/server/data/views.py @@ -24,7 +24,7 @@ def data_edit(): current_user = get_jwt_identity() user = User.query.filter_by(email=current_user).first() openml.config.apikey = user.session_hash - openml.config.server = os.getenv('PYTHON_SERVER') + openml.config.server = os.getenv('BACKEND_SERVER') testing = os.environ.get("TESTING", "False") == "True" if testing: openml.config.start_using_configuration_for_example() @@ -156,7 +156,7 @@ def data_upload(): user = User.query.filter_by(email=current_user).first() user_api_key = user.session_hash openml.config.apikey = user.session_hash - openml.config.server = os.getenv('PYTHON_SERVER') + openml.config.server = os.getenv('BACKEND_SERVER') testing = os.environ.get("TESTING", "False") == "True" print(testing) if testing: @@ -250,7 +250,7 @@ def data_tag(): current_user = get_jwt_identity() user = User.query.filter_by(email=current_user).first() openml.config.apikey = user.session_hash - openml.config.server = os.getenv('PYTHON_SERVER') + openml.config.server = os.getenv('BACKEND_SERVER') testing = os.environ.get("TESTING", "False") == "True" if testing: openml.config.start_using_configuration_for_example() From d039a7a6ff6ac6a7dbfda76da5c2435161d20272 Mon Sep 17 00:00:00 2001 From: Joost van der Gaag Date: Tue, 19 Jul 2022 11:34:52 +0200 Subject: [PATCH 07/10] Use server URL for Base URL for dashboard files (make it configurable) --- .flaskenv_TEMPLATE | 1 + server/src/dashboard/flow_callbacks.py | 8 ++++++-- server/src/dashboard/helpers.py | 3 +++ server/src/dashboard/layouts.py | 3 +++ server/src/dashboard/overviews.py | 3 +++ server/src/dashboard/run_callbacks.py | 4 +++- server/src/dashboard/task_callbacks.py | 13 +++++++++---- server/src/dashboard/tests/test_flows.py | 4 ++++ server/src/dashboard/tests/test_runs.py | 3 +++ server/src/dashboard/tests/test_tasks.py | 3 +++ 10 files changed, 38 insertions(+), 7 deletions(-) diff --git a/.flaskenv_TEMPLATE b/.flaskenv_TEMPLATE index 3adf155c..9d1d4c3a 100644 --- a/.flaskenv_TEMPLATE +++ b/.flaskenv_TEMPLATE @@ -16,3 +16,4 @@ REDIRECT_URL=https://localhost:5000 BACKEND_SERVER=http://website/api/v1/xml/ SEND_EMAIL=False TESTING=True +BACKEND_BASE_URL=http://website/ diff --git a/server/src/dashboard/flow_callbacks.py b/server/src/dashboard/flow_callbacks.py index dbf34c8e..43b62e6b 100644 --- a/server/src/dashboard/flow_callbacks.py +++ b/server/src/dashboard/flow_callbacks.py @@ -1,8 +1,10 @@ import re +import os import pandas as pd import plotly.graph_objs as go from dash.dependencies import Input, Output +import openml from openml import evaluations, tasks from .dash_config import DASH_CACHING @@ -10,6 +12,8 @@ TIMEOUT = 5 * 60 if DASH_CACHING else 0 +SERVER_BASE_URL = os.getenv('BACKEND_BASE_URL', "https://www.openml.org/") +openml.config.server = os.getenv('BACKEND_SERVER') def register_flow_callbacks(app, cache): @app.callback( @@ -84,11 +88,11 @@ def update_flow_plots(pathname, metric, tasktype, parameter): tick_text = [] # Set clickable labels for run_id in df["run_id"].values: - link = ' ' + link = ' ' run_link.append(link) for data_id in df["data_id"].values: - link = '' + link = '' tick_text.append(link) hover_text = [] if parameter == "None": diff --git a/server/src/dashboard/helpers.py b/server/src/dashboard/helpers.py index eca2ad14..5d39a002 100644 --- a/server/src/dashboard/helpers.py +++ b/server/src/dashboard/helpers.py @@ -1,3 +1,4 @@ +import os import logging import time from contextlib import contextmanager @@ -5,12 +6,14 @@ import numpy as np import pandas as pd import scipy.stats +import openml from openml import datasets, runs from sklearn.model_selection import train_test_split logger = logging.getLogger("dashboard") logger.setLevel(logging.DEBUG) +openml.config.server = os.getenv('BACKEND_SERVER') def get_run_df(run_id: int): run = runs.get_run(int(run_id), ignore_cache=True) diff --git a/server/src/dashboard/layouts.py b/server/src/dashboard/layouts.py index 959ccb3d..b2e635b0 100644 --- a/server/src/dashboard/layouts.py +++ b/server/src/dashboard/layouts.py @@ -4,10 +4,13 @@ import dash_core_components as dcc import dash_html_components as html import dash_table as dt +import openml from openml import datasets, evaluations, runs, setups, study from .helpers import get_metadata, get_run_df, logger +openml.config.server = os.getenv('BACKEND_SERVER') + # TODO: Move to assets (Copied from Joaquin's react font) font = [ "Nunito Sans", diff --git a/server/src/dashboard/overviews.py b/server/src/dashboard/overviews.py index 169691dd..407110a3 100644 --- a/server/src/dashboard/overviews.py +++ b/server/src/dashboard/overviews.py @@ -1,14 +1,17 @@ +import os import dash_core_components as dcc import dash_html_components as html import pandas as pd import plotly.graph_objs as go from dash.dependencies import Input, Output +import openml from openml import datasets, flows, runs, tasks from openml.extensions.sklearn import SklearnExtension from .dash_config import DASH_CACHING TIMEOUT = 5 * 60 if DASH_CACHING else 0 +openml.config.server = os.getenv('BACKEND_SERVER') font = [ "Nunito Sans", diff --git a/server/src/dashboard/run_callbacks.py b/server/src/dashboard/run_callbacks.py index 78bf066f..1f8ca41a 100644 --- a/server/src/dashboard/run_callbacks.py +++ b/server/src/dashboard/run_callbacks.py @@ -1,5 +1,6 @@ import io import re +import os # import arff import urllib.request @@ -19,6 +20,7 @@ TIMEOUT = 5 * 60 if DASH_CACHING else 0 +SERVER_BASE_URL = os.getenv('BACKEND_BASE_URL', "https://www.openml.org/") def register_run_callbacks(app, cache): @app.callback( @@ -94,7 +96,7 @@ def pr_chart(pathname, rows): return "Only classification supported", "Only classification supported" pred_id = df[df["evaluations"] == "predictions"]["results"].values[0] url = ( - "https://www.openml.org/data/download/{}".format(pred_id) + SERVER_BASE_URL + "data/download/{}".format(pred_id) + "/predictions.arff" ) ftp_stream = urllib.request.urlopen(url) diff --git a/server/src/dashboard/task_callbacks.py b/server/src/dashboard/task_callbacks.py index 289e759f..1940846c 100644 --- a/server/src/dashboard/task_callbacks.py +++ b/server/src/dashboard/task_callbacks.py @@ -1,4 +1,5 @@ import re +import os import dash_core_components as dcc import dash_html_components as html @@ -8,12 +9,15 @@ from dash.dependencies import Input, Output +import openml from openml import evaluations from openml.extensions.sklearn import SklearnExtension from .helpers import get_highest_rank from .dash_config import DASH_CACHING +openml.config.server = os.getenv('BACKEND_SERVER') + font = [ "Nunito Sans", "-apple-system", @@ -28,6 +32,7 @@ "Segoe UI Symbol", ] +SERVER_BASE_URL = os.getenv('BACKEND_BASE_URL', "https://www.openml.org/") TIMEOUT = 5 * 60 if DASH_CACHING else 0 @@ -97,11 +102,11 @@ def update_task_plots(pathname, metric, n_clicks): truncated = [] # Plotly hack to add href to each data point for run_id in df["run_id"].values: - link = ' ' + link = ' ' run_link.append(link) # Plotly hack to link flow names for flow_id in df["flow_id"].values: - link = '' + link = '' tick_text.append(link) # Truncate flow names (50 chars) for flow in df["flow_name"].values: @@ -153,11 +158,11 @@ def update_task_plots(pathname, metric, n_clicks): tick_text = [] run_link = [] for run_id in df["run_id"].values: - link = ' ' + link = ' ' run_link.append(link) for flow_id in df["flow_id"].values: - link = '' + link = '' tick_text.append(link) df["upload_time"] = pd.to_datetime(df["upload_time"]) diff --git a/server/src/dashboard/tests/test_flows.py b/server/src/dashboard/tests/test_flows.py index f9d42cae..9fec27af 100644 --- a/server/src/dashboard/tests/test_flows.py +++ b/server/src/dashboard/tests/test_flows.py @@ -1,9 +1,13 @@ import time +import os +import openml from openml import evaluations from server.src.dashboard.dash_config import BASE_URL +openml.config.server = os.getenv('BACKEND_SERVER') + # def test_flow_page_loading(dash_br): # dash_br.server_url = BASE_URL + 'flow/405' # time.sleep(10) diff --git a/server/src/dashboard/tests/test_runs.py b/server/src/dashboard/tests/test_runs.py index 6a50045e..593ad3b8 100644 --- a/server/src/dashboard/tests/test_runs.py +++ b/server/src/dashboard/tests/test_runs.py @@ -1,11 +1,14 @@ import time +import os import numpy as np import pandas as pd +import openml from openml import runs from server.src.dashboard.dash_config import BASE_URL +openml.config.server = os.getenv('BACKEND_SERVER') def uncommon_string(s1, s2): st1 = set(s1) diff --git a/server/src/dashboard/tests/test_tasks.py b/server/src/dashboard/tests/test_tasks.py index c87c5704..ac597e01 100644 --- a/server/src/dashboard/tests/test_tasks.py +++ b/server/src/dashboard/tests/test_tasks.py @@ -1,9 +1,12 @@ import time +import os +import openml from openml import evaluations from server.src.dashboard.dash_config import BASE_URL +openml.config.server = os.getenv('BACKEND_SERVER') def test_task_page_loading(dash_br): dash_br.server_url = f"{BASE_URL}task/2" From 1644cd08a0b96abe318a2bd250b986831d4cb317 Mon Sep 17 00:00:00 2001 From: Joost van der Gaag Date: Wed, 20 Jul 2022 13:49:41 +0200 Subject: [PATCH 08/10] included Docker settings for downloading datasets not working in Dashboard --- server/src/dashboard/helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/dashboard/helpers.py b/server/src/dashboard/helpers.py index 5d39a002..6f9d1585 100644 --- a/server/src/dashboard/helpers.py +++ b/server/src/dashboard/helpers.py @@ -99,6 +99,11 @@ def get_data_metadata(data_id): start = time.time() meta_features, data, _ = get_metadata(data_id) + # Test replacing the substring to when Docker is active + if (os.getenv('DASHBOARD_USE_DOCKER_CONTAINER_NAME', 'False') == "True"): + data.url = data.url.replace("localhost", os.getenv('DASHBOARD_PHP_CONTAINER_NAME', 'website')) + print(data) + x, y, categorical, attribute_names = data.get_data() df = pd.DataFrame(x, columns=attribute_names) From 2d6892e41fc96b096baddb442eabde821207afae Mon Sep 17 00:00:00 2001 From: Joost van der Gaag Date: Mon, 25 Jul 2022 09:29:11 +0200 Subject: [PATCH 09/10] Small commend update --- server/src/dashboard/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/dashboard/helpers.py b/server/src/dashboard/helpers.py index 6f9d1585..a071a397 100644 --- a/server/src/dashboard/helpers.py +++ b/server/src/dashboard/helpers.py @@ -99,7 +99,7 @@ def get_data_metadata(data_id): start = time.time() meta_features, data, _ = get_metadata(data_id) - # Test replacing the substring to when Docker is active + # Replacing the substring to when Docker is active if (os.getenv('DASHBOARD_USE_DOCKER_CONTAINER_NAME', 'False') == "True"): data.url = data.url.replace("localhost", os.getenv('DASHBOARD_PHP_CONTAINER_NAME', 'website')) print(data) From bc437b8cf1ab40633464ddb5cab43a575285ae1d Mon Sep 17 00:00:00 2001 From: ihsanullah2131 Date: Sun, 31 Jul 2022 15:52:24 +0200 Subject: [PATCH 10/10] Text added to use OpenML in research --- server/src/client/app/src/pages/docs/GetInvolved.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/client/app/src/pages/docs/GetInvolved.js b/server/src/client/app/src/pages/docs/GetInvolved.js index cde22958..f0fec42c 100644 --- a/server/src/client/app/src/pages/docs/GetInvolved.js +++ b/server/src/client/app/src/pages/docs/GetInvolved.js @@ -214,6 +214,12 @@ export default class GetInvolved extends React.Component { color={blue[500]} text="Share new interesting datasets, models, and experiments." /> + @@ -254,7 +260,7 @@ export default class GetInvolved extends React.Component { We want to empower people to change the world for the better. You can help by contributing useful datasets and machine learning pipelines, or by extending OpenML to make it more useful in - science and discovery. + science and discovery, or by using OpenML in your research projects.