diff --git a/.flaskenv b/.flaskenv deleted file mode 100644 index 48d9643d..00000000 --- a/.flaskenv +++ /dev/null @@ -1,15 +0,0 @@ -FLASK_APP=autoapp.py -ELASTICSEARCH_SERVER=https://www.openml.org/es -OPENML_SERVER=https://www.openml.org -FLASK_ENV=development -SMTP_SERVER=smtp.mailtrap.io -SMTP_PORT=2525 -DATABASE_URI=sqlite:///openml.db -EMAIL_SENDER=m@mailtrao.io -SMTP_LOGIN= -SMTP_PASS= -APP_SECRET_KEY=abcd -JWT_SECRET_KEY=abcd -TESTING=True -SERVER_URL=https://localhost:5000/ -REDIRECT_URL=https://localhost:5000 diff --git a/.flaskenv_TEMPLATE b/.flaskenv_TEMPLATE index 59c6c542..9d1d4c3a 100644 --- a/.flaskenv_TEMPLATE +++ b/.flaskenv_TEMPLATE @@ -5,8 +5,15 @@ FLASK_ENV=development DATABASE_URI=mysql+pymysql://root:@localhost/openml SMTP_SERVER=smtp.mailtrap.io SMTP_PORT=2525 +EMAIL_SENDER= SMTP_LOGIN= SMTP_PASS= -APP_SECRET_KEY= +APP_SECRET_KEY=abcd +JWT_SECRET_KEY=abcd EMAIL_SERVER=localhost:5000 +SERVER_URL=https://localhost:5000 REDIRECT_URL=https://localhost:5000 +BACKEND_SERVER=http://website/api/v1/xml/ +SEND_EMAIL=False +TESTING=True +BACKEND_BASE_URL=http://website/ diff --git a/.gitignore b/.gitignore index 0feddd1e..4bf9d95a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ .env.test.local .env.production.local .env +.reactenv +.reactenv_aws +.flaskenv npm-debug.log* yarn-debug.log* @@ -55,4 +58,4 @@ temp_data/ users.sql node_modules -node_modules.nosync +node_modules.nosync \ No newline at end of file diff --git a/TEMPLATE.reactenv b/TEMPLATE.reactenv new file mode 100644 index 00000000..c28bf33d --- /dev/null +++ b/TEMPLATE.reactenv @@ -0,0 +1,7 @@ +# URL that tells React where to find the Flask app. +# When running on a remote server, change it to the server URL +REACT_APP_SERVER_URL=http://localhost:5000/ +# URL used to connect to the PHP website for downloading data +REACT_APP_OLD_SERVER_URL=http://localhost/ +# Elastic Search URL +REACT_APP_ES_URL=http://localhost:9200/ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..c994d361 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "openml.org", + "lockfileVersion": 2, + "requires": true, + "packages": {} +} diff --git a/requirements.txt b/requirements.txt index 46f37779..2cdf89de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ Flask-Compress==1.4.0 Flask-Cors==3.0.9 Flask-Dance==3.0.0 Flask-DebugToolbar==0.10.1 -Flask-JWT-Extended==3.24.1 +Flask-JWT-Extended==3.25.1 Flask-Login==0.4.1 Flask-Mail==0.9.1 Flask-Migrate==2.5.2 @@ -69,7 +69,7 @@ pluggy==0.13.1 poyo==0.5.0 py==1.10.0 pycparser==2.19 -PyJWT==2.4.0 +PyJWT PyMySQL==0.9.3 pyOpenSSL==19.0.0 pyparsing==2.4.6 diff --git a/server/data/views.py b/server/data/views.py index 94fe29b7..18d96844 100644 --- a/server/data/views.py +++ b/server/data/views.py @@ -24,7 +24,8 @@ def data_edit(): current_user = get_jwt_identity() user = User.query.filter_by(email=current_user).first() openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") + openml.config.server = os.getenv('BACKEND_SERVER') + testing = os.environ.get("TESTING", "False") == "True" if testing: openml.config.start_using_configuration_for_example() url = request.args.get("url") @@ -155,10 +156,15 @@ def data_upload(): user = User.query.filter_by(email=current_user).first() user_api_key = user.session_hash openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") + openml.config.server = os.getenv('BACKEND_SERVER') + testing = os.environ.get("TESTING", "False") == "True" + print(testing) if testing: + print("Set testing server") openml.config.start_using_configuration_for_example() # openml.config.start_using_configuration_for_example() + # else: + # openml.config.stop_using_configuration_for_example() print(request) data_file = request.files["dataset"] @@ -244,7 +250,8 @@ def data_tag(): current_user = get_jwt_identity() user = User.query.filter_by(email=current_user).first() openml.config.apikey = user.session_hash - testing = os.environ.get("TESTING") + openml.config.server = os.getenv('BACKEND_SERVER') + testing = os.environ.get("TESTING", "False") == "True" if testing: openml.config.start_using_configuration_for_example() url = request.args.get("url") diff --git a/server/public/views.py b/server/public/views.py index 3a4604f9..e98f22c6 100644 --- a/server/public/views.py +++ b/server/public/views.py @@ -1,5 +1,6 @@ import datetime import hashlib +import os from flask_cors import CORS from server.extensions import db @@ -16,7 +17,7 @@ blueprint = Blueprint("public", __name__) CORS(blueprint) - +do_send_email = os.environ.get("SEND_EMAIL", "True") == "True" @blueprint.route("/signup", methods=["POST"]) def signupfunc(): @@ -37,7 +38,10 @@ def signupfunc(): user.remember_code = "0000" user.created_on = "0000" user.last_login = "0000" - user.active = "0" + if do_send_email: + user.active = "0" + else: + user.active = "1" user.first_name = register_obj["first_name"] user.last_name = register_obj["last_name"] user.company = "0000" @@ -53,7 +57,8 @@ def signupfunc(): timestamp = timestamp.strftime("%d %H:%M:%S") md5_digest = hashlib.md5(timestamp.encode()).hexdigest() user.update_activation_code(md5_digest) - confirmation_email(user.email, md5_digest) + if do_send_email: + confirmation_email(user.email, md5_digest) db.session.add(user) # db.session.commit() # user_ = User.query.filter_by(email=register_obj["email"]).first() @@ -74,7 +79,8 @@ def password(): user = User.query.filter_by(email=jobj["email"]).first() user.update_forgotten_code(md5_digest) # user.update_forgotten_time(timestamp) - forgot_password_email(user.email, md5_digest) + if do_send_email: + forgot_password_email(user.email, md5_digest) db.session.merge(user) db.session.commit() return jsonify({"msg": "Token sent"}), 200 @@ -89,7 +95,8 @@ def confirmation_token(): md5_digest = hashlib.md5(timestamp.encode()).hexdigest() user = User.query.filter_by(email=jobj["email"]).first() user.update_activation_code(md5_digest) - confirmation_email(user.email, md5_digest) + if do_send_email: + confirmation_email(user.email, md5_digest) # updating user groups here user_ = UserGroups(user_id=user.id, group_id=2) db.session.merge(user) diff --git a/server/src/client/app/src/components/Sidebar.js b/server/src/client/app/src/components/Sidebar.js index b7d74c07..754720a5 100755 --- a/server/src/client/app/src/components/Sidebar.js +++ b/server/src/client/app/src/components/Sidebar.js @@ -79,8 +79,8 @@ const Brand = styled(ListItem)` props.searchcolor && props.currenttheme === 1 ? props.searchcolor : props.searchcolor - ? props.theme.sidebar.background - : props.theme.sidebar.header.background}; + ? props.theme.sidebar.background + : props.theme.sidebar.header.background}; padding-left: ${props => props.theme.spacing(3)}; font-size: 13pt; height: 56px; @@ -290,12 +290,16 @@ class Sidebar extends React.Component { axiosCancelToken = axios.CancelToken.source(); countUpdate = async () => { - const ELASTICSEARCH_SERVER = "https://www.openml.org/es/"; + const ELASTICSEARCH_SERVER = process.env.REACT_APP_ES_URL || "https://www.openml.org/es/"; const data = { size: 0, - query: { bool: { should: [ { term: { status: "active" } }, - { bool: { must_not: { exists: { field: "status" } } } } ] } }, + query: { + bool: { + should: [{ term: { status: "active" } }, + { bool: { must_not: { exists: { field: "status" } } } }] + } + }, aggs: { count_by_type: { terms: { field: "_type", size: 100 } } } }; @@ -318,11 +322,11 @@ class Sidebar extends React.Component { .catch(error => { console.log(error); }); - + // second query for benchmark counts const bench_data = { size: 0, - query: { bool : { filter : { bool: { should: [ {"wildcard": { "name": "*benchmark*" }}, {"wildcard": { "name": "*suite*" }}] } }}} + query: { bool: { filter: { bool: { should: [{ "wildcard": { "name": "*benchmark*" } }, { "wildcard": { "name": "*suite*" } }] } } } } }; axios .post(ELASTICSEARCH_SERVER + "study/study/_search", bench_data, headers) @@ -428,7 +432,7 @@ class Sidebar extends React.Component { currentcolor={context.getColor()} badge={ context.type === undefined && - this.state.counts[category.entity_type] + this.state.counts[category.entity_type] ? this.state.counts[category.entity_type] : 0 } @@ -467,24 +471,24 @@ class Sidebar extends React.Component { component={NavLink} searchExpand={ category.entity_type === context.type && - context.searchCollapsed + context.searchCollapsed ? () => context.collapseSearch(false) : undefined } badge={ category.entity_type === context.type ? (context.filters.measure_type && - route.subtype.split("_")[1] === - context.filters.measure_type - .value) || + route.subtype.split("_")[1] === + context.filters.measure_type + .value) || (context.filters.study_type && route.subtype === - context.filters.study_type.value) // Only show subtype counts if a subtype is selected + context.filters.study_type.value) // Only show subtype counts if a subtype is selected ? context.counts : 0 - : this.state.counts[category.entity_type] - ? this.state.counts[category.entity_type] - : 0 + : this.state.counts[category.entity_type] + ? this.state.counts[category.entity_type] + : 0 } /> ))} @@ -513,20 +517,20 @@ class Sidebar extends React.Component { : this.state.counts[category.entity_type] : context.type === undefined && this.state.counts[category.entity_type] - ? this.state.counts[category.entity_type] - : 0 + ? this.state.counts[category.entity_type] + : 0 } activecategory={ (location.pathname !== "/search" && location.pathname === category.path) || - (category.entity_type === context.type && - context.type !== undefined) + (category.entity_type === context.type && + context.type !== undefined) ? "true" : "false" } searchExpand={ category.entity_type === context.type && - context.searchCollapsed + context.searchCollapsed ? () => context.collapseSearch(false) : undefined } @@ -557,7 +561,7 @@ class Sidebar extends React.Component { } searchExpand={ category.entity_type === context.type && - context.searchCollapsed + context.searchCollapsed ? context.collapseSearch : undefined } @@ -579,7 +583,7 @@ class Sidebar extends React.Component {     @@ -154,18 +156,22 @@ function Public() { function Settings() { return ( - - Profile - +
+ + Profile + - + - - - - {/**/} + + + + {/**/} + - +
); } diff --git a/server/src/client/app/src/pages/auth/SignIn.js b/server/src/client/app/src/pages/auth/SignIn.js index d407ec52..4a11775d 100755 --- a/server/src/client/app/src/pages/auth/SignIn.js +++ b/server/src/client/app/src/pages/auth/SignIn.js @@ -38,6 +38,8 @@ function SignIn() { const context = useContext(MainContext); function sendtoflask(event) { + console.log("Start signin") + console.log("Event", event) event.preventDefault(); axios .post(process.env.REACT_APP_SERVER_URL + "login", { @@ -143,7 +145,6 @@ function SignIn() { variant="contained" color="primary" mb={2} - to="/" style={{ marginTop: 20 }} > Sign in diff --git a/server/src/client/app/src/pages/docs/GetInvolved.js b/server/src/client/app/src/pages/docs/GetInvolved.js index 4b76e50a..e35a5274 100644 --- a/server/src/client/app/src/pages/docs/GetInvolved.js +++ b/server/src/client/app/src/pages/docs/GetInvolved.js @@ -214,6 +214,12 @@ export default class GetInvolved extends React.Component { color={blue[500]} text="Share new interesting datasets, models, and experiments." /> + @@ -254,7 +260,7 @@ export default class GetInvolved extends React.Component { We want to empower people to change the world for the better. You can help by contributing useful datasets and machine learning pipelines, or by extending OpenML to make it more useful in - science and discovery. + science and discovery, or by using OpenML in your research projects. diff --git a/server/src/client/app/src/pages/search/Dataset.js b/server/src/client/app/src/pages/search/Dataset.js index 80d0fcc7..77cadc53 100644 --- a/server/src/client/app/src/pages/search/Dataset.js +++ b/server/src/client/app/src/pages/search/Dataset.js @@ -34,6 +34,8 @@ const Action = styled.div` justify-content: center; `; +const SERVER_URL = process.env.REACT_APP_OLD_SERVER_URL || "https://www.openml.org/"; + export class DatasetItem extends React.Component { constructor() { super(); @@ -56,7 +58,7 @@ export class DatasetItem extends React.Component { {context => ( - + xml @@ -64,7 +66,7 @@ export class DatasetItem extends React.Component { - + json diff --git a/server/src/client/app/src/pages/search/api.js b/server/src/client/app/src/pages/search/api.js index ed7c5718..f730009d 100644 --- a/server/src/client/app/src/pages/search/api.js +++ b/server/src/client/app/src/pages/search/api.js @@ -58,7 +58,7 @@ export function getProperty(obj, param) { } } -const ELASTICSEARCH_SERVER = "https://www.openml.org/es/"; +const ELASTICSEARCH_SERVER = process.env.REACT_APP_ES_URL || "https://www.openml.org/es/"; // general search export function search( @@ -119,7 +119,7 @@ export function search( }, _source: fields.filter(l => !!l) }; - if (sort !== "match"){ + if (sort !== "match") { params["sort"] = { [sort]: { order: order @@ -160,7 +160,7 @@ export function search( //get specific item export function getItem(type, itemId) { - return fetch(ELASTICSEARCH_SERVER + "/" + type + "/" + type + "/" + itemId, { + return fetch(ELASTICSEARCH_SERVER + type + "/" + type + "/" + itemId, { headers: { Accept: "application/json", "Content-Type": "application/json" @@ -173,8 +173,8 @@ export function getItem(type, itemId) { if (data["found"] !== true) { throw Error( 'No task with id "' + - itemId + - '" found. It may have been removed or renamed' + itemId + + '" found. It may have been removed or renamed' ); } return Promise.resolve(data["_source"]); @@ -183,7 +183,7 @@ export function getItem(type, itemId) { // Not used? export function getList(itemId) { - return fetch(ELASTICSEARCH_SERVER + "/data/data/list/tag/" + itemId, { + return fetch(ELASTICSEARCH_SERVER + "data/data/list/tag/" + itemId, { headers: { Accept: "application/json", "Content-Type": "application/json" @@ -196,8 +196,8 @@ export function getList(itemId) { if (data["found"] !== true) { throw Error( 'No task with id "' + - itemId + - '" found. It may have been removed or renamed' + itemId + + '" found. It may have been removed or renamed' ); } return Promise.resolve(data["_source"]); diff --git a/server/src/dashboard/flow_callbacks.py b/server/src/dashboard/flow_callbacks.py index dbf34c8e..43b62e6b 100644 --- a/server/src/dashboard/flow_callbacks.py +++ b/server/src/dashboard/flow_callbacks.py @@ -1,8 +1,10 @@ import re +import os import pandas as pd import plotly.graph_objs as go from dash.dependencies import Input, Output +import openml from openml import evaluations, tasks from .dash_config import DASH_CACHING @@ -10,6 +12,8 @@ TIMEOUT = 5 * 60 if DASH_CACHING else 0 +SERVER_BASE_URL = os.getenv('BACKEND_BASE_URL', "https://www.openml.org/") +openml.config.server = os.getenv('BACKEND_SERVER') def register_flow_callbacks(app, cache): @app.callback( @@ -84,11 +88,11 @@ def update_flow_plots(pathname, metric, tasktype, parameter): tick_text = [] # Set clickable labels for run_id in df["run_id"].values: - link = ' ' + link = ' ' run_link.append(link) for data_id in df["data_id"].values: - link = '' + link = '' tick_text.append(link) hover_text = [] if parameter == "None": diff --git a/server/src/dashboard/helpers.py b/server/src/dashboard/helpers.py index eca2ad14..a071a397 100644 --- a/server/src/dashboard/helpers.py +++ b/server/src/dashboard/helpers.py @@ -1,3 +1,4 @@ +import os import logging import time from contextlib import contextmanager @@ -5,12 +6,14 @@ import numpy as np import pandas as pd import scipy.stats +import openml from openml import datasets, runs from sklearn.model_selection import train_test_split logger = logging.getLogger("dashboard") logger.setLevel(logging.DEBUG) +openml.config.server = os.getenv('BACKEND_SERVER') def get_run_df(run_id: int): run = runs.get_run(int(run_id), ignore_cache=True) @@ -96,6 +99,11 @@ def get_data_metadata(data_id): start = time.time() meta_features, data, _ = get_metadata(data_id) + # Replacing the substring to when Docker is active + if (os.getenv('DASHBOARD_USE_DOCKER_CONTAINER_NAME', 'False') == "True"): + data.url = data.url.replace("localhost", os.getenv('DASHBOARD_PHP_CONTAINER_NAME', 'website')) + print(data) + x, y, categorical, attribute_names = data.get_data() df = pd.DataFrame(x, columns=attribute_names) diff --git a/server/src/dashboard/layouts.py b/server/src/dashboard/layouts.py index 959ccb3d..b2e635b0 100644 --- a/server/src/dashboard/layouts.py +++ b/server/src/dashboard/layouts.py @@ -4,10 +4,13 @@ import dash_core_components as dcc import dash_html_components as html import dash_table as dt +import openml from openml import datasets, evaluations, runs, setups, study from .helpers import get_metadata, get_run_df, logger +openml.config.server = os.getenv('BACKEND_SERVER') + # TODO: Move to assets (Copied from Joaquin's react font) font = [ "Nunito Sans", diff --git a/server/src/dashboard/overviews.py b/server/src/dashboard/overviews.py index 169691dd..407110a3 100644 --- a/server/src/dashboard/overviews.py +++ b/server/src/dashboard/overviews.py @@ -1,14 +1,17 @@ +import os import dash_core_components as dcc import dash_html_components as html import pandas as pd import plotly.graph_objs as go from dash.dependencies import Input, Output +import openml from openml import datasets, flows, runs, tasks from openml.extensions.sklearn import SklearnExtension from .dash_config import DASH_CACHING TIMEOUT = 5 * 60 if DASH_CACHING else 0 +openml.config.server = os.getenv('BACKEND_SERVER') font = [ "Nunito Sans", diff --git a/server/src/dashboard/run_callbacks.py b/server/src/dashboard/run_callbacks.py index 78bf066f..1f8ca41a 100644 --- a/server/src/dashboard/run_callbacks.py +++ b/server/src/dashboard/run_callbacks.py @@ -1,5 +1,6 @@ import io import re +import os # import arff import urllib.request @@ -19,6 +20,7 @@ TIMEOUT = 5 * 60 if DASH_CACHING else 0 +SERVER_BASE_URL = os.getenv('BACKEND_BASE_URL', "https://www.openml.org/") def register_run_callbacks(app, cache): @app.callback( @@ -94,7 +96,7 @@ def pr_chart(pathname, rows): return "Only classification supported", "Only classification supported" pred_id = df[df["evaluations"] == "predictions"]["results"].values[0] url = ( - "https://www.openml.org/data/download/{}".format(pred_id) + SERVER_BASE_URL + "data/download/{}".format(pred_id) + "/predictions.arff" ) ftp_stream = urllib.request.urlopen(url) diff --git a/server/src/dashboard/task_callbacks.py b/server/src/dashboard/task_callbacks.py index 289e759f..1940846c 100644 --- a/server/src/dashboard/task_callbacks.py +++ b/server/src/dashboard/task_callbacks.py @@ -1,4 +1,5 @@ import re +import os import dash_core_components as dcc import dash_html_components as html @@ -8,12 +9,15 @@ from dash.dependencies import Input, Output +import openml from openml import evaluations from openml.extensions.sklearn import SklearnExtension from .helpers import get_highest_rank from .dash_config import DASH_CACHING +openml.config.server = os.getenv('BACKEND_SERVER') + font = [ "Nunito Sans", "-apple-system", @@ -28,6 +32,7 @@ "Segoe UI Symbol", ] +SERVER_BASE_URL = os.getenv('BACKEND_BASE_URL', "https://www.openml.org/") TIMEOUT = 5 * 60 if DASH_CACHING else 0 @@ -97,11 +102,11 @@ def update_task_plots(pathname, metric, n_clicks): truncated = [] # Plotly hack to add href to each data point for run_id in df["run_id"].values: - link = ' ' + link = ' ' run_link.append(link) # Plotly hack to link flow names for flow_id in df["flow_id"].values: - link = '' + link = '' tick_text.append(link) # Truncate flow names (50 chars) for flow in df["flow_name"].values: @@ -153,11 +158,11 @@ def update_task_plots(pathname, metric, n_clicks): tick_text = [] run_link = [] for run_id in df["run_id"].values: - link = ' ' + link = ' ' run_link.append(link) for flow_id in df["flow_id"].values: - link = '' + link = '' tick_text.append(link) df["upload_time"] = pd.to_datetime(df["upload_time"]) diff --git a/server/src/dashboard/tests/test_flows.py b/server/src/dashboard/tests/test_flows.py index f9d42cae..9fec27af 100644 --- a/server/src/dashboard/tests/test_flows.py +++ b/server/src/dashboard/tests/test_flows.py @@ -1,9 +1,13 @@ import time +import os +import openml from openml import evaluations from server.src.dashboard.dash_config import BASE_URL +openml.config.server = os.getenv('BACKEND_SERVER') + # def test_flow_page_loading(dash_br): # dash_br.server_url = BASE_URL + 'flow/405' # time.sleep(10) diff --git a/server/src/dashboard/tests/test_runs.py b/server/src/dashboard/tests/test_runs.py index 6a50045e..593ad3b8 100644 --- a/server/src/dashboard/tests/test_runs.py +++ b/server/src/dashboard/tests/test_runs.py @@ -1,11 +1,14 @@ import time +import os import numpy as np import pandas as pd +import openml from openml import runs from server.src.dashboard.dash_config import BASE_URL +openml.config.server = os.getenv('BACKEND_SERVER') def uncommon_string(s1, s2): st1 = set(s1) diff --git a/server/src/dashboard/tests/test_tasks.py b/server/src/dashboard/tests/test_tasks.py index c87c5704..ac597e01 100644 --- a/server/src/dashboard/tests/test_tasks.py +++ b/server/src/dashboard/tests/test_tasks.py @@ -1,9 +1,12 @@ import time +import os +import openml from openml import evaluations from server.src.dashboard.dash_config import BASE_URL +openml.config.server = os.getenv('BACKEND_SERVER') def test_task_page_loading(dash_br): dash_br.server_url = f"{BASE_URL}task/2"