diff --git a/.editorconfig b/.editorconfig index 7ce4a2e..9ff6626 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,6 +19,10 @@ indent_style = space [*.{yml,yaml}] indent_style = space +[*.{py}] +indent_size = 4 +indent_style = space + [*.{cmd,bat}] end_of_line = crlf diff --git a/poetry.lock b/poetry.lock index 8ee44cd..8d57a35 100644 --- a/poetry.lock +++ b/poetry.lock @@ -112,10 +112,7 @@ files = [ [package.dependencies] jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" -urllib3 = [ - {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, - {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}, -] +urllib3 = {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""} [package.extras] crt = ["awscrt (==0.19.17)"] @@ -1069,7 +1066,6 @@ files = [ [package.dependencies] anyio = ">=3.4.0,<5" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] @@ -1107,22 +1103,6 @@ files = [ {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] -[[package]] -name = "urllib3" -version = "1.26.18" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, - {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, -] - -[package.extras] -brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - [[package]] name = "urllib3" version = "2.0.7" @@ -1180,5 +1160,5 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [metadata] lock-version = "2.0" -python-versions = "^3.9" -content-hash = "13a1caba532774b068839d5aa13f51866d7d93b5b698e8126cf0340d44ecf8cb" +python-versions = "^3.10" +content-hash = "247d988436dbf9ae30809f2b1fb325a6f5afd0069023a80e9d8a96d4c2e68d5d" diff --git a/pyproject.toml b/pyproject.toml index 5004216..0853221 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,9 @@ authors = ["Powell Fendley", "Five Grant"] readme = "README.md" packages = [{include = "service"}, {include = "tests"}] + [tool.poetry.dependencies] -python = "^3.9" +python = "^3.10" requests = "^2.31.0" fastapi = "^0.96.0" rq = "^1.15.0" @@ -21,6 +22,7 @@ poethepoet = "^0.21.1" # juliacall = { version="^0.9.14", optional = true } dill = "^0.3.7" + [tool.poetry.scripts] mockrabbitmq = "service.utils.rabbitmq:mock_rabbitmq_consumer" @@ -35,6 +37,7 @@ mock = "^5.1.0" fakeredis = "^2.17.0" httpx = "^0.24.1" + [tool.poe.tasks] install-pyciemss = "pip install --no-cache-dir git+https://github.com/fivegrant/pyciemss.git@087bc64d935f2ab5090330f1f7d6bde930404115 --use-pep517" @@ -47,3 +50,7 @@ pythonpath = "service" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + + +[tool.ruff] +ignore = ["E501"] diff --git a/service/execute.py b/service/execute.py index ff334fe..4325d36 100644 --- a/service/execute.py +++ b/service/execute.py @@ -1,7 +1,6 @@ import logging # from juliacall import newmodule -from settings import settings from utils.tds import ( update_tds_status, cleanup_job_dir, @@ -10,9 +9,6 @@ from pyciemss.interfaces import sample, calibrate, ensemble_sample # noqa: F401 -TDS_SIMULATIONS = "/simulations/" -TDS_URL = settings.TDS_URL - # jl = newmodule("SciMLIntegration") # jl.seval("using SciMLIntegration, PythonCall") @@ -21,9 +17,8 @@ def run(request, *, job_id): - logging.debug(f"STARTED {job_id} (username: {request.username})") - sim_results_url = TDS_URL + TDS_SIMULATIONS + job_id - update_tds_status(sim_results_url, status="running", start=True) + logging.debug(f"STARTED {job_id} (user_id: {request.user_id})") + update_tds_status(job_id, status="running", start=True) # if request.engine == "ciemss": operation_name = request.__class__.pyciemss_lib_function @@ -36,6 +31,6 @@ def run(request, *, job_id): # operation = request.__class__.sciml_lib_function # output = operation(job_id, jl) - attach_files(output, TDS_URL, TDS_SIMULATIONS, job_id) + attach_files(output, job_id) cleanup_job_dir(job_id) - logging.debug(f"FINISHED {job_id} (username: {request.username})") + logging.debug(f"FINISHED {job_id} (user_id: {request.user_id})") diff --git a/service/models.py b/service/models.py index 442efa3..5881394 100644 --- a/service/models.py +++ b/service/models.py @@ -15,11 +15,6 @@ from utils.convert import convert_to_static_interventions, convert_to_solution_mapping from utils.rabbitmq import gen_rabbitmq_hook # noqa: F401 from utils.tds import fetch_dataset, fetch_model, fetch_inferred_parameters -from settings import settings - -TDS_CONFIGURATIONS = "/model_configurations/" -TDS_SIMULATIONS = "/simulations/" -TDS_URL = settings.TDS_URL class Timespan(BaseModel): @@ -95,7 +90,7 @@ class QuantityOfInterest(BaseModel): class OperationRequest(BaseModel): pyciemss_lib_function: ClassVar[str] = "" engine: str = Field("ciemss", example="ciemss") - username: str = Field("not_provided", example="not_provided") + user_id: str = Field("not_provided", example="not_provided") def gen_pyciemss_args(self, job_id): raise NotImplementedError("PyCIEMSS cannot handle this operation") @@ -137,15 +132,13 @@ class Simulate(OperationRequest): def gen_pyciemss_args(self, job_id): # Get model from TDS - amr_path = fetch_model( - self.model_config_id, TDS_URL, TDS_CONFIGURATIONS, job_id - ) + amr_path = fetch_model(self.model_config_id, job_id) interventions = convert_to_static_interventions(self.interventions) extra_options = self.extra.dict() inferred_parameters = fetch_inferred_parameters( - extra_options.pop("inferred_parameters"), TDS_URL, job_id + extra_options.pop("inferred_parameters"), job_id ) return { @@ -159,9 +152,7 @@ def gen_pyciemss_args(self, job_id): } def run_sciml_operation(self, job_id, julia_context): - amr_path = fetch_model( - self.model_config_id, TDS_URL, TDS_CONFIGURATIONS, job_id - ) + amr_path = fetch_model(self.model_config_id, job_id) with open(amr_path, "r") as file: amr = file.read() result = julia_context.simulate(amr, self.timespan.start, self.timespan.end) @@ -209,11 +200,9 @@ class Calibrate(OperationRequest): ) def gen_pyciemss_args(self, job_id): - amr_path = fetch_model( - self.model_config_id, TDS_URL, TDS_CONFIGURATIONS, job_id - ) + amr_path = fetch_model(self.model_config_id, job_id) - dataset_path = fetch_dataset(self.dataset.dict(), TDS_URL, job_id) + dataset_path = fetch_dataset(self.dataset.dict(), job_id) # TODO: Test RabbitMQ try: @@ -271,10 +260,7 @@ def gen_pyciemss_args(self, job_id): solution_mappings = [ convert_to_solution_mapping(config) for config in self.model_configs ] - amr_paths = [ - fetch_model(config.id, TDS_URL, TDS_CONFIGURATIONS, job_id) - for config in self.model_configs - ] + amr_paths = [fetch_model(config.id, job_id) for config in self.model_configs] return { "model_paths_or_jsons": amr_paths, @@ -310,7 +296,7 @@ class Config: # pyciemss_lib_function: ClassVar[ # str # ] = "load_and_calibrate_and_sample_ensemble_model" -# username: str = Field("not_provided", example="not_provided") +# user_id: str = Field("not_provided", example="not_provided") # model_configs: List[ModelConfig] = Field( # [], # example=[], @@ -327,11 +313,11 @@ class Config: # solution_mappings = [config.solution_mappings for config # in self.model_configs] # amr_paths = [ -# fetch_model(config.id, TDS_URL, TDS_CONFIGURATIONS, job_id) +# fetch_model(config.id, job_id) # for config in self.model_configs # ] -# dataset_path = fetch_dataset(self.dataset.dict(), TDS_URL, job_id) +# dataset_path = fetch_dataset(self.dataset.dict(), job_id) # # Generate timepoints # time_count = self.timespan.end - self.timespan.start diff --git a/service/settings.py b/service/settings.py index 1873f65..96cf7fa 100644 --- a/service/settings.py +++ b/service/settings.py @@ -11,6 +11,8 @@ class Settings(BaseSettings): """ TDS_URL: str = "http://data-service-api:8000" + TDS_USER: str = "user" + TDS_PASSWORD: str = "password" REDIS_HOST: str = "redis" REDIS_PORT: int = 6379 RABBITMQ_HOST: str = "rabbitmq.pyciemss" diff --git a/service/utils/rq_helpers.py b/service/utils/rq_helpers.py index 16f6e3a..f110e5f 100644 --- a/service/utils/rq_helpers.py +++ b/service/utils/rq_helpers.py @@ -3,8 +3,6 @@ import logging from uuid import uuid4 -import json -import requests from fastapi import Response, status from redis import Redis @@ -13,10 +11,7 @@ from rq.job import Job from settings import settings -from utils.tds import update_tds_status - -TDS_SIMULATIONS = "/simulations/" -TDS_URL = settings.TDS_URL +from utils.tds import update_tds_status, create_tds_job, cancel_tds_job logging.basicConfig() logging.getLogger().setLevel(logging.DEBUG) @@ -27,42 +22,37 @@ def get_redis(): def update_status_on_job_fail(job, connection, etype, value, traceback): - update_tds_status(TDS_URL + TDS_SIMULATIONS + str(job.id), "error") + update_tds_status(str(job.id), "error") log_message = f""" ############################### There was an exception in CIEMSS Service - + job: {job.id} - {etype}: {value} + {etype}: {value} ################################ """ logging.exception(log_message) def create_job(request_payload, sim_type, redis_conn): - job_id = f"ciemss-{uuid4()}" + workflow_id = f"{uuid4()}" - post_url = TDS_URL + TDS_SIMULATIONS payload = { - "id": job_id, + "name": workflow_id, "execution_payload": request_payload.dict(), "result_files": [], "type": sim_type, "status": "queued", "engine": request_payload.engine, - "workflow_id": job_id, + "workflow_id": workflow_id, } logging.info(payload) - response = requests.post(post_url, json=payload) - if response.status_code >= 300: - raise Exception( - ( - "Failed to create simulation on TDS " - f"(status: {response.status_code}): {json.dumps(payload)}" - ) - ) - logging.info(response.content) + + res = create_tds_job(payload) + job_id = res["id"] + + logging.info(res) queue = Queue(connection=redis_conn, default_timeout=-1) queue.enqueue_call( @@ -80,7 +70,7 @@ def fetch_job_status(job_id, redis_conn): """Fetch a job's results from RQ. Args: - job_id (str): The id of the job being run in RQ. + job_id (uuid): The id of the job being run in RQ. Returns: Response: @@ -111,10 +101,7 @@ def kill_job(job_id, redis_conn): else: job.cancel() - url = TDS_URL + TDS_SIMULATIONS + str(job_id) - tds_payload = requests.get(url).json() - tds_payload["status"] = "cancelled" - requests.put(url, json=json.loads(json.dumps(tds_payload, default=str))) + cancel_tds_job(str(job_id)) result = job.get_status() return result diff --git a/service/utils/tds.py b/service/utils/tds.py index 9ab8430..85f4603 100644 --- a/service/utils/tds.py +++ b/service/utils/tds.py @@ -1,10 +1,10 @@ """ -Module used to interact with the Terarium Data Service (TDS) +Module used to interact with the Terarium Data Service (TDS) """ from __future__ import annotations import logging -import urllib + import os import shutil import json @@ -15,14 +15,47 @@ from fastapi import HTTPException - from settings import settings -TDS_SIMULATIONS = "/simulations/" TDS_URL = settings.TDS_URL +TDS_USER = settings.TDS_USER +TDS_PASSWORD = settings.TDS_PASSWORD +TDS_SIMULATIONS = "/simulations" +TDS_DATASETS = "/datasets" +TDS_CONFIGURATIONS = "/model-configurations" + + +def tds_session(): + session = requests.Session() + session.auth = (TDS_USER, TDS_PASSWORD) + session.headers.update( + {"Content-Type": "application/json", "X-Enable-Snake-Case": ""} + ) + return session -def update_tds_status(url, status, result_files=[], start=False, finish=False): +def create_tds_job(payload): + post_url = TDS_URL + TDS_SIMULATIONS + response = tds_session().post(post_url, json=payload) + if response.status_code >= 300: + raise Exception( + ( + "Failed to create simulation on TDS " + f"(status: {response.status_code}): {json.dumps(payload)}" + ) + ) + return response.json() + + +def cancel_tds_job(job_id): + url = TDS_URL + TDS_SIMULATIONS + "/" + str(job_id) + tds_payload = tds_session().get(url).json() + tds_payload["status"] = "cancelled" + return tds_session().put(url, json=json.loads(json.dumps(tds_payload, default=str))) + + +def update_tds_status(job_id, status, result_files=[], start=False, finish=False): + url = TDS_URL + TDS_SIMULATIONS + "/" + str(job_id) logging.debug( "Updating simulation `%s` -- %s start: %s; finish: %s; result_files: %s", url, @@ -31,7 +64,7 @@ def update_tds_status(url, status, result_files=[], start=False, finish=False): finish, result_files, ) - tds_payload = requests.get(url) + tds_payload = tds_session().get(url) tds_payload = tds_payload.json() if start: @@ -43,9 +76,17 @@ def update_tds_status(url, status, result_files=[], start=False, finish=False): if result_files: tds_payload["result_files"] = result_files - update_response = requests.put( + update_response = tds_session().put( url, json=json.loads(json.dumps(tds_payload, default=str)) ) + if update_response.status_code >= 300: + logging.debug("error", update_response.reason) + raise Exception( + ( + "Failed to update simulation on TDS " + f"(status: {update_response.status_code}): {json.dumps(tds_payload, default=str)}" + ) + ) return update_response @@ -61,55 +102,72 @@ def cleanup_job_dir(job_id): shutil.rmtree(path) -def fetch_model(model_config_id, tds_api, config_endpoint, job_id): +def fetch_model(model_config_id, job_id): job_dir = get_job_dir(job_id) logging.debug(f"Fetching model {model_config_id}") - url_components = [tds_api, config_endpoint, model_config_id] - model_url = "" - for component in url_components: - model_url = urllib.parse.urljoin(model_url, component) - model_response = requests.get(model_url) + + model_url = TDS_URL + TDS_CONFIGURATIONS + "/" + model_config_id + + model_response = tds_session().get(model_url) if model_response.status_code == 404: raise HTTPException(status_code=404, detail="Model not found") + amr_path = os.path.join(job_dir, f"./{model_config_id}.json") with open(amr_path, "w") as file: json.dump(model_response.json()["configuration"], file) return amr_path -def fetch_dataset(dataset: dict, tds_api, job_id): +def fetch_dataset(dataset: dict, job_id): import pandas as pd job_dir = get_job_dir(job_id) logging.debug(f"Fetching dataset {dataset['id']}") dataset_url = ( - f"{tds_api}/datasets/{dataset['id']}/" + f"{TDS_URL}{TDS_DATASETS}/{dataset['id']}/" f"download-url?filename={dataset['filename']}" ) - response = requests.get(dataset_url) + response = tds_session().get(dataset_url) + + # Small hack to rename mapping, namely timestamp => Timestamp if timestamp exist as a value + key_to_rename = None + for item in dataset["mappings"].items(): + if item[1] == "timestamp": + key_to_rename = item[0] + if key_to_rename is not None: + logging.debug(f"Overwriting {key_to_rename}") + logging.debug("") + dataset["mappings"][key_to_rename] = "Timestamp" + if response.status_code >= 300: raise HTTPException(status_code=400, detail="Unable to retrieve dataset") df = pd.read_csv(response.json()["url"]) df = df.rename(columns=dataset["mappings"]) + + # Shift Timestamp to first position + col = df.pop("Timestamp") + df.insert(0, "Timestamp", col) + # drop columns that aren't being mapped if len(df.columns) > len(dataset["mappings"]) and len(dataset["mappings"]) > 0: extra_columns = set(df.columns) - set(dataset["mappings"].values()) df.drop(columns=list(extra_columns), inplace=True) + dataset_path = os.path.join(job_dir, "./temp.json") with open(dataset_path, "w") as file: df.to_csv(file, index=False) return dataset_path -def fetch_inferred_parameters(parameters_id: Optional[str], tds_api, job_id): +def fetch_inferred_parameters(parameters_id: Optional[str], job_id): if parameters_id is None: return job_dir = get_job_dir(job_id) logging.debug(f"Fetching inferred parameters {parameters_id}") - download_url = ( - f"{tds_api}/simulations/{parameters_id}/download-url?filename=parameters.dill" - ) - parameters_url = requests.get(download_url).json()["url"] + download_url = f"{TDS_URL}{TDS_SIMULATIONS}/{parameters_id}/download-url?filename=parameters.dill" + parameters_url = tds_session().get(download_url).json()["url"] + # response = tds_session().get(parameters_url) + response = requests.get(parameters_url) if response.status_code >= 300: raise HTTPException(status_code=400, detail="Unable to retrieve parameters") @@ -119,8 +177,8 @@ def fetch_inferred_parameters(parameters_id: Optional[str], tds_api, job_id): return dill.loads(response.content) -def attach_files(output: dict, tds_api, simulation_endpoint, job_id, status="complete"): - sim_results_url = tds_api + simulation_endpoint + job_id +def attach_files(output: dict, job_id, status="complete"): + sim_results_url = TDS_URL + TDS_SIMULATIONS + "/" + str(job_id) job_dir = get_job_dir(job_id) files = {} @@ -160,8 +218,9 @@ def attach_files(output: dict, tds_api, simulation_endpoint, job_id, status="com if status != "error": for location, handle in files.items(): upload_url = f"{sim_results_url}/upload-url?filename={handle}" - upload_response = requests.get(upload_url) + upload_response = tds_session().get(upload_url) presigned_upload_url = upload_response.json()["url"] + with open(location, "rb") as f: upload_response = requests.put(presigned_upload_url, f) if upload_response.status_code >= 300: @@ -176,6 +235,6 @@ def attach_files(output: dict, tds_api, simulation_endpoint, job_id, status="com # Update simulation object with status and filepaths. update_tds_status( - sim_results_url, status=status, result_files=list(files.values()), finish=True + job_id, status=status, result_files=list(files.values()), finish=True ) logging.info("uploaded files to %s", job_id) diff --git a/tests/examples/calibrate/input/request.json b/tests/examples/calibrate/input/request.json index fe7ad3f..3229711 100644 --- a/tests/examples/calibrate/input/request.json +++ b/tests/examples/calibrate/input/request.json @@ -1,26 +1,26 @@ { - "engine": "ciemss", - "username": "not_provided", - "model_config_id": "0da53e71-52c4-49fd-b957-2219af712fdd", - "dataset": { - "id": "cc52e71f-c744-4883-9412-0858d8455754", - "filename": "test.csv", - "mappings": { - "tstep": "Timestamp", - "S": "Susceptible" - } - }, - "timespan": { - "start": 0, - "end": 90 - }, - "extra": { - "num_samples": 100, - "start_time": -1e-10, - "num_iterations": 1000, - "lr": 0.03, - "verbose": false, - "num_particles": 1, - "method": "dopri5" - } -} \ No newline at end of file + "engine": "ciemss", + "user_id": "not_provided", + "model_config_id": "0da53e71-52c4-49fd-b957-2219af712fdd", + "dataset": { + "id": "cc52e71f-c744-4883-9412-0858d8455754", + "filename": "test.csv", + "mappings": { + "tstep": "Timestamp", + "S": "Susceptible" + } + }, + "timespan": { + "start": 0, + "end": 90 + }, + "extra": { + "num_samples": 100, + "start_time": -1e-10, + "num_iterations": 1000, + "lr": 0.03, + "verbose": false, + "num_particles": 1, + "method": "dopri5" + } +} diff --git a/tests/examples/calibrate/output/tds_simulation.json b/tests/examples/calibrate/output/tds_simulation.json index 9b5c77a..578bfd4 100644 --- a/tests/examples/calibrate/output/tds_simulation.json +++ b/tests/examples/calibrate/output/tds_simulation.json @@ -1,45 +1,44 @@ { - "id": "ciemss-0478a0f7-21b3-4241-afa2-252e1c1992d8", - "name": null, - "description": null, - "timestamp": "2023-08-03T21:39:12", - "engine": "ciemss", - "type": "calibrate", - "status": "complete", - "execution_payload": { - "engine": "ciemss", - "username": "not_provided", - "model_config_id": "0da53e71-52c4-49fd-b957-2219af712fdd", - "dataset": { - "id": "cc52e71f-c744-4883-9412-0858d8455754", - "filename": "test.csv", - "mappings": { - "tstep": "Timestamp", - "S": "Susceptible" - } - }, - "timespan": { - "start": 0, - "end": 90 - }, - "extra": { - "num_samples": 100, - "start_time": -1e-10, - "num_iterations": 1000, - "lr": 0.03, - "verbose": false, - "num_particles": 1, - "method": "dopri5" - } - }, - "start_time": "2023-08-03T21:37:44.918532", - "completed_time": "2023-08-03T21:39:12.343505", - "workflow_id": "ciemss-0478a0f7-21b3-4241-afa2-252e1c1992d8", - "user_id": null, - "project_id": null, - "result_files": [ - "result.csv", - "visualization.json", - "eval.csv" - ] -} \ No newline at end of file + "id": "0478a0f7-21b3-4241-afa2-252e1c1992d8", + "name": null, + "description": null, + "timestamp": "2023-08-03T21:39:12", + "engine": "ciemss", + "type": "calibrate", + "status": "complete", + "execution_payload": { + "engine": "ciemss", + "user_id": "not_provided", + "model_config_id": "0da53e71-52c4-49fd-b957-2219af712fdd", + "dataset": { + "id": "cc52e71f-c744-4883-9412-0858d8455754", + "filename": "test.csv", + "mappings": { + "tstep": "Timestamp", + "S": "Susceptible" + } + }, + "timespan": { + "start": 0, + "end": 90 + }, + "extra": { + "num_samples": 100, + "start_time": -1e-10, + "num_iterations": 1000, + "lr": 0.03, + "verbose": false, + "num_particles": 1, + "method": "dopri5" + } + }, + "start_time": "2023-08-03T21:37:44.918532", + "completed_time": "2023-08-03T21:39:12.343505", + "user_id": null, + "project_id": null, + "result_files": [ + "result.csv", + "visualization.json", + "eval.csv" + ] +} diff --git a/tests/examples/ensemble-calibrate/output/tds_simulation.json b/tests/examples/ensemble-calibrate/output/tds_simulation.json index dad4652..9ee2181 100644 --- a/tests/examples/ensemble-calibrate/output/tds_simulation.json +++ b/tests/examples/ensemble-calibrate/output/tds_simulation.json @@ -1,56 +1,55 @@ { - "id": "ciemss-30862181-ab25-4361-b9ce-591555a5e3cb", - "name": null, - "description": null, - "timestamp": "2023-08-03T22:15:05", - "engine": "ciemss", - "type": "ensemble-calibrate", - "status": "complete", - "execution_payload": { - "engine": "ciemss", - "username": "not_provided", - "model_configs": [ - { - "id": "398d2a33-1cbe-44cf-9b66-a6976b947809", - "solution_mappings": { - "Infected": "Cases", - "Hospitalizations": "hospitalized_population" - }, - "weight": 0.5 - }, - { - "id": "fab58d86-cf1e-4990-809a-58c029545a3a", - "solution_mappings": { - "Infected": "Infections", - "Hospitalizations": "hospitalized_population" - }, - "weight": 0.5 - } - ], - "timespan": { - "start": 0, - "end": 5 - }, - "dataset": { - "id": "cc52e71f-c744-4883-9412-0858d8455754", - "filename": "ensemble.csv", - "mappings": {} - }, - "extra": { - "num_samples": 200, - "total_population": 1000, - "num_iterations": 8, - "time_unit": "days" - } - }, - "start_time": "2023-08-03T22:14:35.363910", - "completed_time": "2023-08-03T22:15:05.202220", - "workflow_id": "ciemss-30862181-ab25-4361-b9ce-591555a5e3cb", - "user_id": null, - "project_id": null, - "result_files": [ - "result.csv", - "visualization.json", - "eval.csv" - ] -} \ No newline at end of file + "id": "30862181-ab25-4361-b9ce-591555a5e3cb", + "name": null, + "description": null, + "timestamp": "2023-08-03T22:15:05", + "engine": "ciemss", + "type": "ensemble-calibrate", + "status": "complete", + "execution_payload": { + "engine": "ciemss", + "user_id": "not_provided", + "model_configs": [ + { + "id": "398d2a33-1cbe-44cf-9b66-a6976b947809", + "solution_mappings": { + "Infected": "Cases", + "Hospitalizations": "hospitalized_population" + }, + "weight": 0.5 + }, + { + "id": "fab58d86-cf1e-4990-809a-58c029545a3a", + "solution_mappings": { + "Infected": "Infections", + "Hospitalizations": "hospitalized_population" + }, + "weight": 0.5 + } + ], + "timespan": { + "start": 0, + "end": 5 + }, + "dataset": { + "id": "cc52e71f-c744-4883-9412-0858d8455754", + "filename": "ensemble.csv", + "mappings": {} + }, + "extra": { + "num_samples": 200, + "total_population": 1000, + "num_iterations": 8, + "time_unit": "days" + } + }, + "start_time": "2023-08-03T22:14:35.363910", + "completed_time": "2023-08-03T22:15:05.202220", + "user_id": null, + "project_id": null, + "result_files": [ + "result.csv", + "visualization.json", + "eval.csv" + ] +} diff --git a/tests/examples/ensemble-simulate-temp-quick-fix/output/tds_simulation.json b/tests/examples/ensemble-simulate-temp-quick-fix/output/tds_simulation.json index 94f20d6..4705b2b 100644 --- a/tests/examples/ensemble-simulate-temp-quick-fix/output/tds_simulation.json +++ b/tests/examples/ensemble-simulate-temp-quick-fix/output/tds_simulation.json @@ -1,48 +1,47 @@ { - "id": "ciemss-9036f8a8-7e55-4e77-aeec-e8d4ca120d67", - "name": null, - "description": null, - "timestamp": "2023-08-03T21:13:45", - "engine": "ciemss", - "type": "ensemble-simulate", - "status": "complete", - "execution_payload": { - "engine": "ciemss", - "username": "not_provided", - "model_configs": [ - { - "id": "398d2a33-1cbe-44cf-9b66-a6976b947809", - "solution_mappings": { - "Infected": "Cases", - "Hospitalizations": "hospitalized_population" - }, - "weight": 0.5 - }, - { - "id": "fab58d86-cf1e-4990-809a-58c029545a3a", - "solution_mappings": { - "Infected": "Infections", - "Hospitalizations": "hospitalized_population" - }, - "weight": 0.5 - } - ], - "timespan": { - "start": 0, - "end": 5 - }, - "extra": { - "num_samples": 100 - } - }, - "start_time": "2023-08-03T21:13:31.641100", - "completed_time": "2023-08-03T21:13:45.138033", - "workflow_id": "ciemss-9036f8a8-7e55-4e77-aeec-e8d4ca120d67", - "user_id": null, - "project_id": null, - "result_files": [ - "result.csv", - "visualization.json", - "eval.csv" - ] -} \ No newline at end of file + "id": "9036f8a8-7e55-4e77-aeec-e8d4ca120d67", + "name": null, + "description": null, + "timestamp": "2023-08-03T21:13:45", + "engine": "ciemss", + "type": "ensemble-simulate", + "status": "complete", + "execution_payload": { + "engine": "ciemss", + "username": "not_provided", + "model_configs": [ + { + "id": "398d2a33-1cbe-44cf-9b66-a6976b947809", + "solution_mappings": { + "Infected": "Cases", + "Hospitalizations": "hospitalized_population" + }, + "weight": 0.5 + }, + { + "id": "fab58d86-cf1e-4990-809a-58c029545a3a", + "solution_mappings": { + "Infected": "Infections", + "Hospitalizations": "hospitalized_population" + }, + "weight": 0.5 + } + ], + "timespan": { + "start": 0, + "end": 5 + }, + "extra": { + "num_samples": 100 + } + }, + "start_time": "2023-08-03T21:13:31.641100", + "completed_time": "2023-08-03T21:13:45.138033", + "user_id": null, + "project_id": null, + "result_files": [ + "result.csv", + "visualization.json", + "eval.csv" + ] +} diff --git a/tests/examples/ensemble-simulate/output/tds_simulation.json b/tests/examples/ensemble-simulate/output/tds_simulation.json index 94f20d6..38ef234 100644 --- a/tests/examples/ensemble-simulate/output/tds_simulation.json +++ b/tests/examples/ensemble-simulate/output/tds_simulation.json @@ -1,48 +1,47 @@ { - "id": "ciemss-9036f8a8-7e55-4e77-aeec-e8d4ca120d67", - "name": null, - "description": null, - "timestamp": "2023-08-03T21:13:45", - "engine": "ciemss", - "type": "ensemble-simulate", - "status": "complete", - "execution_payload": { - "engine": "ciemss", - "username": "not_provided", - "model_configs": [ - { - "id": "398d2a33-1cbe-44cf-9b66-a6976b947809", - "solution_mappings": { - "Infected": "Cases", - "Hospitalizations": "hospitalized_population" - }, - "weight": 0.5 - }, - { - "id": "fab58d86-cf1e-4990-809a-58c029545a3a", - "solution_mappings": { - "Infected": "Infections", - "Hospitalizations": "hospitalized_population" - }, - "weight": 0.5 - } - ], - "timespan": { - "start": 0, - "end": 5 - }, - "extra": { - "num_samples": 100 - } - }, - "start_time": "2023-08-03T21:13:31.641100", - "completed_time": "2023-08-03T21:13:45.138033", - "workflow_id": "ciemss-9036f8a8-7e55-4e77-aeec-e8d4ca120d67", - "user_id": null, - "project_id": null, - "result_files": [ - "result.csv", - "visualization.json", - "eval.csv" - ] -} \ No newline at end of file + "id": "9036f8a8-7e55-4e77-aeec-e8d4ca120d67", + "name": null, + "description": null, + "timestamp": "2023-08-03T21:13:45", + "engine": "ciemss", + "type": "ensemble-simulate", + "status": "complete", + "execution_payload": { + "engine": "ciemss", + "user_id": "not_provided", + "model_configs": [ + { + "id": "398d2a33-1cbe-44cf-9b66-a6976b947809", + "solution_mappings": { + "Infected": "Cases", + "Hospitalizations": "hospitalized_population" + }, + "weight": 0.5 + }, + { + "id": "fab58d86-cf1e-4990-809a-58c029545a3a", + "solution_mappings": { + "Infected": "Infections", + "Hospitalizations": "hospitalized_population" + }, + "weight": 0.5 + } + ], + "timespan": { + "start": 0, + "end": 5 + }, + "extra": { + "num_samples": 100 + } + }, + "start_time": "2023-08-03T21:13:31.641100", + "completed_time": "2023-08-03T21:13:45.138033", + "user_id": null, + "project_id": null, + "result_files": [ + "result.csv", + "visualization.json", + "eval.csv" + ] +} diff --git a/tests/examples/simulate-with-parameters/output/tds_simulation.json b/tests/examples/simulate-with-parameters/output/tds_simulation.json index 54c7c4c..ecf93cd 100644 --- a/tests/examples/simulate-with-parameters/output/tds_simulation.json +++ b/tests/examples/simulate-with-parameters/output/tds_simulation.json @@ -1,32 +1,31 @@ { - "id": "ciemss-9ed74639-7778-4bb9-96fd-7509d68cd425", - "name": null, - "description": null, - "timestamp": "2023-08-03T21:03:45", - "engine": "ciemss", - "type": "simulate", - "status": "complete", - "execution_payload": { - "engine": "ciemss", - "username": "not_provided", - "model_config_id": "0da53e71-52c4-49fd-b957-2219af712fdd", - "timespan": { - "start": 0, - "end": 90 - }, - "interventions": [], - "extra": { - "num_samples": 100 - } - }, - "start_time": "2023-08-03T21:03:02.287988", - "completed_time": "2023-08-03T21:03:45.708012", - "workflow_id": "ciemss-9ed74639-7778-4bb9-96fd-7509d68cd425", - "user_id": null, - "project_id": null, - "result_files": [ - "result.csv", - "visualization.json", - "eval.csv" - ] -} \ No newline at end of file + "id": "9ed74639-7778-4bb9-96fd-7509d68cd425", + "name": null, + "description": null, + "timestamp": "2023-08-03T21:03:45", + "engine": "ciemss", + "type": "simulate", + "status": "complete", + "execution_payload": { + "engine": "ciemss", + "username": "not_provided", + "model_config_id": "0da53e71-52c4-49fd-b957-2219af712fdd", + "timespan": { + "start": 0, + "end": 90 + }, + "interventions": [], + "extra": { + "num_samples": 100 + } + }, + "start_time": "2023-08-03T21:03:02.287988", + "completed_time": "2023-08-03T21:03:45.708012", + "user_id": null, + "project_id": null, + "result_files": [ + "result.csv", + "visualization.json", + "eval.csv" + ] +} diff --git a/tests/examples/simulate/input/request.json b/tests/examples/simulate/input/request.json index 6472778..f5a60c3 100644 --- a/tests/examples/simulate/input/request.json +++ b/tests/examples/simulate/input/request.json @@ -1,15 +1,19 @@ { - "engine": "ciemss", - "username": "not_provided", - "model_config_id": "sidarthe", - "interventions": [ - {"timestep": 1.0, "name": "beta", "value": 0.4} - ], - "timespan": { - "start": 0, - "end": 90 - }, - "extra": { - "num_samples": 100 - } + "engine": "ciemss", + "user_id": "not_provided", + "model_config_id": "sidarthe", + "interventions": [ + { + "timestep": 1.0, + "name": "beta", + "value": 0.4 + } + ], + "timespan": { + "start": 0, + "end": 90 + }, + "extra": { + "num_samples": 100 + } } diff --git a/tests/examples/simulate/output/tds_simulation.json b/tests/examples/simulate/output/tds_simulation.json index 54c7c4c..c81894d 100644 --- a/tests/examples/simulate/output/tds_simulation.json +++ b/tests/examples/simulate/output/tds_simulation.json @@ -1,32 +1,31 @@ { - "id": "ciemss-9ed74639-7778-4bb9-96fd-7509d68cd425", - "name": null, - "description": null, - "timestamp": "2023-08-03T21:03:45", - "engine": "ciemss", - "type": "simulate", - "status": "complete", - "execution_payload": { - "engine": "ciemss", - "username": "not_provided", - "model_config_id": "0da53e71-52c4-49fd-b957-2219af712fdd", - "timespan": { - "start": 0, - "end": 90 - }, - "interventions": [], - "extra": { - "num_samples": 100 - } - }, - "start_time": "2023-08-03T21:03:02.287988", - "completed_time": "2023-08-03T21:03:45.708012", - "workflow_id": "ciemss-9ed74639-7778-4bb9-96fd-7509d68cd425", - "user_id": null, - "project_id": null, - "result_files": [ - "result.csv", - "visualization.json", - "eval.csv" - ] -} \ No newline at end of file + "id": "9ed74639-7778-4bb9-96fd-7509d68cd425", + "name": null, + "description": null, + "timestamp": "2023-08-03T21:03:45", + "engine": "ciemss", + "type": "simulate", + "status": "complete", + "execution_payload": { + "engine": "ciemss", + "user_id": "not_provided", + "model_config_id": "0da53e71-52c4-49fd-b957-2219af712fdd", + "timespan": { + "start": 0, + "end": 90 + }, + "interventions": [], + "extra": { + "num_samples": 100 + } + }, + "start_time": "2023-08-03T21:03:02.287988", + "completed_time": "2023-08-03T21:03:45.708012", + "user_id": null, + "project_id": null, + "result_files": [ + "result.csv", + "visualization.json", + "eval.csv" + ] +} diff --git a/tests/integration/depr_ensemble_calibrate.py b/tests/integration/depr_ensemble_calibrate.py index b10b13e..bef4028 100644 --- a/tests/integration/depr_ensemble_calibrate.py +++ b/tests/integration/depr_ensemble_calibrate.py @@ -17,19 +17,19 @@ def test_ensemble_calibrate_example( ] for config_id in config_ids: model = json.loads(example_context["fetch"](config_id + ".json")) - requests_mock.get(f"{TDS_URL}/model_configurations/{config_id}", json=model) + requests_mock.get(f"{TDS_URL}/model-configurations/{config_id}", json=model) dataset_id = example_context["request"]["dataset"]["id"] filename = example_context["request"]["dataset"]["filename"] dataset = example_context["fetch"](filename, True) dataset_loc = {"method": "GET", "url": dataset} requests_mock.get( - f"{TDS_URL}/datasets/{dataset_id}/download-url?filename={filename}", + f"{TDS_URL}/datasets/{dataset_id}/download-csv?filename={filename}", json=dataset_loc, ) requests_mock.get("http://dataset", text=dataset) - requests_mock.post(f"{TDS_URL}/simulations/", json={"id": None}) + requests_mock.post(f"{TDS_URL}/simulations", json={"id": None}) response = client.post( "/ensemble-calibrate", diff --git a/tests/integration/test_calibrate.py b/tests/integration/test_calibrate.py index a710072..3f461a0 100644 --- a/tests/integration/test_calibrate.py +++ b/tests/integration/test_calibrate.py @@ -11,6 +11,8 @@ def test_calibrate_example( example_context, client, worker, file_storage, file_check, requests_mock ): + job_id = "0478a0f7-21b3-4241-afa2-252e1c1992d8" + request = example_context["request"] config_id = request["model_config_id"] model = json.loads(example_context["fetch"](config_id + ".json")) @@ -24,7 +26,7 @@ def test_calibrate_example( json=dataset_loc, ) - requests_mock.post(f"{TDS_URL}/simulations/", json={"id": None}) + requests_mock.post(f"{TDS_URL}/simulations", json={"id": str(job_id)}) response = client.post( "/calibrate", @@ -45,7 +47,7 @@ def test_calibrate_example( requests_mock.put( f"{TDS_URL}/simulations/{simulation_id}", json={"status": "success"} ) - requests_mock.get(f"{TDS_URL}/model_configurations/{config_id}", json=model) + requests_mock.get(f"{TDS_URL}/model-configurations/{config_id}", json=model) worker.work(burst=True) diff --git a/tests/integration/test_ensemble_simulate.py b/tests/integration/test_ensemble_simulate.py index ae36eb4..f6d9d03 100644 --- a/tests/integration/test_ensemble_simulate.py +++ b/tests/integration/test_ensemble_simulate.py @@ -12,15 +12,17 @@ def test_ensemble_simulate_example( example_context, client, worker, file_storage, file_check, requests_mock ): + job_id = "9036f8a8-7e55-4e77-aeec-e8d4ca120d67" + request = example_context["request"] config_ids = [ config["id"] for config in example_context["request"]["model_configs"] ] for config_id in config_ids: model = json.loads(example_context["fetch"](config_id + ".json")) - requests_mock.get(f"{TDS_URL}/model_configurations/{config_id}", json=model) + requests_mock.get(f"{TDS_URL}/model-configurations/{config_id}", json=model) - requests_mock.post(f"{TDS_URL}/simulations/", json={"id": None}) + requests_mock.post(f"{TDS_URL}/simulations", json={"id": str(job_id)}) response = client.post( "/ensemble-simulate", diff --git a/tests/integration/test_simulate.py b/tests/integration/test_simulate.py index defc915..7a26127 100644 --- a/tests/integration/test_simulate.py +++ b/tests/integration/test_simulate.py @@ -11,11 +11,13 @@ def test_simulate_example( example_context, client, worker, file_storage, file_check, requests_mock ): + job_id = "9ed74639-7778-4bb9-96fd-7509d68cd425" + request = example_context["request"] config_id = request["model_config_id"] model = json.loads(example_context["fetch"](config_id + ".json")) - requests_mock.post(f"{TDS_URL}/simulations/", json={"id": None}) + requests_mock.post(f"{TDS_URL}/simulations", json={"id": str(job_id)}) response = client.post( "/simulate", @@ -36,7 +38,7 @@ def test_simulate_example( requests_mock.put( f"{TDS_URL}/simulations/{simulation_id}", json={"status": "success"} ) - requests_mock.get(f"{TDS_URL}/model_configurations/{config_id}", json=model) + requests_mock.get(f"{TDS_URL}/model-configurations/{config_id}", json=model) worker.work(burst=True) diff --git a/tests/test_conversions.py b/tests/test_conversions.py index a087f79..179ba4b 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -35,7 +35,7 @@ def test_example_conversion(self, example_context, requests_mock): config_id = example_context["request"]["model_config_id"] model = json.loads(example_context["fetch"](config_id + ".json")) - requests_mock.get(f"{TDS_URL}/model_configurations/{config_id}", json=model) + requests_mock.get(f"{TDS_URL}/model-configurations/{config_id}", json=model) ### Act and Assert @@ -53,7 +53,7 @@ def test_example_conversion(self, example_context, requests_mock): config_id = example_context["request"]["model_config_id"] model = json.loads(example_context["fetch"](config_id + ".json")) - requests_mock.get(f"{TDS_URL}/model_configurations/{config_id}", json=model) + requests_mock.get(f"{TDS_URL}/model-configurations/{config_id}", json=model) dataset_id = example_context["request"]["dataset"]["id"] filename = example_context["request"]["dataset"]["filename"] @@ -82,7 +82,7 @@ def test_example_conversion(self, example_context, requests_mock): ] for config_id in config_ids: model = json.loads(example_context["fetch"](config_id + ".json")) - requests_mock.get(f"{TDS_URL}/model_configurations/{config_id}", json=model) + requests_mock.get(f"{TDS_URL}/model-configurations/{config_id}", json=model) ### Act and Assert @@ -103,7 +103,7 @@ def test_example_conversion(self, example_context, requests_mock): # ] # for config_id in config_ids: # model = json.loads(example_context["fetch"](config_id + ".json")) -# requests_mock.get(f"{TDS_URL}/model_configurations/{config_id}", +# requests_mock.get(f"{TDS_URL}/model-configurations/{config_id}", # json=model) # dataset_id = example_context["request"]["dataset"]["id"] @@ -111,7 +111,7 @@ def test_example_conversion(self, example_context, requests_mock): # dataset = example_context["fetch"](filename, True) # dataset_loc = {"method": "GET", "url": dataset} # requests_mock.get( -# f"{TDS_URL}/datasets/{dataset_id}/download-url?filename={filename}", +# f"{TDS_URL}/datasets/{dataset_id}/download-csv?filename={filename}", # json=dataset_loc, # ) # requests_mock.get("http://dataset", text=dataset)