From 16517406ad1623b1575e71f67f0e70debfad61c1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 13:57:19 -0400 Subject: [PATCH 01/48] DOC: Use new URL for docs skipci --- README.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a7835c48..f4e674bbb 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The python SDK for running [Octue](https://octue.com) data services, digital twins, and applications - get faster data groundwork so you have more time for the science! -Read the docs [here.](https://octue-python-sdk.readthedocs.io/en/latest/) +Read the docs [here.](https://twined.octue.com/) Uses our [twined](https://twined.readthedocs.io/en/latest/) library for data validation. diff --git a/pyproject.toml b/pyproject.toml index 63fb88f31..2eb2ee460 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "octue" -version = "0.68.0" +version = "0.68.1" description = "A package providing template applications for data services, and a python SDK to the Octue API." readme = "README.md" authors = ["Marcus Lugg ", "Thomas Clark "] From 969946ea5e409acc143abc6191f478c95bf94911 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 14:55:15 -0400 Subject: [PATCH 02/48] ENH: Update docs URL in CLI help text skipci --- octue/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cli.py b/octue/cli.py index 87b3e4ea0..f6cb78bf9 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -46,7 +46,7 @@ def octue_cli(logger_uri, log_level): """The CLI for Octue SDKs and APIs, most notably Twined. - Read more in the docs: https://octue-python-sdk.readthedocs.io/en/latest/ + Read more in the docs: https://twined.octue.com """ global_cli_context["logger_uri"] = logger_uri global_cli_context["log_handler"] = None From e92baf883ccb3a870d8ce87abdc8d19da383d78e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 17:28:11 -0400 Subject: [PATCH 03/48] REF: Factor out template setting as a class skipci --- octue/definitions.py | 5 ++ octue/twined/templates/template.py | 31 +++++++++++ tests/base.py | 17 +----- tests/twined/templates/test_template_apps.py | 55 ++++++-------------- 4 files changed, 54 insertions(+), 54 deletions(-) create mode 100644 octue/twined/templates/template.py diff --git a/octue/definitions.py b/octue/definitions.py index c036e1f43..88a08f36d 100644 --- a/octue/definitions.py +++ b/octue/definitions.py @@ -1,4 +1,9 @@ import importlib.metadata +import os GOOGLE_COMPUTE_PROVIDERS = {"GOOGLE_CLOUD_FUNCTION"} LOCAL_SDK_VERSION = importlib.metadata.version("octue") + +_root_dir = os.path.dirname(os.path.abspath(__file__)) +TEMPLATES_PATH = os.path.join(_root_dir, "twined", "templates") +DATA_PATH = os.path.join(os.path.dirname(_root_dir), "tests", "data") diff --git a/octue/twined/templates/template.py b/octue/twined/templates/template.py new file mode 100644 index 000000000..106111539 --- /dev/null +++ b/octue/twined/templates/template.py @@ -0,0 +1,31 @@ +import os +import shutil +import sys +import uuid + +from octue.definitions import DATA_PATH, TEMPLATES_PATH + + +class Template: + def __init__(self): + self.start_path = os.getcwd() + self.template_twine = None + self.template_path = None + self.app_test_path = None + self.teardown_templates = [] + + def set_template(self, template): + """Set up the working directory and data paths to run one of the provided templates.""" + self.template_path = os.path.join(TEMPLATES_PATH, template) + self.template_twine = os.path.join(TEMPLATES_PATH, template, "twine.json") + + # Duplicate the template's data/ directory to a test-specific replica + self.app_test_path = os.path.join(DATA_PATH, str(uuid.uuid4())) + shutil.copytree(self.template_path, self.app_test_path) + + # Add this template to the list to remove in teardown + self.teardown_templates.append(self.app_test_path) + sys.path.insert(0, self.app_test_path) + + # Run from within the app folder context + os.chdir(self.app_test_path) diff --git a/tests/base.py b/tests/base.py index 63ccf2b95..139ba6a36 100644 --- a/tests/base.py +++ b/tests/base.py @@ -5,6 +5,7 @@ from octue.cloud import storage from octue.cloud.storage import GoogleCloudStorageClient +from octue.definitions import DATA_PATH from octue.resources import Datafile, Dataset, Manifest from tests import TEST_BUCKET_NAME from tests.twined.base import TestResultModifier @@ -19,23 +20,9 @@ class BaseTestCase(unittest.TestCase): setattr(unittest.TestResult, "startTestRun", test_result_modifier.startTestRun) setattr(unittest.TestResult, "stopTestRun", test_result_modifier.stopTestRun) - def setUp(self): - """Set up the test case by: - - Adding the paths to the test data and app templates directories to the test case - - Making `unittest` ignore excess ResourceWarnings so tests' console outputs are clearer. This has to be done - even if these warnings are ignored elsewhere as unittest forces warnings to be displayed by default. - - :return None: - """ - root_dir = os.path.dirname(os.path.abspath(__file__)) - self.data_path = os.path.join(root_dir, "data") - self.templates_path = os.path.join(os.path.dirname(root_dir), "octue", "twined", "templates") - - super().setUp() - def create_valid_dataset(self, **kwargs): """Create a valid dataset with two valid datafiles (they're the same file in this case).""" - path = os.path.join(self.data_path, "basic_files", "configuration", "test-dataset") + path = os.path.join(DATA_PATH, "basic_files", "configuration", "test-dataset") return Dataset( path=path, diff --git a/tests/twined/templates/test_template_apps.py b/tests/twined/templates/test_template_apps.py index cce2d531e..6a71055d5 100644 --- a/tests/twined/templates/test_template_apps.py +++ b/tests/twined/templates/test_template_apps.py @@ -15,6 +15,7 @@ from octue.twined.cloud.emulators import ChildEmulator from octue.twined.cloud.service_id import create_sruid from octue.twined.runner import Runner +from octue.twined.templates.template import Template from octue.utils.processes import ProcessesContextManager from tests import MOCK_SERVICE_REVISION_TAG, TEST_BUCKET_NAME from tests.base import BaseTestCase @@ -25,11 +26,11 @@ class TemplateAppsTestCase(BaseTestCase): def test_fractal_template_with_default_configuration(self): """Ensure the `fractal` app can be configured with its default configuration and run.""" - self.set_template("template-fractal") + self.template.set_template("template-fractal") runner = Runner( - app_src=self.template_path, - twine=self.template_twine, + app_src=self.template.template_path, + twine=self.template.template_twine, configuration_values=os.path.join("data", "configuration", "configuration_values.json"), ) @@ -37,12 +38,12 @@ def test_fractal_template_with_default_configuration(self): def test_using_manifests_template(self): """Ensure the `using-manifests` template app works correctly.""" - self.set_template("template-using-manifests") + self.template.set_template("template-using-manifests") output_location = f"gs://{TEST_BUCKET_NAME}" runner = Runner( - app_src=self.template_path, - twine=self.template_twine, + app_src=self.template.template_path, + twine=self.template.template_twine, configuration_values=os.path.join("data", "configuration", "values.json"), output_location=output_location, ) @@ -70,9 +71,9 @@ def test_child_services_template(self): parent sends coordinates to both children, receiving the elevation and wind speed from them at these locations. """ cli_path = os.path.join(REPOSITORY_ROOT, "octue", "cli.py") - self.set_template("template-child-services") + self.template.set_template("template-child-services") - elevation_service_path = os.path.join(self.template_path, "elevation_service") + elevation_service_path = os.path.join(self.template.template_path, "elevation_service") elevation_service_revision_tag = str(uuid.uuid4()) elevation_process = subprocess.Popen( @@ -88,7 +89,7 @@ def test_child_services_template(self): env={**os.environ, "OCTUE_SERVICE_REVISION_TAG": elevation_service_revision_tag}, ) - wind_speed_service_path = os.path.join(self.template_path, "wind_speed_service") + wind_speed_service_path = os.path.join(self.template.template_path, "wind_speed_service") wind_speed_service_revision_tag = str(uuid.uuid4()) wind_speed_process = subprocess.Popen( @@ -104,7 +105,7 @@ def test_child_services_template(self): env={**os.environ, "OCTUE_SERVICE_REVISION_TAG": wind_speed_service_revision_tag}, ) - parent_service_path = os.path.join(self.template_path, "parent_service") + parent_service_path = os.path.join(self.template.template_path, "parent_service") namespace = "template-child-services" with open(os.path.join(parent_service_path, "octue.yaml")) as f: @@ -140,8 +141,8 @@ def test_child_services_template(self): def test_child_services_template_using_emulated_children(self): """Test the child services template app using emulated children.""" - self.set_template("template-child-services") - parent_service_path = os.path.join(self.template_path, "parent_service") + self.template.set_template("template-child-services") + parent_service_path = os.path.join(self.template.template_path, "parent_service") with open(os.path.join(parent_service_path, "octue.yaml")) as f: children = yaml.safe_load(f)["services"][0]["children"] @@ -198,36 +199,12 @@ def setUp(self): :return None: """ super().setUp() - self.start_path = os.getcwd() - - # Initialise so these variables are assigned on the instance - self.template_twine = None - self.template_path = None - self.app_test_path = None - self.teardown_templates = [] - - def set_template(template): - """Set up the working directory and data paths to run one of the provided templates.""" - self.template_path = os.path.join(self.templates_path, template) - self.template_twine = os.path.join(self.templates_path, template, "twine.json") - - # Duplicate the template's data/ directory to a test-specific replica - self.app_test_path = os.path.join(self.data_path, str(uuid.uuid4())) - shutil.copytree(self.template_path, self.app_test_path) - - # Add this template to the list to remove in teardown - self.teardown_templates.append(self.app_test_path) - sys.path.insert(0, self.app_test_path) - - # Run from within the app folder context - os.chdir(self.app_test_path) - - self.set_template = set_template + self.template = Template() def tearDown(self): """Remove the temporary template app directories.""" super().tearDown() - os.chdir(self.start_path) - for path in self.teardown_templates: + os.chdir(self.template.start_path) + for path in self.template.teardown_templates: sys.path.remove(path) shutil.rmtree(path) From 6ae0a46c0bd795f65734b85e6bfbe82470b05a88 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 18:07:25 -0400 Subject: [PATCH 04/48] DOC: Update CLI help text --- octue/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octue/cli.py b/octue/cli.py index f6cb78bf9..2144dc98c 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -195,7 +195,7 @@ def remote(sruid, input_values, input_manifest, project_id, asynchronous, servic def local(input_values, input_manifest, attributes, service_config): """Ask a question to a local Octue Twined service. - This command is similar to running `octue twined start` and asking the resulting local service revision a question + This command is similar to running `octue twined service start` and asking the resulting local service revision a question via Pub/Sub. Instead of starting a local Pub/Sub service revision, however, no Pub/Sub subscription or subscriber is created; the question is instead passed directly to local the service revision without Pub/Sub being involved. Everything after this runs the same, though, with the service revision emitting any events via Pub/Sub as usual. From fecabb8ebcb60491075ae9f1f84df15425b81165 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 18:07:46 -0400 Subject: [PATCH 05/48] FEA: Add `octue twined question ask example` command skipci --- octue/cli.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/octue/cli.py b/octue/cli.py index 2144dc98c..94f308594 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -25,6 +25,7 @@ from octue.twined.exceptions import ServiceAlreadyExists from octue.twined.resources import Child, service_backends from octue.twined.runner import Runner +from octue.twined.templates.template import Template from octue.utils.decoders import OctueJSONDecoder from octue.utils.encoders import OctueJSONEncoder @@ -241,6 +242,58 @@ def local(input_values, input_manifest, attributes, service_config): click.echo(json.dumps(answer, cls=OctueJSONEncoder)) +@ask.command() +@click.option( + "-i", + "--input-values", + type=str, + default=None, + help="Any input values for the question, serialised as a JSON-encoded string.", +) +@click.option( + "-m", + "--input-manifest", + type=str, + default=None, + help="An optional input manifest for the question, serialised as a JSON-encoded string.", +) +@click.option( + "-a", + "--attributes", + type=str, + default=None, + help="An optional full set of event attributes for the question, serialised as a JSON-encoded string. If not " + "provided, the question will be an originator question.", +) +@click.option( + "-c", + "--service-config", + type=click.Path(dir_okay=False), + default=None, + help="The path to an `octue.yaml` file defining the service to run. If not provided, the " + "`OCTUE_SERVICE_CONFIGURATION_PATH` environment variable is used if present, otherwise the local path `octue.yaml` " + "is used.", +) +def example(input_values, input_manifest, attributes, service_config): + """Ask a question to a local example Twined service for demonstrating the CLI.""" + template = Template() + template.set_template("template-using-manifests") + + runner = Runner( + app_src=template.template_path, + twine=template.template_twine, + configuration_values=os.path.join(template.template_path, "data", "configuration", "values.json"), + ) + + analysis = runner.run(input_manifest=os.path.join(template.template_path, "data", "input", "manifest.json")) + click.echo( + json.dumps( + {"kind": "result", "output_values": analysis.output_values, "output_manifest": analysis.output_manifest}, + cls=OctueJSONEncoder, + ) + ) + + @question.group() def events(): """Get and replay events from past and current questions.""" diff --git a/pyproject.toml b/pyproject.toml index 2eb2ee460..7d824ee7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "octue" -version = "0.68.1" +version = "0.69.0" description = "A package providing template applications for data services, and a python SDK to the Octue API." readme = "README.md" authors = ["Marcus Lugg ", "Thomas Clark "] From e0cdaf60a562b76dbb321e7fde24b5f875984f4f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 18:14:31 -0400 Subject: [PATCH 06/48] DOC: Simplify installation instructions --- docs/installation.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 0fef71cb0..fb051453d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,26 +2,26 @@ ## Pip +```shell +pip install octue +``` + +or + ```shell pip install octue==x.y.z ``` ## Poetry -Read more about Poetry [here](https://python-poetry.org). - ```shell -poetry add octue=x.y.z +poetry add octue ``` -## Add to your dependencies - -To use a specific version of Twined in your python application, -simply add: +or ```shell -octue==x.y.z +poetry add octue=x.y.z ``` -to your `requirements.txt` or `setup.py` file, where `x.y.z` is your -preferred version of the SDK (we recommend the latest stable version). +Read more about Poetry [here](https://python-poetry.org). From e3206f41d63da510b1268761706faf4caec7e5ec Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 18:23:51 -0400 Subject: [PATCH 07/48] ENH: Add dummy output values to example command --- octue/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/octue/cli.py b/octue/cli.py index 94f308594..243f4e253 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -286,9 +286,11 @@ def example(input_values, input_manifest, attributes, service_config): ) analysis = runner.run(input_manifest=os.path.join(template.template_path, "data", "input", "manifest.json")) + output_values = {"some": "output", "heights": [1, 2, 3, 4, 5]} + click.echo( json.dumps( - {"kind": "result", "output_values": analysis.output_values, "output_manifest": analysis.output_manifest}, + {"kind": "result", "output_values": output_values, "output_manifest": analysis.output_manifest}, cls=OctueJSONEncoder, ) ) From ec1cb3598ea4f2dd03058beaa01c055498642b8c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 18:27:11 -0400 Subject: [PATCH 08/48] FIX: Clean up after template class skipci --- octue/cli.py | 2 ++ octue/twined/templates/template.py | 6 ++++++ tests/twined/templates/test_template_apps.py | 6 +----- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 243f4e253..6421b6e47 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -295,6 +295,8 @@ def example(input_values, input_manifest, attributes, service_config): ) ) + template.cleanup() + @question.group() def events(): diff --git a/octue/twined/templates/template.py b/octue/twined/templates/template.py index 106111539..7e2645a71 100644 --- a/octue/twined/templates/template.py +++ b/octue/twined/templates/template.py @@ -29,3 +29,9 @@ def set_template(self, template): # Run from within the app folder context os.chdir(self.app_test_path) + + def cleanup(self): + os.chdir(self.start_path) + for path in self.teardown_templates: + sys.path.remove(path) + shutil.rmtree(path) diff --git a/tests/twined/templates/test_template_apps.py b/tests/twined/templates/test_template_apps.py index 6a71055d5..e53bf010e 100644 --- a/tests/twined/templates/test_template_apps.py +++ b/tests/twined/templates/test_template_apps.py @@ -1,5 +1,4 @@ import os -import shutil import subprocess import sys import time @@ -204,7 +203,4 @@ def setUp(self): def tearDown(self): """Remove the temporary template app directories.""" super().tearDown() - os.chdir(self.template.start_path) - for path in self.template.teardown_templates: - sys.path.remove(path) - shutil.rmtree(path) + self.template.cleanup() From 7cc9ad8e47f67e7221db8dc0709203d0cc927ad2 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 23 Sep 2025 18:29:17 -0400 Subject: [PATCH 09/48] DOC: Add getting started guide to index page of docs skipci --- docs/index.md | 79 ++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/docs/index.md b/docs/index.md index e10ad91e5..05913d6ba 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,59 +1,54 @@ -# Introduction +# Getting started -The python SDK for [Octue](https://octue.com) Twined scientific data services and digital twins - get faster data -groundwork so you have more time for the science! +Create and use data services with Twined, a framework from Octue. -!!! info "Definition" +This guide walks you through getting started with Twined to use an existing data service deployed in Google Cloud. - **Twined service** +By the end, you will be able to use the Twined CLI to run an analysis on an example data service, sending it input data and receiving its output. - A data service or digital twin built with the Twined framework that can be asked questions, process them, and - return answers. Twined services can communicate with each other with minimal extra setup. +## Prerequisites -## Key features +Before you begin, ensure you: -**Unified cloud/local file, dataset, and manifest operations** +- Are familiar with Python +- Have the following tools installed: + - Python >= 3.10 + - The `octue` python library (see [installation instructions](installation.md)) -- Create and build datasets easily -- Organise them with timestamps, labels, and tags -- Filter and combine them using this metadata -- Store them locally or in the cloud (or both for low-latency reading/writing with cloud-guaranteed data availability) -- Use internet/cloud-based datasets as if they were local e.g. - - `https://example.com/important_dataset.dat` - - `gs://example-bucket/important_dataset.dat` -- Create manifests (a set of datasets needed for a particular analysis) to modularise your dataset input/output +## Authentication -**Ask existing services questions from anywhere** +No authentication is needed to run the example data service. To authenticate for real data services, see [here](authentication.md). -- Send them data to process from anywhere -- Automatically have their logs, monitor messages, and any errors forwarded to you and displayed as if they were local -- Receive their output data as JSON -- Receive a manifest of any output datasets they produce for you to download or access as you wish +## Run your first analysis -**Create, run, and deploy your apps as services** +### Request -- No need to change your app - just wrap it -- Use the `octue` CLI to run your service locally or deploy it to Google Kubernetes Engine (GKE) -- Create JSON-schema interfaces to explicitly define the form of configuration, input, and output data -- Ask other services questions as part of your app (i.e. build trees of services) -- Automatically display readable, colourised logs, or use your own log handler -- Avoid time-consuming and confusing devops, cloud configuration, and backend maintenance +The following command runs an analysis in the local example data service: -**High standards, quick responses, and good intentions** +```shell +octue twined question ask example --input-values='{"some": "data"}' +``` -- Open-source and transparent on GitHub - anyone can see the code and raise an issue -- Automated testing, standards, releases, and deployment -- High test coverage -- Works on MacOS, Linux, and Windows -- Developed not-for-profit for the renewable energy industry +### Response -## Need help, found a bug, or want to request a new feature? +The response contains log messages in `stderr` followed by the result as JSON in `stdout`: -We use [GitHub Issues](https://github.com/octue/octue-sdk-python/issues) to manage: +```text +[2025-09-23 18:20:13,513 | WARNING | octue.resources.dataset] is empty at instantiation time (path 'cleaned_met_mast_data'). +[2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Starting clean up of files in +[2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Averaging window set to 600s +[2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis didn't produce output values. +[2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis produced an output manifest. +[2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] No output location was set in the service configuration - can't upload output datasets. +[2025-09-23 18:20:13,726 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] Validated outputs against the twine. +{"kind": "result", "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}} +``` -- Bug reports -- Feature requests -- Support requests +## Next steps -Bug reports, feature requests and support requests, may also be made directly to your Octue support contact, or via the -[support pages](https://www.octue.com/contact). +Congratulations on running your first analysis! For additional information, check out the following resources: + +- {Link to other relevant documentation such as API Reference} +- {Link to other features that are available in the API} +- {Provide links to additional tutorials and articles about the API} +- {Provide links to community and support groups, FAQs, troubleshooting guides, etc.} From 87a251ca2bc784b6a9649933f0df6af873b91587 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 13:23:58 -0400 Subject: [PATCH 10/48] DOC: Add command for real data service to getting started --- docs/index.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index 05913d6ba..b826bc07f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,10 +1,11 @@ # Getting started -Create and use data services with Twined, a framework from Octue. +Twined is a python framework for creating and using data services and digital twins. -This guide walks you through getting started with Twined to use an existing data service deployed in Google Cloud. +This guide walks you through using an example Twined data service locally (the process for using a real one is almost +identical). -By the end, you will be able to use the Twined CLI to run an analysis on an example data service, sending it input data and receiving its output. +By the end, you will be able to use the Twined CLI to run an analysis on a data service, sending it input data and receiving its output. ## Prerequisites @@ -23,12 +24,20 @@ No authentication is needed to run the example data service. To authenticate for ### Request -The following command runs an analysis in the local example data service: +The following shell command runs an analysis in the local example data service: ```shell octue twined question ask example --input-values='{"some": "data"}' ``` +!!! info + + To ask a question to a real data service, the command is almost the same: + + ```shell + octue twined question ask remote --input-values='{"some": "data"}' + ``` + ### Response The response contains log messages in `stderr` followed by the result as JSON in `stdout`: From 4492e0dd8872419327050093bdd80c6bf3d9e423 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 13:28:31 -0400 Subject: [PATCH 11/48] DOC: Fix nested bulleted list --- docs/index.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index b826bc07f..c69264cb4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,10 +11,14 @@ By the end, you will be able to use the Twined CLI to run an analysis on a data Before you begin, ensure you: + + - Are familiar with Python - Have the following tools installed: - - Python >= 3.10 - - The `octue` python library (see [installation instructions](installation.md)) + - Python >= 3.10 + - The `octue` python library (see [installation instructions](installation.md)) + + ## Authentication From 764ba2817ac0b887c331505d7f1478bcffc0ce29 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 13:32:35 -0400 Subject: [PATCH 12/48] DOC: Adjust wording --- docs/index.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index c69264cb4..e9da78e76 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,10 +2,11 @@ Twined is a python framework for creating and using data services and digital twins. -This guide walks you through using an example Twined data service locally (the process for using a real one is almost -identical). +This guide walks you through using an example Twined service locally. The process for using a real one is almost +identical. -By the end, you will be able to use the Twined CLI to run an analysis on a data service, sending it input data and receiving its output. +By the end, you will be able to use the Twined CLI to run an analysis on a data service, sending it input data and +receiving output data. ## Prerequisites @@ -22,7 +23,8 @@ Before you begin, ensure you: ## Authentication -No authentication is needed to run the example data service. To authenticate for real data services, see [here](authentication.md). +No authentication is needed to run the example data service. To authenticate for real data services, see +[authentication instructions](authentication.md). ## Run your first analysis From 679c47beca0e06df6eb70bc6f043c54e579120c3 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 13:35:36 -0400 Subject: [PATCH 13/48] DOC: Explain question/answer terminology --- docs/index.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/index.md b/docs/index.md index e9da78e76..1708ab850 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,15 +28,20 @@ No authentication is needed to run the example data service. To authenticate for ## Run your first analysis -### Request +!!! info + + In Twined, sending input data to a service is called "asking a question". The service will run an analysis on the + question and send back any output data - this is called called "receiving an answer". -The following shell command runs an analysis in the local example data service: +### Send a question + +The following shell command asks a question to the local example data service: ```shell octue twined question ask example --input-values='{"some": "data"}' ``` -!!! info +!!! tip To ask a question to a real data service, the command is almost the same: @@ -44,9 +49,9 @@ octue twined question ask example --input-values='{"some": "data"}' octue twined question ask remote --input-values='{"some": "data"}' ``` -### Response +### Receive an answer -The response contains log messages in `stderr` followed by the result as JSON in `stdout`: +The output from asking a question usually contains log messages in `stderr` followed by the answer as JSON in `stdout`: ```text [2025-09-23 18:20:13,513 | WARNING | octue.resources.dataset] is empty at instantiation time (path 'cleaned_met_mast_data'). From 134b375045c2f98a610a062eeceb1e0f0f889fce Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 13:43:34 -0400 Subject: [PATCH 14/48] DOC: Add shell redirection tip and add JSON link --- docs/index.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 1708ab850..d2e94fee9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -51,7 +51,7 @@ octue twined question ask example --input-values='{"some": "data"}' ### Receive an answer -The output from asking a question usually contains log messages in `stderr` followed by the answer as JSON in `stdout`: +The output from asking a question usually contains log messages followed by the answer as [JSON](https://en.wikipedia.org/wiki/JSON): ```text [2025-09-23 18:20:13,513 | WARNING | octue.resources.dataset] is empty at instantiation time (path 'cleaned_met_mast_data'). @@ -64,6 +64,18 @@ The output from asking a question usually contains log messages in `stderr` foll {"kind": "result", "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}} ``` +!!! tip + + You can pipe the output JSON into other CLI tools or redirect it to a file: + + ```shell + # Fromat the result using the `jq` command line tool + octue twined question ask example --input-values='{"some": "data"}' | jq + + # Store the result in a file + octue twined question ask example --input-values='{"some": "data"}' > result.json + ``` + ## Next steps Congratulations on running your first analysis! For additional information, check out the following resources: From a02fb8ce0d6994d615637d422f00f2c07e3e5d33 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 13:46:42 -0400 Subject: [PATCH 15/48] DOC: Simplify installation instructions skipci --- docs/installation.md | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index fb051453d..f3548d281 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,27 +1,44 @@ # Installation +The Twined framework is provided through the [Octue SDK](https://github.com/octue/octue-sdk-python). + ## Pip ```shell pip install octue ``` -or - -```shell -pip install octue==x.y.z -``` - ## Poetry +Read more about Poetry [here](https://python-poetry.org). + ```shell poetry add octue ``` -or +## Check installation + +If the installation worked correctly, the `octue` CLI will be available: ```shell -poetry add octue=x.y.z +octue --help ``` -Read more about Poetry [here](https://python-poetry.org). +```text +Usage: octue [OPTIONS] COMMAND [ARGS]... + + The CLI for Octue SDKs and APIs, most notably Twined. + + Read more in the docs: https://twined.octue.com + +Options: + --logger-uri TEXT Stream logs to a websocket at the given URI. + --log-level [debug|info|warning|error] + Log level used for the analysis. [default: + info] + --version Show the version and exit. + -h, --help Show this message and exit. + +Commands: + twined The Twined CLI. +``` From 8ae7fb5c83db97b65d960274925367e6f5823b37 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 13:50:30 -0400 Subject: [PATCH 16/48] REF: Remove `--logger-uri` option from CLI BREAKING CHANGE: Stop providing the `--logger-uri` option to the CLI --- README.md | 1 - docs/installation.md | 1 - octue/cli.py | 12 ++---------- octue/log_handlers.py | 38 -------------------------------------- tests/test_log_handlers.py | 36 +----------------------------------- 5 files changed, 3 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index f4e674bbb..83d5876c1 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,6 @@ Usage: octue [OPTIONS] COMMAND [ARGS]... Read more in the docs: https://octue-python-sdk.readthedocs.io/en/latest/ Options: - --logger-uri TEXT Stream logs to a websocket at the given URI. --log-level [debug|info|warning|error] Log level used for the analysis. [default: info] diff --git a/docs/installation.md b/docs/installation.md index f3548d281..20c3a9297 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -32,7 +32,6 @@ Usage: octue [OPTIONS] COMMAND [ARGS]... Read more in the docs: https://twined.octue.com Options: - --logger-uri TEXT Stream logs to a websocket at the given URI. --log-level [debug|info|warning|error] Log level used for the analysis. [default: info] diff --git a/octue/cli.py b/octue/cli.py index 6421b6e47..fd26c79be 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -11,7 +11,7 @@ from octue.cloud import storage from octue.cloud.storage import GoogleCloudStorageClient from octue.definitions import LOCAL_SDK_VERSION -from octue.log_handlers import apply_log_handler, get_remote_handler +from octue.log_handlers import apply_log_handler from octue.resources import Manifest from octue.twined.cloud.events.answer_question import answer_question from octue.twined.cloud.events.question import make_question_event @@ -35,7 +35,6 @@ @click.group(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option("--logger-uri", default=None, show_default=True, help="Stream logs to a websocket at the given URI.") @click.option( "--log-level", default="info", @@ -44,20 +43,14 @@ help="Log level used for the analysis.", ) @click.version_option(version=LOCAL_SDK_VERSION) -def octue_cli(logger_uri, log_level): +def octue_cli(log_level): """The CLI for Octue SDKs and APIs, most notably Twined. Read more in the docs: https://twined.octue.com """ - global_cli_context["logger_uri"] = logger_uri - global_cli_context["log_handler"] = None global_cli_context["log_level"] = log_level.upper() - apply_log_handler(log_level=log_level.upper()) - if global_cli_context["logger_uri"]: - global_cli_context["log_handler"] = get_remote_handler(logger_uri=global_cli_context["logger_uri"]) - @octue_cli.group() def twined(): @@ -694,7 +687,6 @@ def start(service_config, revision_tag, timeout, no_rm): run_function = functools.partial( runner.run, analysis_log_level=global_cli_context["log_level"], - analysis_log_handler=global_cli_context["log_handler"], ) backend_configuration_values = (service_configuration.configuration_values or {}).get("backend") diff --git a/octue/log_handlers.py b/octue/log_handlers.py index 85658947a..b30a1a08f 100644 --- a/octue/log_handlers.py +++ b/octue/log_handlers.py @@ -1,7 +1,6 @@ import logging import logging.handlers import os -from urllib.parse import urlparse from octue.definitions import GOOGLE_COMPUTE_PROVIDERS @@ -150,43 +149,6 @@ def apply_log_handler( return handler -def get_remote_handler( - logger_uri, - formatter=None, - include_line_number=False, - include_process_name=False, - include_thread_name=False, -): - """Get a log handler for streaming logs to a remote URI accessed via HTTP or HTTPS. The default octue log formatter - is used if no formatter is provided. - - :param str logger_uri: the URI to stream the logs to - :param logging.Formatter|None formatter: if provided, this formatter is used and the other formatting options are ignored - :param bool include_line_number: if `True`, include the line number in the log context - :param bool include_process_name: if `True`, include the process name in the log context - :param bool include_thread_name: if `True`, include the thread name in the log context - :return logging.Handler: - """ - parsed_uri = urlparse(logger_uri) - - if parsed_uri.scheme not in {"ws", "wss"}: - raise ValueError( - f"Only WS and WSS protocols currently supported for remote logger URI. Received {logger_uri!r}." - ) - - handler = logging.handlers.SocketHandler(host=parsed_uri.hostname, port=parsed_uri.port) - - formatter = formatter or create_octue_formatter( - get_log_record_attributes_for_environment(), - include_line_number=include_line_number, - include_process_name=include_process_name, - include_thread_name=include_thread_name, - ) - - handler.setFormatter(formatter) - return handler - - def get_log_record_attributes_for_environment(): """Get the correct log record attributes for the environment. If the environment is in Google Cloud, get log record attributes not including the timestamp in the log context to avoid the date appearing twice in the Google Cloud diff --git a/tests/test_log_handlers.py b/tests/test_log_handlers.py index 300f7081d..95674a964 100644 --- a/tests/test_log_handlers.py +++ b/tests/test_log_handlers.py @@ -1,14 +1,9 @@ import importlib -import logging import os import sys from unittest import mock -from octue.log_handlers import ( - LOG_RECORD_ATTRIBUTES_WITH_TIMESTAMP, - LOG_RECORD_ATTRIBUTES_WITHOUT_TIMESTAMP, - get_remote_handler, -) +from octue.log_handlers import LOG_RECORD_ATTRIBUTES_WITH_TIMESTAMP, LOG_RECORD_ATTRIBUTES_WITHOUT_TIMESTAMP from tests.base import BaseTestCase @@ -83,32 +78,3 @@ def test_extra_log_record_attributes_are_included_if_relevant_environment_variab include_process_name=True, include_thread_name=True, ) - - -class TestGetRemoteHandler(BaseTestCase): - def test_get_remote_handler_parses_ws_properly(self): - """Assert that the remote log handler parses URIs properly.""" - handler = get_remote_handler(logger_uri="ws://0.0.0.1:3000") - assert handler.host == "0.0.0.1" - assert handler.port == 3000 - - def test_wss_is_supported(self): - """Test that HTTPS is supported by the remote log handler.""" - handler = get_remote_handler(logger_uri="wss://0.0.0.1:3000/log") - assert handler.host == "0.0.0.1" - assert handler.port == 3000 - - def test_non_ws_or_wss_protocol_raises_error(self): - """Ensure an error is raised if a protocol other than HTTP or HTTPS is used for the logger URI.""" - with self.assertRaises(ValueError): - get_remote_handler(logger_uri="https://0.0.0.1:3000/log") - - def test_remote_logger_emits_messages(self): - """Test that the remote log handler emits messages.""" - logger = logging.getLogger("test-logger") - logger.addHandler(get_remote_handler(logger_uri="wss://0.0.0.0:80")) - logger.setLevel("DEBUG") - - with mock.patch("logging.handlers.SocketHandler.emit") as mock_emit: - logger.debug("Hello") - mock_emit.assert_called() From ff3f054ef14ffa5dcf830756af9452b9447f2178 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 14:02:03 -0400 Subject: [PATCH 17/48] CHO: Remove `check-yaml` pre-commit check --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb490c791..50e960cd7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,6 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer - - id: check-yaml - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit From b5aae2ed2c50fca61459e2c548e1c3fa916be2e6 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 14:02:17 -0400 Subject: [PATCH 18/48] DOC: Enable tabbed docs --- mkdocs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 8d9eff6b6..9323e61c1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -51,6 +51,7 @@ theme: - search.suggest - search.highlight - search.share + - content.tabs.link palette: # Palette toggle for light mode @@ -74,6 +75,11 @@ markdown_extensions: - attr_list - md_in_html - pymdownx.blocks.caption + - pymdownx.tabbed: + alternate_style: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower plugins: - privacy From 7709f7d6d7ff3094248fab592fcbb79cdfc7696e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 14:02:51 -0400 Subject: [PATCH 19/48] DOC: Add CLI tab to getting started guide --- docs/index.md | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/docs/index.md b/docs/index.md index d2e94fee9..092231a8e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,7 +17,7 @@ Before you begin, ensure you: - Are familiar with Python - Have the following tools installed: - Python >= 3.10 - - The `octue` python library (see [installation instructions](installation.md)) + - The `octue` python library / CLI (see [installation instructions](installation.md)) @@ -37,9 +37,11 @@ No authentication is needed to run the example data service. To authenticate for The following shell command asks a question to the local example data service: -```shell -octue twined question ask example --input-values='{"some": "data"}' -``` +=== "CLI" + + ```shell + octue twined question ask example --input-values='{"some": "data"}' + ``` !!! tip @@ -51,30 +53,32 @@ octue twined question ask example --input-values='{"some": "data"}' ### Receive an answer -The output from asking a question usually contains log messages followed by the answer as [JSON](https://en.wikipedia.org/wiki/JSON): +=== "CLI" -```text -[2025-09-23 18:20:13,513 | WARNING | octue.resources.dataset] is empty at instantiation time (path 'cleaned_met_mast_data'). -[2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Starting clean up of files in -[2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Averaging window set to 600s -[2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis didn't produce output values. -[2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis produced an output manifest. -[2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] No output location was set in the service configuration - can't upload output datasets. -[2025-09-23 18:20:13,726 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] Validated outputs against the twine. -{"kind": "result", "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}} -``` + The output from asking a question usually contains log messages followed by the answer as [JSON](https://en.wikipedia.org/wiki/JSON): -!!! tip + ```text + [2025-09-23 18:20:13,513 | WARNING | octue.resources.dataset] is empty at instantiation time (path 'cleaned_met_mast_data'). + [2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Starting clean up of files in + [2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Averaging window set to 600s + [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis didn't produce output values. + [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis produced an output manifest. + [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] No output location was set in the service configuration - can't upload output datasets. + [2025-09-23 18:20:13,726 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] Validated outputs against the twine. + {"kind": "result", "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}} + ``` - You can pipe the output JSON into other CLI tools or redirect it to a file: + !!! tip - ```shell - # Fromat the result using the `jq` command line tool - octue twined question ask example --input-values='{"some": "data"}' | jq + You can pipe the output JSON into other CLI tools or redirect it to a file: - # Store the result in a file - octue twined question ask example --input-values='{"some": "data"}' > result.json - ``` + ```shell + # Fromat the result using the `jq` command line tool + octue twined question ask example --input-values='{"some": "data"}' | jq + + # Store the result in a file + octue twined question ask example --input-values='{"some": "data"}' > result.json + ``` ## Next steps From a65147216d7a835b5e80cd78bd5355e4b67aa90d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 14:04:50 -0400 Subject: [PATCH 20/48] DOC: Add empty python tab to getting started guide skipci --- docs/index.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 092231a8e..ae0356e9d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -43,12 +43,18 @@ The following shell command asks a question to the local example data service: octue twined question ask example --input-values='{"some": "data"}' ``` -!!! tip + !!! tip + + To ask a question to a real data service, the command is almost the same: + + ```shell + octue twined question ask remote --input-values='{"some": "data"}' + ``` - To ask a question to a real data service, the command is almost the same: +=== "Python" ```shell - octue twined question ask remote --input-values='{"some": "data"}' + ``` ### Receive an answer @@ -80,6 +86,12 @@ The following shell command asks a question to the local example data service: octue twined question ask example --input-values='{"some": "data"}' > result.json ``` +=== "Python" + + ```shell + + ``` + ## Next steps Congratulations on running your first analysis! For additional information, check out the following resources: From a7ad869ef4aa6b8c866fc37fbfe0c430ce1f459c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 14:26:03 -0400 Subject: [PATCH 21/48] DOC: Add python code to getting started guide skipci --- docs/index.md | 56 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/docs/index.md b/docs/index.md index ae0356e9d..6c37a3877 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,7 +14,7 @@ Before you begin, ensure you: -- Are familiar with Python +- Are familiar with Python and/or the command line - Have the following tools installed: - Python >= 3.10 - The `octue` python library / CLI (see [installation instructions](installation.md)) @@ -33,9 +33,9 @@ No authentication is needed to run the example data service. To authenticate for In Twined, sending input data to a service is called "asking a question". The service will run an analysis on the question and send back any output data - this is called called "receiving an answer". -### Send a question +### Ask a question -The following shell command asks a question to the local example data service: +The following command asks a question to the local example data service. === "CLI" @@ -45,23 +45,57 @@ The following shell command asks a question to the local example data service: !!! tip - To ask a question to a real data service, the command is almost the same: + To ask a question to a real data service, use the `remote` subcommand and specify the ID of the service: ```shell - octue twined question ask remote --input-values='{"some": "data"}' + octue twined question ask remote some-org/a-service:1.2.0 --input-values='{"some": "data"}' ``` === "Python" - ```shell + !!! info + + A child is a Twined service you ask a question to (in the sense of child and parent nodes in a tree; this only + becomes important when services use other Twined services as part of their analysis). + + ```python + from octue.twined.resources import Child + child = Child( + id="local/example:latest", + backend={ + "name": "GCPPubSubBackend", + "project_id": "example-project", + }, + ) + + answer, question_uuid = child.ask(input_values={"some": "data"}) ``` + !!! tip + + To ask a question to a real data service, just specify its ID and project ID: + + ```python + from octue.twined.resources import Child + + child = Child( + id="some-org/a-service:1.2.0", + backend={ + "name": "GCPPubSubBackend", + "project_id": "some-org", + }, + ) + + answer, question_uuid = child.ask(input_values={"some": "data"}) + ``` + ### Receive an answer === "CLI" - The output from asking a question usually contains log messages followed by the answer as [JSON](https://en.wikipedia.org/wiki/JSON): + The output is automatically written to the command line. It contains log messages followed by the answer as + [JSON](https://en.wikipedia.org/wiki/JSON): ```text [2025-09-23 18:20:13,513 | WARNING | octue.resources.dataset] is empty at instantiation time (path 'cleaned_met_mast_data'). @@ -88,8 +122,14 @@ The following shell command asks a question to the local example data service: === "Python" - ```shell + ```python + answer + >>> { + "kind": "result", + "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, + "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}, + } ``` ## Next steps From 9efdc80525311f9c597f3ccc39ab6c09d13389ad Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 24 Sep 2025 14:32:51 -0400 Subject: [PATCH 22/48] DOC: Enable code snippet copy buttons --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 9323e61c1..920b0018e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -52,6 +52,7 @@ theme: - search.highlight - search.share - content.tabs.link + - content.code.copy palette: # Palette toggle for light mode From 8f75681735688dc2c0a0bc898071eb1be1ca8f99 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 11:11:47 -0400 Subject: [PATCH 23/48] DOC: Adjust wording and fix typo --- docs/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 6c37a3877..fa398fea3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,8 +2,8 @@ Twined is a python framework for creating and using data services and digital twins. -This guide walks you through using an example Twined service locally. The process for using a real one is almost -identical. +This guide walks you through using an example Twined service locally. The process for using a real one (deployed locally +or in the cloud) is almost identical. By the end, you will be able to use the Twined CLI to run an analysis on a data service, sending it input data and receiving output data. @@ -113,7 +113,7 @@ The following command asks a question to the local example data service. You can pipe the output JSON into other CLI tools or redirect it to a file: ```shell - # Fromat the result using the `jq` command line tool + # Format the result using the `jq` command line tool octue twined question ask example --input-values='{"some": "data"}' | jq # Store the result in a file From 4eb926ca35950600147d7cb4e111c660323adbcd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 11:16:00 -0400 Subject: [PATCH 24/48] DOC: Add "next steps" to end of getting started guide --- docs/index.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index fa398fea3..402e21d94 100644 --- a/docs/index.md +++ b/docs/index.md @@ -136,7 +136,8 @@ The following command asks a question to the local example data service. Congratulations on running your first analysis! For additional information, check out the following resources: -- {Link to other relevant documentation such as API Reference} -- {Link to other features that are available in the API} -- {Provide links to additional tutorials and articles about the API} -- {Provide links to community and support groups, FAQs, troubleshooting guides, etc.} +- Set up infrastructure for a data service(s) using Terraform +- Create a data service +- Run a data service locally +- See the library and CLI reference +- Get support From 3f3877eaf709e2eb078759d19444e1a76a192233 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 11:21:27 -0400 Subject: [PATCH 25/48] DOC: Adjust wording and style on licence and version history pages skipci --- docs/license.md | 10 +++++----- docs/version_history.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/license.md b/docs/license.md index 8fdf9f723..f2f47bcad 100644 --- a/docs/license.md +++ b/docs/license.md @@ -1,10 +1,10 @@ # License -## The Boring Bit +## The boring bit -See [the octue-sdk-python license](https://github.com/octue/octue-sdk-python/blob/main/LICENSE). +See the Twined license [here](https://github.com/octue/octue-sdk-python/blob/main/LICENSE). -## Third Party Libraries +## Third party libraries -**octue-sdk-python** includes or is linked against code from third party -libraries - see [our attributions page](https://github.com/octue/octue-sdk-python/blob/main/ATTRIBUTIONS.md). +**Twined** includes or is linked against code from third party libraries - see +[our attributions page](https://github.com/octue/octue-sdk-python/blob/main/ATTRIBUTIONS.md). diff --git a/docs/version_history.md b/docs/version_history.md index 8068e4209..31bcacf2a 100644 --- a/docs/version_history.md +++ b/docs/version_history.md @@ -1,4 +1,4 @@ -# Version History {#chapter-version-history} +# Version history See our [releases on GitHub.](https://github.com/octue/octue-sdk-python/releases) From 7e8a3bbbf3ba7f69c30fc77ae39d2ef8fa40dcf5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 11:47:00 -0400 Subject: [PATCH 26/48] FEA: Add `ExampleChild` and use in CLI --- octue/cli.py | 19 +++------- octue/twined/resources/example.py | 59 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 14 deletions(-) create mode 100644 octue/twined/resources/example.py diff --git a/octue/cli.py b/octue/cli.py index fd26c79be..4c7072fba 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -24,8 +24,8 @@ from octue.twined.definitions import MANIFEST_FILENAME, VALUES_FILENAME from octue.twined.exceptions import ServiceAlreadyExists from octue.twined.resources import Child, service_backends +from octue.twined.resources.example import ExampleChild from octue.twined.runner import Runner -from octue.twined.templates.template import Template from octue.utils.decoders import OctueJSONDecoder from octue.utils.encoders import OctueJSONEncoder @@ -269,27 +269,18 @@ def local(input_values, input_manifest, attributes, service_config): ) def example(input_values, input_manifest, attributes, service_config): """Ask a question to a local example Twined service for demonstrating the CLI.""" - template = Template() - template.set_template("template-using-manifests") + example_child = ExampleChild() - runner = Runner( - app_src=template.template_path, - twine=template.template_twine, - configuration_values=os.path.join(template.template_path, "data", "configuration", "values.json"), - ) - - analysis = runner.run(input_manifest=os.path.join(template.template_path, "data", "input", "manifest.json")) - output_values = {"some": "output", "heights": [1, 2, 3, 4, 5]} + with example_child: + analysis = example_child.ask() click.echo( json.dumps( - {"kind": "result", "output_values": output_values, "output_manifest": analysis.output_manifest}, + {"kind": "result", "output_values": analysis.output_values, "output_manifest": analysis.output_manifest}, cls=OctueJSONEncoder, ) ) - template.cleanup() - @question.group() def events(): diff --git a/octue/twined/resources/example.py b/octue/twined/resources/example.py new file mode 100644 index 000000000..d67ca4bdd --- /dev/null +++ b/octue/twined/resources/example.py @@ -0,0 +1,59 @@ +import os + +from octue.twined.definitions import DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL +from octue.twined.resources import Child +from octue.twined.runner import Runner +from octue.twined.templates.template import Template + + +class ExampleChild(Child): + def __init__(self): + self.id = "local/example:latest" + self.template = Template() + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + self.template.cleanup() + + def ask( + self, + input_values=None, + input_manifest=None, + children=None, + subscribe_to_logs=True, + allow_local_files=False, + handle_monitor_message=None, + record_events=True, + save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", + question_uuid=None, + parent_question_uuid=None, + originator_question_uuid=None, + originator=None, + push_endpoint=None, + asynchronous=False, + retry_count=0, + cpus=None, + memory=None, + ephemeral_storage=None, + raise_errors=True, + max_retries=0, + prevent_retries_when=None, + log_errors=True, + timeout=86400, + maximum_heartbeat_interval=DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL, + ): + self.template.set_template("template-using-manifests") + + runner = Runner( + app_src=self.template.template_path, + twine=self.template.template_twine, + configuration_values=os.path.join(self.template.template_path, "data", "configuration", "values.json"), + ) + + analysis = runner.run( + input_manifest=os.path.join(self.template.template_path, "data", "input", "manifest.json") + ) + analysis.output_values = {"some": "output", "heights": [1, 2, 3, 4, 5]} + return analysis From 3a7c97ceff333b6c317b15d1448745f30e84484e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 11:49:03 -0400 Subject: [PATCH 27/48] DOC: Use `ExampleChild` in getting started guide skipci --- docs/index.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/index.md b/docs/index.md index 402e21d94..a250f0742 100644 --- a/docs/index.md +++ b/docs/index.md @@ -59,22 +59,16 @@ The following command asks a question to the local example data service. becomes important when services use other Twined services as part of their analysis). ```python - from octue.twined.resources import Child - - child = Child( - id="local/example:latest", - backend={ - "name": "GCPPubSubBackend", - "project_id": "example-project", - }, - ) + from octue.twined.resources.example import ExampleChild + child = ExampleChild() answer, question_uuid = child.ask(input_values={"some": "data"}) ``` !!! tip - To ask a question to a real data service, just specify its ID and project ID: + To ask a question to a real data service, use `Child` instead of `ExampleChild` and specify the service's ID and + project ID: ```python from octue.twined.resources import Child From 5a3c10795e46236daa83b0141b019ecf1284fc16 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 16:57:14 -0400 Subject: [PATCH 28/48] DOC: Add infrastructure page to docs skipci --- docs/infrastructure.md | 30 ++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 31 insertions(+) create mode 100644 docs/infrastructure.md diff --git a/docs/infrastructure.md b/docs/infrastructure.md new file mode 100644 index 000000000..92d69eb11 --- /dev/null +++ b/docs/infrastructure.md @@ -0,0 +1,30 @@ +Twined data services can be run locally, but deploying them in the cloud is generally preferable for reliability and +performance. Twined makes it easy to do this with two [Terraform](https://terraform.io) modules (ready-made +infrastructure as code / IaC) that deploy a managed Kubernetes cluster to run services on. The combined infrastructure +is called a **Twined services network**. + +This guide walks you through setting up a Twined services network - the infrastructure needed to deploy a Twined +service. It only takes a few commands. + +## Prerequisites + +Before you begin, ensure you: + + + +- Are familiar with Terraform/IaC and the command line +- Have the following set up: + - [Terraform CLI](https://developer.hashicorp.com/terraform/install) >= 1.8.0 installed + - A Google Cloud account and project with billing set up + - Optionally, a [Terraform Cloud](https://app.terraform.io/public/signup/account?product_intent=terraform) account + + + +## Spin up infrastructure + +Follow the instructions [here](https://github.com/octue/terraform-octue-twined-core), making sure to follow up with the +[second step](https://github.com/octue/terraform-octue-twined-cluster) mentioned in the "Important" box at the top. + +## Next steps + +Congratulations on setting up a Twined services network! Next up, create your first data service. diff --git a/mkdocs.yml b/mkdocs.yml index 920b0018e..04e0e0bf6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,6 +4,7 @@ site_url: https://twined.octue.com nav: - index.md - installation.md + - Setting up infrastructure: infrastructure.md - Datafiles, datasets, and manifests: - data_containers/index.md - data_containers/datafile.md From 6f74b5afc7dd58a42b546ffce40b2416e084addc Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 17:04:07 -0400 Subject: [PATCH 29/48] DOC: Add support page --- docs/support.md | 8 ++++++++ mkdocs.yml | 1 + 2 files changed, 9 insertions(+) create mode 100644 docs/support.md diff --git a/docs/support.md b/docs/support.md new file mode 100644 index 000000000..412a8d522 --- /dev/null +++ b/docs/support.md @@ -0,0 +1,8 @@ +Need help, found a bug, or want to request a new feature? We use GitHub Issues to manage: + +- Feature requests +- Bug reports +- Support requests + +Need something else? Contact us [here](https://octue.com/contact). You can also directly message your Octue support +contact. diff --git a/mkdocs.yml b/mkdocs.yml index 04e0e0bf6..0e449f823 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,6 +34,7 @@ nav: # - api.md - license.md - version_history.md + - support.md theme: name: material From a45e7a67d87696b548efec3ad78e30b1616f859d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 17:05:03 -0400 Subject: [PATCH 30/48] DOC: Add links to getting started guide skipci --- docs/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index a250f0742..7466bc02e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -130,8 +130,8 @@ The following command asks a question to the local example data service. Congratulations on running your first analysis! For additional information, check out the following resources: -- Set up infrastructure for a data service(s) using Terraform +- [Set up infrastructure for a data service(s) using Terraform](infrastructure.md) - Create a data service - Run a data service locally - See the library and CLI reference -- Get support +- [Get support](support.md) From eadb77cdc3e666e09f76122e193e6be4f65f58ec Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 26 Sep 2025 17:50:38 -0400 Subject: [PATCH 31/48] DOC: Add link to github issues skipci --- docs/support.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/support.md b/docs/support.md index 412a8d522..6eae4df4a 100644 --- a/docs/support.md +++ b/docs/support.md @@ -1,4 +1,5 @@ -Need help, found a bug, or want to request a new feature? We use GitHub Issues to manage: +Need help, found a bug, or want to request a new feature? We use +[GitHub Issues](https://github.com/octue/octue-sdk-python/issues) to manage: - Feature requests - Bug reports From b1b59728b167fe30a7e8843bc5fc0e7ae989afa8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 14:53:34 -0400 Subject: [PATCH 32/48] ENH: Merge `example` CLI subcommand into `remote` subcommand skipci --- docs/index.md | 8 +++---- octue/cli.py | 65 ++++++++++++++------------------------------------- 2 files changed, 22 insertions(+), 51 deletions(-) diff --git a/docs/index.md b/docs/index.md index 7466bc02e..c82e7fb71 100644 --- a/docs/index.md +++ b/docs/index.md @@ -40,12 +40,12 @@ The following command asks a question to the local example data service. === "CLI" ```shell - octue twined question ask example --input-values='{"some": "data"}' + octue twined question ask remote example/service:latest --input-values='{"some": "data"}' ``` !!! tip - To ask a question to a real data service, use the `remote` subcommand and specify the ID of the service: + To ask a question to a real data service, just specify its ID: ```shell octue twined question ask remote some-org/a-service:1.2.0 --input-values='{"some": "data"}' @@ -108,10 +108,10 @@ The following command asks a question to the local example data service. ```shell # Format the result using the `jq` command line tool - octue twined question ask example --input-values='{"some": "data"}' | jq + octue twined question ask remote example/service:latest --input-values='{"some": "data"}' | jq # Store the result in a file - octue twined question ask example --input-values='{"some": "data"}' > result.json + octue twined question ask remote example/service:latest --input-values='{"some": "data"}' > result.json ``` === "Python" diff --git a/octue/cli.py b/octue/cli.py index 4c7072fba..5effd6dab 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -114,6 +114,24 @@ def remote(sruid, input_values, input_manifest, project_id, asynchronous, servic octue question ask remote your-org/example-service:1.2.0 """ + if sruid.startswith("example/"): + example_child = ExampleChild() + + with example_child: + analysis = example_child.ask() + + click.echo( + json.dumps( + { + "kind": "result", + "output_values": analysis.output_values, + "output_manifest": analysis.output_manifest, + }, + cls=OctueJSONEncoder, + ) + ) + return + service_configuration = ServiceConfiguration.from_file(service_config, allow_not_found=True) if service_configuration: @@ -235,53 +253,6 @@ def local(input_values, input_manifest, attributes, service_config): click.echo(json.dumps(answer, cls=OctueJSONEncoder)) -@ask.command() -@click.option( - "-i", - "--input-values", - type=str, - default=None, - help="Any input values for the question, serialised as a JSON-encoded string.", -) -@click.option( - "-m", - "--input-manifest", - type=str, - default=None, - help="An optional input manifest for the question, serialised as a JSON-encoded string.", -) -@click.option( - "-a", - "--attributes", - type=str, - default=None, - help="An optional full set of event attributes for the question, serialised as a JSON-encoded string. If not " - "provided, the question will be an originator question.", -) -@click.option( - "-c", - "--service-config", - type=click.Path(dir_okay=False), - default=None, - help="The path to an `octue.yaml` file defining the service to run. If not provided, the " - "`OCTUE_SERVICE_CONFIGURATION_PATH` environment variable is used if present, otherwise the local path `octue.yaml` " - "is used.", -) -def example(input_values, input_manifest, attributes, service_config): - """Ask a question to a local example Twined service for demonstrating the CLI.""" - example_child = ExampleChild() - - with example_child: - analysis = example_child.ask() - - click.echo( - json.dumps( - {"kind": "result", "output_values": analysis.output_values, "output_manifest": analysis.output_manifest}, - cls=OctueJSONEncoder, - ) - ) - - @question.group() def events(): """Get and replay events from past and current questions.""" From 2d56154360b86bbc28ac0b0ad9c6c4b10dbe227e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 15:09:16 -0400 Subject: [PATCH 33/48] FIX: Return correct values from `ExampleChild.ask` --- octue/cli.py | 15 +++------------ octue/twined/resources/example.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 5effd6dab..012375879 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -118,18 +118,9 @@ def remote(sruid, input_values, input_manifest, project_id, asynchronous, servic example_child = ExampleChild() with example_child: - analysis = example_child.ask() - - click.echo( - json.dumps( - { - "kind": "result", - "output_values": analysis.output_values, - "output_manifest": analysis.output_manifest, - }, - cls=OctueJSONEncoder, - ) - ) + answer, _ = example_child.ask() + + click.echo(json.dumps(answer, cls=OctueJSONEncoder)) return service_configuration = ServiceConfiguration.from_file(service_config, allow_not_found=True) diff --git a/octue/twined/resources/example.py b/octue/twined/resources/example.py index d67ca4bdd..b39eacf36 100644 --- a/octue/twined/resources/example.py +++ b/octue/twined/resources/example.py @@ -1,4 +1,5 @@ import os +import uuid from octue.twined.definitions import DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL from octue.twined.resources import Child @@ -55,5 +56,13 @@ def ask( analysis = runner.run( input_manifest=os.path.join(self.template.template_path, "data", "input", "manifest.json") ) + analysis.output_values = {"some": "output", "heights": [1, 2, 3, 4, 5]} - return analysis + + answer = { + "kind": "result", + "output_values": analysis.output_values, + "output_manifest": analysis.output_manifest, + } + + return answer, str(uuid.uuid4()) From 28439565a31e60c2cef191f821b28feaad0c8dbd Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 15:11:02 -0400 Subject: [PATCH 34/48] REF: Remove unnecessary context manager from `ExampleChild` --- octue/cli.py | 5 +---- octue/twined/resources/example.py | 7 +------ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index 012375879..c9043f240 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -116,10 +116,7 @@ def remote(sruid, input_values, input_manifest, project_id, asynchronous, servic """ if sruid.startswith("example/"): example_child = ExampleChild() - - with example_child: - answer, _ = example_child.ask() - + answer, _ = example_child.ask() click.echo(json.dumps(answer, cls=OctueJSONEncoder)) return diff --git a/octue/twined/resources/example.py b/octue/twined/resources/example.py index b39eacf36..0f529e552 100644 --- a/octue/twined/resources/example.py +++ b/octue/twined/resources/example.py @@ -12,12 +12,6 @@ def __init__(self): self.id = "local/example:latest" self.template = Template() - def __enter__(self): - pass - - def __exit__(self, exc_type, exc_val, exc_tb): - self.template.cleanup() - def ask( self, input_values=None, @@ -57,6 +51,7 @@ def ask( input_manifest=os.path.join(self.template.template_path, "data", "input", "manifest.json") ) + self.template.cleanup() analysis.output_values = {"some": "output", "heights": [1, 2, 3, 4, 5]} answer = { From a8d675beb8438cdadcff80bd87b4a5ccc957cc86 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 15:18:04 -0400 Subject: [PATCH 35/48] ENH: Give `ExampleChild` same signature as `Child` --- docs/index.md | 12 +++++++++--- octue/cli.py | 2 +- octue/twined/resources/example.py | 5 ++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/index.md b/docs/index.md index c82e7fb71..0fa912c19 100644 --- a/docs/index.md +++ b/docs/index.md @@ -61,14 +61,20 @@ The following command asks a question to the local example data service. ```python from octue.twined.resources.example import ExampleChild - child = ExampleChild() + child = ExampleChild( + id="example/service:latest", + backend={ + "name": "GCPPubSubBackend", + "project_id": "example", + }, + ) answer, question_uuid = child.ask(input_values={"some": "data"}) ``` !!! tip - To ask a question to a real data service, use `Child` instead of `ExampleChild` and specify the service's ID and - project ID: + To ask a question to a real data service, use `Child` instead of `ExampleChild` and specify the service's + ID and project ID: ```python from octue.twined.resources import Child diff --git a/octue/cli.py b/octue/cli.py index c9043f240..8bcad29d4 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -115,7 +115,7 @@ def remote(sruid, input_values, input_manifest, project_id, asynchronous, servic octue question ask remote your-org/example-service:1.2.0 """ if sruid.startswith("example/"): - example_child = ExampleChild() + example_child = ExampleChild("", {}) answer, _ = example_child.ask() click.echo(json.dumps(answer, cls=OctueJSONEncoder)) return diff --git a/octue/twined/resources/example.py b/octue/twined/resources/example.py index 0f529e552..56a0893b1 100644 --- a/octue/twined/resources/example.py +++ b/octue/twined/resources/example.py @@ -8,9 +8,8 @@ class ExampleChild(Child): - def __init__(self): - self.id = "local/example:latest" - self.template = Template() + def __init__(self, id, backend): + pass def ask( self, From dcaff9bbfd35b3c75220dd51c2f34d156a52a035 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 15:18:46 -0400 Subject: [PATCH 36/48] REF: Use analysis ID in example child and simplify template variable --- octue/twined/resources/example.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/octue/twined/resources/example.py b/octue/twined/resources/example.py index 56a0893b1..d2c4d2150 100644 --- a/octue/twined/resources/example.py +++ b/octue/twined/resources/example.py @@ -1,5 +1,4 @@ import os -import uuid from octue.twined.definitions import DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL from octue.twined.resources import Child @@ -38,19 +37,18 @@ def ask( timeout=86400, maximum_heartbeat_interval=DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL, ): - self.template.set_template("template-using-manifests") + template = Template() + template.set_template("template-using-manifests") runner = Runner( - app_src=self.template.template_path, - twine=self.template.template_twine, - configuration_values=os.path.join(self.template.template_path, "data", "configuration", "values.json"), + app_src=template.template_path, + twine=template.template_twine, + configuration_values=os.path.join(template.template_path, "data", "configuration", "values.json"), ) - analysis = runner.run( - input_manifest=os.path.join(self.template.template_path, "data", "input", "manifest.json") - ) + analysis = runner.run(input_manifest=os.path.join(template.template_path, "data", "input", "manifest.json")) - self.template.cleanup() + template.cleanup() analysis.output_values = {"some": "output", "heights": [1, 2, 3, 4, 5]} answer = { @@ -59,4 +57,4 @@ def ask( "output_manifest": analysis.output_manifest, } - return answer, str(uuid.uuid4()) + return answer, analysis.id From fc8737168ab7eba125ca255de8e4df5229f56d1e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 15:19:32 -0400 Subject: [PATCH 37/48] REF: Simplify `ExampleChild.__init__` skipci --- octue/twined/resources/example.py | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/octue/twined/resources/example.py b/octue/twined/resources/example.py index d2c4d2150..9c3d17f10 100644 --- a/octue/twined/resources/example.py +++ b/octue/twined/resources/example.py @@ -1,6 +1,5 @@ import os -from octue.twined.definitions import DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL from octue.twined.resources import Child from octue.twined.runner import Runner from octue.twined.templates.template import Template @@ -10,33 +9,7 @@ class ExampleChild(Child): def __init__(self, id, backend): pass - def ask( - self, - input_values=None, - input_manifest=None, - children=None, - subscribe_to_logs=True, - allow_local_files=False, - handle_monitor_message=None, - record_events=True, - save_diagnostics="SAVE_DIAGNOSTICS_ON_CRASH", - question_uuid=None, - parent_question_uuid=None, - originator_question_uuid=None, - originator=None, - push_endpoint=None, - asynchronous=False, - retry_count=0, - cpus=None, - memory=None, - ephemeral_storage=None, - raise_errors=True, - max_retries=0, - prevent_retries_when=None, - log_errors=True, - timeout=86400, - maximum_heartbeat_interval=DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL, - ): + def ask(self, *args, **kwargs): template = Template() template.set_template("template-using-manifests") From 3c5626abac2eb8658e0000beaff7c780d98ed71e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 15:27:47 -0400 Subject: [PATCH 38/48] ENH: Raise error if "example" is used as a service namespace BREAKING CHANGE: Stop using "example" as a service namespace --- octue/twined/configuration.py | 9 ++++++++- octue/twined/definitions.py | 1 + tests/twined/test_configuration.py | 11 +++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/twined/test_configuration.py diff --git a/octue/twined/configuration.py b/octue/twined/configuration.py index d13e883fa..5c0b7f1ae 100644 --- a/octue/twined/configuration.py +++ b/octue/twined/configuration.py @@ -3,6 +3,8 @@ import yaml +from octue.twined.definitions import DISALLOWED_SERVICE_NAMESPACES + logger = logging.getLogger(__name__) @@ -49,7 +51,12 @@ def __init__( **kwargs, ): self.name = name - self.namespace = namespace + + if namespace in DISALLOWED_SERVICE_NAMESPACES: + raise ValueError(f"{namespace!r} is not an allowed Twined service namespace.") + else: + self.namespace = namespace + self.diagnostics_cloud_path = diagnostics_cloud_path self.service_registries = service_registries self.event_store_table_id = event_store_table_id diff --git a/octue/twined/definitions.py b/octue/twined/definitions.py index 95cbb49dc..3190537f5 100644 --- a/octue/twined/definitions.py +++ b/octue/twined/definitions.py @@ -3,3 +3,4 @@ OUTPUT_STRANDS = ("output_values", "output_manifest") RUN_STRANDS = ("input_values", "input_manifest", "credentials", "children") DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL = 360 +DISALLOWED_SERVICE_NAMESPACES = {"example"} diff --git a/tests/twined/test_configuration.py b/tests/twined/test_configuration.py new file mode 100644 index 000000000..f4bfd653a --- /dev/null +++ b/tests/twined/test_configuration.py @@ -0,0 +1,11 @@ +from octue.twined.configuration import ServiceConfiguration +from tests.base import BaseTestCase + + +class TestServiceConfiguration(BaseTestCase): + def test_error_raised_if_namespace_disallowed(self): + """Test that an error is raised if a disallowed namespace is given.""" + with self.assertRaises(ValueError) as error_context: + ServiceConfiguration(namespace="example", name="service") + + self.assertEqual(error_context.exception.args[0], "'example' is not an allowed Twined service namespace.") From 965535a57336cb2d4906b3c020d86b20d4ecd6b8 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 16:16:11 -0400 Subject: [PATCH 39/48] ENH: Replace `ExampleChild` with `Child` with "example" namespace --- docs/index.md | 23 ++++------------- octue/cli.py | 5 ++-- octue/twined/resources/child.py | 4 +++ octue/twined/resources/example.py | 41 ++++++++++++++----------------- 4 files changed, 30 insertions(+), 43 deletions(-) diff --git a/docs/index.md b/docs/index.md index 0fa912c19..92c524e62 100644 --- a/docs/index.md +++ b/docs/index.md @@ -59,36 +59,23 @@ The following command asks a question to the local example data service. becomes important when services use other Twined services as part of their analysis). ```python - from octue.twined.resources.example import ExampleChild + from octue.twined.resources import Child - child = ExampleChild( + child = Child( id="example/service:latest", backend={ "name": "GCPPubSubBackend", "project_id": "example", }, ) + answer, question_uuid = child.ask(input_values={"some": "data"}) ``` !!! tip - To ask a question to a real data service, use `Child` instead of `ExampleChild` and specify the service's - ID and project ID: - - ```python - from octue.twined.resources import Child - - child = Child( - id="some-org/a-service:1.2.0", - backend={ - "name": "GCPPubSubBackend", - "project_id": "some-org", - }, - ) - - answer, question_uuid = child.ask(input_values={"some": "data"}) - ``` + To ask a question to a real data service, specify its ID and project ID e.g. `some-org/real-service:1.2.0` + instead of `example/service:latest`. ### Receive an answer diff --git a/octue/cli.py b/octue/cli.py index 8bcad29d4..d04f687eb 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -24,7 +24,7 @@ from octue.twined.definitions import MANIFEST_FILENAME, VALUES_FILENAME from octue.twined.exceptions import ServiceAlreadyExists from octue.twined.resources import Child, service_backends -from octue.twined.resources.example import ExampleChild +from octue.twined.resources.example import run_example from octue.twined.runner import Runner from octue.utils.decoders import OctueJSONDecoder from octue.utils.encoders import OctueJSONEncoder @@ -115,8 +115,7 @@ def remote(sruid, input_values, input_manifest, project_id, asynchronous, servic octue question ask remote your-org/example-service:1.2.0 """ if sruid.startswith("example/"): - example_child = ExampleChild("", {}) - answer, _ = example_child.ask() + answer, _ = run_example() click.echo(json.dumps(answer, cls=OctueJSONEncoder)) return diff --git a/octue/twined/resources/child.py b/octue/twined/resources/child.py index ea311c163..02ea16944 100644 --- a/octue/twined/resources/child.py +++ b/octue/twined/resources/child.py @@ -6,6 +6,7 @@ from octue.twined.cloud.pub_sub.service import Service from octue.twined.definitions import DEFAULT_MAXIMUM_HEARTBEAT_INTERVAL from octue.twined.resources import service_backends +from octue.twined.resources.example import run_example logger = logging.getLogger(__name__) @@ -113,6 +114,9 @@ def ask( :raise Exception: if the question raises an error and `raise_errors=True` :return dict|octue.twined.cloud.pub_sub.subscription.Subscription|Exception|None, str: for a synchronous question, a dictionary containing the keys "output_values" and "output_manifest" from the result (or just an exception if the question fails), and the question UUID; for a question with a push endpoint, the push subscription and the question UUID; for an asynchronous question, `None` and the question UUID """ + if self.id.startswith("example/"): + return run_example() + prevent_retries_when = prevent_retries_when or [] inputs = { diff --git a/octue/twined/resources/example.py b/octue/twined/resources/example.py index 9c3d17f10..50da9752c 100644 --- a/octue/twined/resources/example.py +++ b/octue/twined/resources/example.py @@ -1,33 +1,30 @@ import os -from octue.twined.resources import Child -from octue.twined.runner import Runner from octue.twined.templates.template import Template -class ExampleChild(Child): - def __init__(self, id, backend): - pass +def run_example(): + # Avoid circular import. + from octue.twined.runner import Runner - def ask(self, *args, **kwargs): - template = Template() - template.set_template("template-using-manifests") + template = Template() + template.set_template("template-using-manifests") - runner = Runner( - app_src=template.template_path, - twine=template.template_twine, - configuration_values=os.path.join(template.template_path, "data", "configuration", "values.json"), - ) + runner = Runner( + app_src=template.template_path, + twine=template.template_twine, + configuration_values=os.path.join(template.template_path, "data", "configuration", "values.json"), + ) - analysis = runner.run(input_manifest=os.path.join(template.template_path, "data", "input", "manifest.json")) + analysis = runner.run(input_manifest=os.path.join(template.template_path, "data", "input", "manifest.json")) - template.cleanup() - analysis.output_values = {"some": "output", "heights": [1, 2, 3, 4, 5]} + template.cleanup() + analysis.output_values = {"some": "output", "heights": [1, 2, 3, 4, 5]} - answer = { - "kind": "result", - "output_values": analysis.output_values, - "output_manifest": analysis.output_manifest, - } + answer = { + "kind": "result", + "output_values": analysis.output_values, + "output_manifest": analysis.output_manifest, + } - return answer, analysis.id + return answer, analysis.id From cafbd459ebe92a79fee6c184c73e9dc438c8845f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 16:18:21 -0400 Subject: [PATCH 40/48] DOC: Move admonition and clarify it skipci --- docs/index.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/index.md b/docs/index.md index 92c524e62..15eca833a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,11 +53,6 @@ The following command asks a question to the local example data service. === "Python" - !!! info - - A child is a Twined service you ask a question to (in the sense of child and parent nodes in a tree; this only - becomes important when services use other Twined services as part of their analysis). - ```python from octue.twined.resources import Child @@ -72,6 +67,11 @@ The following command asks a question to the local example data service. answer, question_uuid = child.ask(input_values={"some": "data"}) ``` + !!! info + + A child is a Twined service you ask a question to, in the sense of child and parent nodes in a tree. This only + becomes important when services use other Twined services as part of their analysis, forming a tree of services. + !!! tip To ask a question to a real data service, specify its ID and project ID e.g. `some-org/real-service:1.2.0` From 6bb9d4bb2fd8912844a046f8d3dd1f8eead974a1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Wed, 1 Oct 2025 16:23:29 -0400 Subject: [PATCH 41/48] REF: Un-nest CLI commands for asking questions BREAKING CHANGE: Use: - `octue twined question ask` instead of `octue twined question ask remote` - `octue twined question ask-local` instead of `octue twined question ask local` --- octue/cli.py | 13 ++++-------- .../dockerfiles/Dockerfile-python310 | 2 +- .../dockerfiles/Dockerfile-python311 | 2 +- .../dockerfiles/Dockerfile-python312 | 2 +- .../dockerfiles/Dockerfile-python313 | 2 +- tests/test_cli.py | 21 +++++-------------- 6 files changed, 13 insertions(+), 29 deletions(-) diff --git a/octue/cli.py b/octue/cli.py index d04f687eb..8ddb79dff 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -63,12 +63,7 @@ def question(): """Ask a new question to an Octue Twined data service or interact with a previous question.""" -@question.group() -def ask(): - """Ask a new question to an Octue Twined data service.""" - - -@ask.command() +@question.command() @click.argument("sruid", type=str) @click.option( "-i", @@ -107,7 +102,7 @@ def ask(): "`OCTUE_SERVICE_CONFIGURATION_PATH` environment variable is used if present, otherwise the local path `octue.yaml` " "is used.", ) -def remote(sruid, input_values, input_manifest, project_id, asynchronous, service_config): +def ask(sruid, input_values, input_manifest, project_id, asynchronous, service_config): """Ask a question to a remote Octue Twined service. SRUID should be a valid service revision unique identifier for an existing Octue Twined service e.g. @@ -159,7 +154,7 @@ def remote(sruid, input_values, input_manifest, project_id, asynchronous, servic click.echo(json.dumps(answer, cls=OctueJSONEncoder)) -@ask.command() +@question.command() @click.option( "-i", "--input-values", @@ -191,7 +186,7 @@ def remote(sruid, input_values, input_manifest, project_id, asynchronous, servic "`OCTUE_SERVICE_CONFIGURATION_PATH` environment variable is used if present, otherwise the local path `octue.yaml` " "is used.", ) -def local(input_values, input_manifest, attributes, service_config): +def ask_local(input_values, input_manifest, attributes, service_config): """Ask a question to a local Octue Twined service. This command is similar to running `octue twined service start` and asking the resulting local service revision a question diff --git a/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python310 b/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python310 index 4e92e9443..21049fb13 100644 --- a/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python310 +++ b/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python310 @@ -36,4 +36,4 @@ RUN if [ -f "pyproject.toml" ]; then poetry install --only main; \ ENV USE_OCTUE_LOG_HANDLER=1 ENV COMPUTE_PROVIDER=GOOGLE_KUEUE -CMD ["octue", "twined", "question", "ask", "local"] +CMD ["octue", "twined", "question", "ask-local"] diff --git a/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python311 b/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python311 index 8cd98448b..1e6c5a64f 100644 --- a/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python311 +++ b/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python311 @@ -36,4 +36,4 @@ RUN if [ -f "pyproject.toml" ]; then poetry install --only main; \ ENV USE_OCTUE_LOG_HANDLER=1 ENV COMPUTE_PROVIDER=GOOGLE_KUEUE -CMD ["octue", "twined", "question", "ask", "local"] +CMD ["octue", "twined", "question", "ask-local"] diff --git a/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python312 b/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python312 index 750ea49e5..2430425c5 100644 --- a/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python312 +++ b/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python312 @@ -36,4 +36,4 @@ RUN if [ -f "pyproject.toml" ]; then poetry install --only main; \ ENV USE_OCTUE_LOG_HANDLER=1 ENV COMPUTE_PROVIDER=GOOGLE_KUEUE -CMD ["octue", "twined", "question", "ask", "local"] +CMD ["octue", "twined", "question", "ask-local"] diff --git a/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python313 b/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python313 index 03be44435..8cef4183c 100644 --- a/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python313 +++ b/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python313 @@ -36,4 +36,4 @@ RUN if [ -f "pyproject.toml" ]; then poetry install --only main; \ ENV USE_OCTUE_LOG_HANDLER=1 ENV COMPUTE_PROVIDER=GOOGLE_KUEUE -CMD ["octue", "twined", "question", "ask", "local"] +CMD ["octue", "twined", "question", "ask-local"] diff --git a/tests/test_cli.py b/tests/test_cli.py index 1ae688d6b..a02816072 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -57,7 +57,6 @@ def test_with_input_values(self): "twined", "question", "ask", - "remote", self.SRUID, '--input-values={"height": 3}', ], @@ -78,7 +77,6 @@ def test_with_input_manifest(self): "twined", "question", "ask", - "remote", self.SRUID, f"--input-manifest={input_manifest.serialise()}", ], @@ -100,7 +98,6 @@ def test_with_input_values_and_manifest(self): "twined", "question", "ask", - "remote", self.SRUID, f"--input-values={json.dumps(input_values)}", f"--input-manifest={input_manifest.serialise()}", @@ -123,7 +120,6 @@ def test_with_output_manifest(self): "twined", "question", "ask", - "remote", self.SRUID, f"--input-values={json.dumps({'height': 3})}", ], @@ -145,7 +141,6 @@ def test_asynchronous(self): "twined", "question", "ask", - "remote", self.SRUID, '--input-values={"height": 3}', "--asynchronous", @@ -164,7 +159,6 @@ def test_with_no_service_configuration(self): "twined", "question", "ask", - "remote", self.SRUID, '--input-values={"height": 3}', ], @@ -186,8 +180,7 @@ def test_with_input_values(self): [ "twined", "question", - "ask", - "local", + "ask-local", '--input-values={"height": 3}', ], ) @@ -225,8 +218,7 @@ def test_with_input_manifest(self): [ "twined", "question", - "ask", - "local", + "ask-local", f"--input-manifest={input_manifest.serialise()}", ], ) @@ -266,8 +258,7 @@ def test_with_input_values_and_manifest(self): [ "twined", "question", - "ask", - "local", + "ask-local", f"--input-values={json.dumps(input_values)}", f"--input-manifest={input_manifest.serialise()}", ], @@ -306,8 +297,7 @@ def test_with_output_manifest(self): [ "twined", "question", - "ask", - "local", + "ask-local", f"--input-values={json.dumps({'height': 3})}", ], ) @@ -347,8 +337,7 @@ def test_with_attributes(self): [ "twined", "question", - "ask", - "local", + "ask-local", '--input-values={"height": 3}', f"--attributes={json.dumps(original_attributes)}", ], From f49c81c269e32a3ae4fb3b116d0cbe78810c0ba2 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 2 Oct 2025 15:37:57 -0400 Subject: [PATCH 42/48] DOC: Remove old documentation from navigation --- mkdocs.yml | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 0e449f823..b7eb8e0a8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,36 +2,9 @@ site_name: Octue Twined site_url: https://twined.octue.com nav: - - index.md - installation.md + - index.md - Setting up infrastructure: infrastructure.md - - Datafiles, datasets, and manifests: - - data_containers/index.md - - data_containers/datafile.md - - data_containers/dataset.md - - data_containers/manifest.md - - services.md - - asking_questions.md - - creating_services.md - - Twines: - - twines/index.md - - Quickstart: twines/quick_start_create_your_first_twine.md - - twines/anatomy.md - - About Twines: - - twines/about/index.md - - twines/about/about_digital_twins.md - - twines/about/about_requirements.md - - twines/about/about_introducing_json_schema.md - - twines/about/about_other_considerations.md - - updating_services.md - - running_services_locally.md - - deploying_services.md - - testing_services.md - - troubleshooting_services.md - - logging.md - - authentication.md - - inter_service_compatibility.md - # - api.md - license.md - version_history.md - support.md From 60d797f2369930dbd5d097584e4a8b5a847eabd4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 2 Oct 2025 15:39:59 -0400 Subject: [PATCH 43/48] DOC: Remove old documentation skipci --- docs/asking_questions.md | 417 ----------------- docs/authentication.md | 53 --- docs/available_filters.md | 110 ----- docs/creating_apps.md | 167 ------- docs/creating_services.md | 156 ------- docs/data_containers/datafile.md | 312 ------------- docs/data_containers/dataset.md | 230 ---------- docs/data_containers/index.md | 10 - docs/data_containers/manifest.md | 114 ----- docs/deploying_services.md | 79 ---- docs/downloading_datafiles.md | 33 -- docs/index.md | 8 +- docs/inter_service_compatibility.md | 77 ---- docs/logging.md | 55 --- docs/running_services_locally.md | 72 --- docs/services.md | 125 ------ docs/testing_services.md | 290 ------------ docs/troubleshooting_services.md | 131 ------ docs/twines/about/about_digital_twins.md | 53 --- .../about/about_introducing_json_schema.md | 126 ------ .../about/about_other_considerations.md | 112 ----- docs/twines/about/about_requirements.md | 37 -- docs/twines/about/index.md | 16 - docs/twines/anatomy.md | 109 ----- docs/twines/anatomy_children.md | 9 - docs/twines/anatomy_credentials.md | 83 ---- docs/twines/anatomy_manifest.md | 420 ------------------ docs/twines/anatomy_monitors.md | 57 --- docs/twines/anatomy_values.md | 157 ------- docs/twines/examples.md | 200 --------- docs/twines/index.md | 109 ----- docs/twines/lifecycle.md | 34 -- docs/twines/quick_start.md | 5 - .../quick_start_create_your_first_twine.md | 112 ----- docs/updating_services.md | 109 ----- 35 files changed, 4 insertions(+), 4183 deletions(-) delete mode 100644 docs/asking_questions.md delete mode 100644 docs/authentication.md delete mode 100644 docs/available_filters.md delete mode 100644 docs/creating_apps.md delete mode 100644 docs/creating_services.md delete mode 100644 docs/data_containers/datafile.md delete mode 100644 docs/data_containers/dataset.md delete mode 100644 docs/data_containers/index.md delete mode 100644 docs/data_containers/manifest.md delete mode 100644 docs/deploying_services.md delete mode 100644 docs/downloading_datafiles.md delete mode 100644 docs/inter_service_compatibility.md delete mode 100644 docs/logging.md delete mode 100644 docs/running_services_locally.md delete mode 100644 docs/services.md delete mode 100644 docs/testing_services.md delete mode 100644 docs/troubleshooting_services.md delete mode 100644 docs/twines/about/about_digital_twins.md delete mode 100644 docs/twines/about/about_introducing_json_schema.md delete mode 100644 docs/twines/about/about_other_considerations.md delete mode 100644 docs/twines/about/about_requirements.md delete mode 100644 docs/twines/about/index.md delete mode 100644 docs/twines/anatomy.md delete mode 100644 docs/twines/anatomy_children.md delete mode 100644 docs/twines/anatomy_credentials.md delete mode 100644 docs/twines/anatomy_manifest.md delete mode 100644 docs/twines/anatomy_monitors.md delete mode 100644 docs/twines/anatomy_values.md delete mode 100644 docs/twines/examples.md delete mode 100644 docs/twines/index.md delete mode 100644 docs/twines/lifecycle.md delete mode 100644 docs/twines/quick_start.md delete mode 100644 docs/twines/quick_start_create_your_first_twine.md delete mode 100644 docs/updating_services.md diff --git a/docs/asking_questions.md b/docs/asking_questions.md deleted file mode 100644 index e1d78224e..000000000 --- a/docs/asking_questions.md +++ /dev/null @@ -1,417 +0,0 @@ -# Asking services questions - -## What is a question? - -A question is a set of data (input values and/or an input manifest) sent -to a child for processing/analysis. Questions can be: - -- **Synchronous ("ask-and-wait"):** A question whose answer is waited - for in real time - -- **Asynchronous ("fire-and-forget"):** A question whose answer is not - waited for and is instead retrieved later. There are two types: - - - **Regular:** Responses to these questions are automatically stored - in an event store where they can be - [retrieved using the Twined CLI](#retrieving-answers-to-asynchronous-questions) - - - **Push endpoint:** Responses to these questions are pushed to an - HTTP endpoint for asynchronous handling using Octue's - [django-twined](https://django-twined.readthedocs.io/en/latest/) - or custom logic in your own webserver. - -Questions are always asked to a _revision_ of a service. You can ask a -service a question if you have its [SRUID](/services/#service-names), project ID, and the necessary permissions. - -## Asking a question - -```python -from octue.twined.resources import Child - -child = Child( - id="my-organisation/my-service:2.1.7", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, -) - -answer, question_uuid = child.ask( - input_values={"height": 32, "width": 3}, - input_manifest=manifest, -) - -answer["output_values"] ->>> {"some": "data"} - -answer["output_manifest"]["my_dataset"].files ->>> , })> -``` - -!!! warning - - If you're using an environment other than the `main` environment, then before asking any questions to your Twined - services, set the `TWINED_SERVICES_TOPIC_NAME` environment variable to the name of the Twined services Pub/Sub topic - (this is set when [deploying a service network](/deploying_services/#deploying-services-advanced-developers-guide). - It will be in the form `.octue.twined.services` - -!!! note - - Not including a service revision tag will cause your question to be sent to the default revision of the service - (usually the latest deployed version). This is determined by making a request to a - [service registry](https://django-twined.readthedocs.io/en/latest/) if one or more - [registries are defined](#using-a-service-registry). If none of the service registries contain an entry for this - service, a specific service revision tag must be used. - -You can also set the following options when you call `Child.ask`: - -- `children` - If the child has children of its own (i.e. grandchildren of the parent), this optional argument can be - used to override the child's "default" children. This allows you to specify particular versions of grandchildren to - use (see [this subsection below](#overriding-a-childs-children). -- `subscribe_to_logs` - if true, the child will forward its logs to you -- `allow_local_files` - if true, local files/datasets are allowed in any input manifest you supply -- `handle_monitor_message` - if provided a function, it will be called on any monitor messages from the child -- `record_events` -- if true, events received from the parent while it processes the question are saved to the - `Child.received_events` property -- `save_diagnostics` - must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if - turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just - if the analysis fails -- `question_uuid` - if provided, the question will use this UUID instead of a generated one -- `push_endpoint` - if provided, the result and other events produced during the processing of the question will be - pushed to this HTTP endpoint (a URL) -- `asynchronous` - if true, don't wait for an answer to the question (the result and other events can be - [retrieved from the event store later](#retrieving-answers-to-asynchronous-questions) -- `cpus` - the number of CPUs to request for the question; defaults to the number set by the child service -- `memory` - the amount of memory to request for the question e.g. "256Mi" or "1Gi"; defaults to the amount set by the - child service -- `ephemeral_storage` - the amount of ephemeral storage to request for the question e.g. "256Mi" or "1Gi"; defaults to - the amount set by the child service -- `timeout` - how long in seconds to wait for an answer (`None` by default - i.e. don't time out) - -If the question fails: - -- If `raise_errors=False`, the unraised error is returned -- If `raise_errors=False` and `max_retries > 0`, the question is retried up to this number of times -- If `raise_errors=False`, `max_retries > 0`, and `prevent_retries_when` is a list of exception types, the question is - retried unless the error type is in the list -- If `raise_errors=False`, `log_errors=True`, and the question fails after its final retry, the error is logged - -### Exceptions raised by a child - -If a child raises an exception while processing your question, the -exception will always be forwarded and re-raised in your local service -or python session. You can handle exceptions in whatever way you like. - -### Timeouts - -If setting a timeout, bear in mind that the question has to reach the -child, the child has to run its analysis on the inputs sent to it (this -will most likely make up the dominant part of the wait time), and the -answer has to be sent back to the parent. If you're not sure how long a -particular analysis might take, it's best to set the timeout to `None` -initially or ask the owner/maintainer of the child for an estimate. - -## Retrieving answers to asynchronous questions - -To retrieve results and other events from the processing of a question -later, make sure you have the permissions to access the event store and -run: - -```python -from octue.twined.cloud.pub_sub.bigquery import get_events - -events = get_events(question_uuid="53353901-0b47-44e7-9da3-a3ed59990a71") -``` - -**Options** - -- `table_id` - If you're not using the standard deployment, you can - specify a different table here -- `question_uuid` - Retrieve events from this specific question -- `parent_question_uuid` - Retrieve events from questions triggered by - the same parent question (this doesn't include the parent question's - events) -- `originator_question_uuid` - Retrieve events for the entire tree of - questions triggered by an originator question (a question asked - manually through `Child.ask`; this does include the originator - question's events) -- `kind` - Only retrieve this kind of event if present (e.g. "result") -- `include_backend_metadata` - If `True`, retrieve information about the - service backend that produced the event -- `limit` - If set to a positive integer, limit the number of events - returned to this - -!!! note - - Only one of `question_uuid`, `parent_question_uuid`, and `originator_question_uuid` can be provided at one time. - -??? example "See an example output here..." - - ``` python - >>> events - [ - { - "event": { - "kind": "delivery_acknowledgement", - }, - }, - { - "event": { - "kind": "log_record", - "log_record": { - "args": null, - "created": 1709739861.5949728, - "exc_info": null, - "exc_text": null, - "filename": "app.py", - "funcName": "run", - "levelname": "INFO", - "levelno": 20, - "lineno": 28, - "module": "app", - "msecs": 594.9728488922119, - "msg": "Finished example analysis.", - "name": "app", - "pathname": "/workspace/example_service_cloud_run/app.py", - "process": 2, - "processName": "MainProcess", - "relativeCreated": 8560.13798713684, - "stack_info": null, - "thread": 68328473233152, - "threadName": "ThreadPoolExecutor-0_2" - } - }, - }, - { - "event": { - "kind": "heartbeat", - }, - }, - { - "event": { - "kind": "result", - "output_manifest": { - "datasets": { - "example_dataset": { - "files": [ - "gs://octue-sdk-python-test-bucket/example_output_datasets/example_dataset/output.dat" - ], - "id": "419bff6b-08c3-4c16-9eb1-5d1709168003", - "labels": [], - "name": "divergent-strange-gharial-of-pizza", - "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", - "tags": {} - } - }, - "id": "a13713ae-f207-41c6-9e29-0a848ced6039", - "name": null - }, - "output_values": [1, 2, 3, 4, 5] - }, - }, - ] - ``` - ---- - -## Asking multiple questions in parallel - -You can also ask multiple questions to a service in parallel - just -provide questions as dictionaries of `Child.ask` arguments: - -```python -child.ask_multiple( - {"input_values": {"height": 32, "width": 3}}, - {"input_values": {"height": 12, "width": 10}}, - {"input_values": {"height": 7, "width": 32}}, -) ->>> [ - ({"output_values": {"some": "output"}, "output_manifest": None}, '2681ef4e-4ab7-4cf9-8783-aad982d5e324'), - ({"output_values": {"another": "result"}, "output_manifest": None}, '474923bd-14b6-4f4c-9bfe-8148358f35cd'), - ({"output_values": {"different": "result"}, "output_manifest": None}, '9a50daae-2328-4728-9ddd-b2252474f118'), - ] -``` - -This method uses multithreading, allowing all the questions to be asked -at once instead of one after another. - -!!! hint - - The maximum number of threads that can be used to ask questions in - parallel can be set via the `max_workers` argument. It has no effect on - the total number of questions that can be asked, just how many can be in - progress at once. - -## Asking a question within a service - -If you have -[created your own Twined service](/creating_services) and want to ask children questions, you can do this more -easily than above. Children are accessible from the `analysis` object by -the keys you give them in the -[service configuration](/creating_services/#octueyaml) file. For example, you can ask an `elevation` service a -question like this: - -```python -answer, question_uuid = analysis.children["elevation"].ask( - input_values={"longitude": 0, "latitude": 1} -) -``` - -if these values are in your service configuration file: - -```json -{ - "children": [ - { - "key": "wind_speed", - "id": "template-child-services/wind-speed-service:2.1.1", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project" - } - }, - { - "key": "elevation", - "id": "template-child-services/elevation-service:3.1.9", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project" - } - } - ] -} -``` - -and your `twine.json` file includes the child keys in its `children` -field: - -```json -{ - "children": [ - { - "key": "wind_speed", - "purpose": "A service that returns the average wind speed for a given latitude and longitude." - }, - { - "key": "elevation", - "purpose": "A service that returns the elevation for a given latitude and longitude." - } - ] -} -``` - -See the parent service's [service -configuration](https://github.com/octue/octue-sdk-python/blob/main/octue/twined/templates/template-child-services/parent_service/octue.yaml) -and [app.py -file](https://github.com/octue/octue-sdk-python/blob/main/octue/twined/templates/template-child-services/parent_service/app.py) -in the [child-services app -template](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-child-services) -to see this in action. - -## Overriding a child's children - -If the child you're asking a question to has its own children (static -children), you can override these by providing the IDs of the children -you want it to use (dynamic children) to the -`Child.ask` method. Questions that would have gone to the static -children will instead go to the dynamic children. Note that: - -- You must provide the children in the same format as they're provided - in the [service configuration](/creating_services/#octueyaml) -- If you override one static child, you must override others, too -- The dynamic children must have the same keys as the static children - (so the child knows which service to ask which questions) -- You should ensure the dynamic children you provide are compatible with - and appropriate for questions from the child service - -For example, if the child requires these children in its service -configuration: - -```json -[ - { - "key": "wind_speed", - "id": "template-child-services/wind-speed-service:2.1.1", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "octue-sdk-python" - } - }, - { - "key": "elevation", - "id": "template-child-services/elevation-service:3.1.9", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "octue-sdk-python" - } - } -] -``` - -then you can override them like this: - -```python -answer, question_uuid = child.ask( - input_values={"height": 32, "width": 3}, - children=[ - { - "key": "wind_speed", - "id": "my/own-service:1.0.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "octue-sdk-python" - }, - }, - { - "key": "elevation", - "id": "organisation/another-service:0.1.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "octue-sdk-python" - }, - }, - ], -) -``` - -### Overriding beyond the first generation - -It's an intentional choice to only go one generation deep with -overriding children. If you need to be able to specify a whole tree of -children, grandchildren, and so on, please [upvote this -issue.](https://github.com/octue/octue-sdk-python/issues/528) - -## Using a service registry - -When asking a question, you can optionally specify one or more [service -registries](https://django-twined.readthedocs.io/en/latest/) to resolve -SRUIDs against. This checks if the service revision exists (good for -catching typos in SRUIDs) and raises an error if it doesn't. Service -registries can also get the default revision of a service if you don't -provide a revision tag. Asking a question if without specifying a -registry will bypass these checks. - -### Specifying service registries - -You can specify service registries in two ways: - -1. For all questions asked inside a service. In the service - configuration (`octue.yaml` file): - - > ```yaml - > services: - > - namespace: my-organisation - > name: my-app - > service_registries: - > - name: my-registry - > endpoint: blah.com/services - > ``` - -2. For questions to a specific child, inside or outside a service: - - > ```python - > child = Child( - > id="my-organisation/my-service:1.1.0", - > backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, - > service_registries=[ - > {"name": "my-registry", "endpoint": "blah.com/services"}, - > ] - > ) - > ``` diff --git a/docs/authentication.md b/docs/authentication.md deleted file mode 100644 index 32eb069d1..000000000 --- a/docs/authentication.md +++ /dev/null @@ -1,53 +0,0 @@ -# Authentication - -You need authentication while using `octue` to: - -- Access data from Google Cloud Storage -- Use, run, or deploy Twined services - -Authentication is provided by a GCP service account. - -## Creating a service account - -By setting up your Twined service network with the -[Twined Terraform modules](/deploying_services), a set of maintainer service accounts have already been -created with the required permissions. - -## Using a service account - -### Locally - -1. Access your service accounts - [here](https://console.cloud.google.com/iam-admin/serviceaccounts), - making sure the correct project is selected -2. Click on the relevant service account, go to the "Keys" tab, and - create (download) a JSON key for it - it will be called - `-XXXXX.json`. - - !!! danger - - It's best not to store this in your project to prevent accidentally - committing it or building it into a docker image layer. Instead, keep it - somewhere else on your local system with any other service account keys - you already have. - - If you must keep within your project, it's good practice to name the - file `gcp-credentials.json` and make sure that `gcp-cred*` is in your - `.gitignore` and `.dockerignore` files. - -3. If you're developing in a container (like a VSCode `devcontainer`), - mount the file into the container. You can make gcloud available - too - check out [this - tutorial](https://medium.com/datamindedbe/application-default-credentials-477879e31cb5). -4. Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the - absolute path of the key file. If using a `devcontainer`, make sure - this is the path inside the container and not the path on your local - machine. - -### On GCP infrastructure / deployed services - -- Credentials are automatically provided when running code or services - on GCP infrastructure, including the Kubernetes cluster -- `octue` uses these when when running on these platforms, so there's - no need to upload a service account key or include one in service - docker images diff --git a/docs/available_filters.md b/docs/available_filters.md deleted file mode 100644 index df59c77fb..000000000 --- a/docs/available_filters.md +++ /dev/null @@ -1,110 +0,0 @@ -# Available filters - -Lots of filters are available when using the `Dataset.files.filter` -method. We've broken them down by the type of attribute the datafiles -are being filtered by: - -- Numbers (e.g. `int`, `float`): - - - `is` - - `is_not` - - `equals` - - `not_equals` - - `lt` - - `lte` - - `gt` - - `gte` - - `in_range` - - `not_in_range` - -- Iterables (e.g. `list`, `set`, `tuple`, `dictionary`): - - - `is` - - `is_not` - - `equals` - - `not_equals` - - `contains` - - `not_contains` - - `icontains` - - `not_icontains` - -- `bool` - - - `is` - - `is_not` - -- `str` - - - `is` - - `is_not` - - `equals` - - `not_equals` - - `iequals` - - `not_iequals` - - `lt` (less than) - - `lte` (less than or equal) - - `gt` (greater than) - - `gte` (greater than or equal) - - `contains` - - `not_contains` - - `icontains` (case-insensitive contains) - - `not_icontains` - - `starts_with` - - `not_starts_with` - - `ends_with` - - `not_ends_with` - - `in_range` - - `not_in_range` - -- `NoneType` - - - `is` - - `is_not` - -- `LabelSet` - - - `is` - - `is_not` - - `equals` - - `not_equals` - - `contains` - - `not_contains` - - `any_label_contains` - - `not_any_label_contains` - - `any_label_starts_with` - - `not_any_label_starts_with` - - `any_label_ends_with` - - `not_any_label_ends_with` - -- `datetime.datetime` - - - `is` - - `is_not` - - `equals` - - `not_equals` - - `lt` (less than) - - `lte` (less than or equal) - - `gt` (greater than) - - `gte` (greater than or equal) - - `in_range` - - `not_in_range` - - `year_equals` - - `year_in` - - `month_equals` - - `month_in` - - `day_equals` - - `day_in` - - `weekday_equals` - - `weekday_in` - - `iso_weekday_equals` - - `iso_weekday_in` - - `time_equals` - - `time_in` - - `hour_equals` - - `hour_in` - - `minute_equals` - - `minute_in` - - `second_equals` - - `second_in` - - `in_date_range` - - `in_time_range` diff --git a/docs/creating_apps.md b/docs/creating_apps.md deleted file mode 100644 index c9cb0ff26..000000000 --- a/docs/creating_apps.md +++ /dev/null @@ -1,167 +0,0 @@ -# app.py file {#creating_apps} - -The `app.py` file is, as you might expect, the entrypoint to your app. -It can contain any valid python including imports and use of any number -of external packages or internal/local packages. - -## Structure - -The `app.py` file must contain exactly one of the `octue` python app -interfaces to serve as the entrypoint to your code. These take a single -`Analysis` instance as a parameter or attribute: - -- **Option 1:** A function named `run` with the following signature: - - ```python - def run(analysis): - """A function that uses input and configuration from an ``Analysis`` - instance and stores any output values and output manifests on it. - It shouldn't return anything. - - :param octue.resources.Analysis analysis: - :return None: - """ - ... - ``` - -- **Option 2:** A class named `App` with the following signature: - - ```python - class App: - """A class that takes an ``Analysis`` instance and any number of - other parameters in its constructor. It can have any number of - methods but must always have a ``run`` method with the signature - shown below. - - :param octue.resources.Analysis analysis: - :return None: - """ - - def __init__(self, analysis): - self.analysis = analysis - ... - - def run(self): - """A method that that uses input and configuration from an - ``Analysis`` instance and stores any output values and - output manifests on it. It shouldn't return anything. - - :return None: - """ - ... - - ... - ``` - -## Accessing inputs and storing outputs - -Your app must access configuration and input data from and store output -data on the `analysis` parameter (for function-based apps) or attribute (for -class-based apps). This allows standardised configuration/input/output -validation against the twine and interoperability of all Twined services -while leaving you freedom to do any kind of computation. To access the -data, use the following attributes on the `analysis` -parameter/attribute: - -- Configuration values: `analysis.configuration_values` -- Configuration manifest: `analysis.configuration_manifest` -- Input values: `analysis.input_values` -- Input manifest: `analysis.input_manifest` -- Output values: `analysis.output_values` -- Output manifest: `analysis.output_manifest` - -## Sending monitor messages - -As well as sending the final result of the analysis your app produces to -the parent, you can send monitor messages as computation progresses. -This functionality can be used to update a plot or database in real time -as the analysis is in progress. - -```python -def run(analysis): - some_data = {"x": 0, "y", 0.1, "z": 2} - analysis.send_monitor_message(data=some_data) -``` - -Before sending monitor messages, the `monitor_message_schema` field must -be provided in `twine.json`. For example: - -```json -{ - ... - "monitor_message_schema": { - "x": { - "description": "Real component", - "type": "number" - }, - "y": { - "description": "Imaginary component", - "type": "number" - }, - "z": { - "description": "Number of iterations before divergence", - "type": "number", - "minimum": 0 - } - }, - ... -} -``` - -Monitor messages can also be set up to send periodically in time. - -```python -def run(analysis): - - # Define a data structure whose attributes can be accessed in real - # time as they're updated during the analysis. - class DataStructure: - def __init__(self): - self.x = 0 - self.y = 0 - self.z = 0 - - def as_dict(self): - """Add a method that provides the data in the format - required for the monitor messages. - - :return dict: - """ - return {"x": self.x, "y": self.y, "z": self.z} - - # Create an instance of the data structure. - my_updating_data = DataStructure() - - # Use the `as_dict` method to provide up-to-date data from the data - # structure to send as monitor messages every 60s. - analysis.set_up_periodic_monitor_message( - create_monitor_message=my_updating_data.as_dict, - period=60, - ) - - # Run long-running computations on the data structure that update its - # "x", "y", and "z" attributes in real time. The periodic monitor - # message will always send the current values of x, y, and z. - some_function(my_updating_data) -``` - -## Finalising the analysis - -When the analysis has finished, it is automatically finalised. This -means: - -- The output values and manifest are validated against `twine.json` to - ensure they're in the format promised by the service. -- If the app produced an output manifest and the `output_location` field - is set to a cloud directory path in the app configuration, the output - datasets are uploaded to this location. - -!!! note - - You can manually call - `analysis.finalise` if you want to upload any output datasets to a different - location to the one specified in the service configuration. If you do - this, the analysis will not be finalised again - make sure you only call - it when your output data is ready. Note that the `output_location` and - `use_signed_urls_for_output_datasets` settings in the service - configuration are ignored if you call it manually. diff --git a/docs/creating_services.md b/docs/creating_services.md deleted file mode 100644 index d9eef87ba..000000000 --- a/docs/creating_services.md +++ /dev/null @@ -1,156 +0,0 @@ -# Creating services - -One of the main features of Twined is to allow you to easily -create services that can accept questions and return answers. They can -run locally on any machine or be deployed to the cloud. Currently: - -- The backend communication between twins uses Google Pub/Sub whether - they're local or deployed -- Services are deployed to a Kubernetes/Kueue cluster -- The language of the entrypoint must by `python3` (you can call - processes using other languages within this though) - -## Anatomy of a Twined service - -A Twined service is defined by the following files (located in the -repository root by default). - -### app.py - -This is the entrypoint into your code - read more [here](/creating_apps). - -### twine.json - -This file defines the schema for the service's configuration, input, -and output data. Read more -[here](https://twined.readthedocs.io/en/latest/) and see an example -[here](https://twined.readthedocs.io/en/latest/quick_start_create_your_first_twine.html). - -### Dependencies file - -A file specifying your app's dependencies. This is a [setup.py -file](https://docs.python.org/3/distutils/setupscript.html), a -[requirements.txt -file](https://learnpython.com/blog/python-requirements-file/), or a -[pyproject.toml file](https://python-poetry.org/docs/pyproject/) listing -all the python packages your app depends on and the version ranges that -are supported. - -### octue.yaml - -??? example "This describes the service configuration - read more..." - - This file defines the basic structure of your service. It must contain - at least: - - ``` yaml - services: - - namespace: my-organisation - name: my-app - ``` - It may also need the following key-value pairs: - - - `app_source_path: ` - if your `app.py` file is not in the repository root - - All paths should be relative to the repository root. Other valid - entries can be found in the `ServiceConfiguration` constructor. - - !!! warning - - Currently, only one service can be defined per repository, but it must - still appear as a list item of the "services" key. At some point, it - will be possible to define multiple services in one repository. - - If a service's app needs any configuration, asks questions to any - other Twined services, or produces output datafiles/datasets, you will - need to provide some or all of the following values for that service: - - - `configuration_values` - - `configuration_manifest` - - `children` - - `output_location` - - `use_signed_urls_for_output_datasets` - -### Dockerfile (optional) - -??? example "Provide this if your needs exceed the default Octue Dockerfile - read more..." - - Twined services run in a Docker container if they are deployed. They - can also run this way locally. The SDK provides a default `Dockerfile` - for these purposes that will work for most cases: - - - For deploying to [Kubernetes](https://github.com/octue/octue-sdk-python/blob/main/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python313) - - However, you may need to write and provide your own `Dockerfile` if - your app requires: - - - Non-python or system dependencies (e.g. `openfast`, `wget`) - - Python dependencies that aren't installable via `pip` - - Private python packages - - Here are two examples of a custom `Dockerfile` that use different base - images: - - - [A TurbSim service](https://github.com/octue/turbsim-service/blob/main/Dockerfile) - - [An OpenFAST service](https://github.com/octue/openfast-service/blob/main/Dockerfile) - - If you do provide one, you must provide its path relative to your - repository to the [build-twined-services] GitHub Actions [workflow](https://github.com/octue/workflows/blob/main/.github/workflows/build-twined-service.yml). - - As always, if you need help with this, feel free to drop us a message or raise an issue! - - -### Where to specify the namespace, name, and revision tag - -See [here](/services/#service-names) for service naming requirements. - -**Namespace** - -- Required: yes -- Set in: - - `octue.yaml` - - `OCTUE_SERVICE_NAMESPACE` environment variable (takes priority) - -**Name** - -- Required: yes -- Set in: - - `octue.yaml` - - `OCTUE_SERVICE_NAME` environment variable (takes priority) - -**Revision tag** - -- Required: no -- Default: a random "coolname" (e.g. `hungry-hippo`) -- Set in: - - `OCTUE_SERVICE_REVISION_TAG` environment variable - - If using `octue twined service start` command, the `--revision-tag` option (takes priority) - -## Template apps - -We've created some template apps for you to look at and play around -with. We recommend going through them in this order: - -1. The [fractal app template](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-fractal) - introduces a basic Twined service that returns output values to its - parent. -2. The [using-manifests app template](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-using-manifests) - introduces using a manifest of output datasets to return output - files to its parent. -3. The [child-services app template](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-child-services) - introduces asking questions to child services and using their - answers to form an output to return to its parent. - -## Deploying services automatically - -Automated deployment with Octue means: - -- Your service runs in Google Kubernetes Engine (GKE), ready to accept - questions from and return answers to other services. -- You don't need to do anything to update your deployed service with - new code changes - the service simply gets rebuilt and re-deployed - each time you push a commit to your `main` branch, or merge a pull - request into it (other branches and deployment strategies are - available, but this is the default). -- Serverless is the default - your service only runs when questions from - other services are sent to it, meaning there are minimal costs to - having it deployed but not in use. - -If you'd like help deploying services, contact us. To do it yourself, see [here](/deploying_services). diff --git a/docs/data_containers/datafile.md b/docs/data_containers/datafile.md deleted file mode 100644 index 31093f691..000000000 --- a/docs/data_containers/datafile.md +++ /dev/null @@ -1,312 +0,0 @@ -# Datafile - -!!! info "Definitions" - - **Datafile** - - A single local or cloud file, its metadata, and helper methods. - - **Locality** - - A datafile has one of these localities: - - - **Cloud-based:** it exists only in the cloud - - **Local:** it exists only on your local filesystem - - **Cloud-based and local:** it's cloud-based but has been - downloaded for low-latency reading/writing - -!!! tip - - Use a datafile to work with a file if you want to: - - - Read/write to local and cloud files in the same way - - Include it in a `dataset `{.interpreted-text role="doc"} that - can be sent to a Twined service for processing - - Add metadata to it for future sorting and filtering - -## Key features - -### Work with local and cloud data - -Working with a datafile is the same whether it's local or cloud-based. -It's also almost identical to using [python built-in open -function](https://docs.python.org/3/library/functions.html#open). For -example, to write to a datafile: - -```python -from octue.resources import Datafile - -datafile = Datafile("path/to/file.dat") - -# Or: - -datafile = Datafile("gs://my-bucket/path/to/file.dat") - -with datafile.open("w") as f: - f.write("Some data") - datafile.labels.add("processed") -``` - -All the same file modes you'd use with the [python built-in open -context manager](https://docs.python.org/3/library/functions.html#open) -are available for datafiles e.g. `"r"` and `"a"`. - -### Automatic lazy downloading - -Save time and bandwidth by only downloading when necessary. - -Downloading data from cloud datafiles is automatic and lazy so you get -both low-latency content read/write and quick metadata reads. This makes -viewing and filtering by the metadata of cloud datasets and datafiles -quick and avoids unnecessary data transfer, energy usage, and costs. - -Datafile content isn't downloaded until you: - -- Try to read or write its contents using the `Datafile.open` context manager -- Call its `download` method -- Use its `local_path` property - -Read more about downloading files [here](/downloading_datafiles). - -### CLI command friendly - -Use any command line tool on your datafiles. Datafiles are python -objects, but they represent real files that can be fed to any CLI -command you like - -```python -import subprocess -output = subprocess.check_output(["openfast", datafile.local_path]) -``` - -### Easy and expandable custom metadata - -Find the needle in the haystack by making your data searchable. You can -set the following metadata on a datafile: - -- Timestamp -- Labels (a set of lowercase strings) -- Tags (a dictionary of key-value pairs) - -This metadata is stored locally in a `.octue` file for local datafiles -or on the cloud objects for cloud datafiles and is used during -`Datafile` instantiation. It can be accessed like this: - -```python -datafile.timestamp ->>> datetime.datetime(2022, 5, 4, 17, 57, 57, 136739) - -datafile.labels ->>> {"processed"} - -datafile.tags ->>> {"organisation": "octue", "energy": "renewable"} -``` - -You can update the metadata by setting it on the instance while inside -the `Datafile.open` context manager. - -```python -with datafile.open("a"): - datafile.labels.add("updated") -``` - -You can do this outside the context manager too, but you then need to -call the update method: - -```python -datafile.labels.add("updated") -datafile.update_metadata() -``` - -### Upload an existing local datafile - -Back up and share your datafiles for collaboration. You can upload an -existing local datafile to the cloud without using the -`Datafile.open` context manager if you don't need to modify its contents: - -```python -datafile.upload("gs://my-bucket/my_datafile.dat", update_metadata=True) -``` - -### Get file and metadata hashes - -Make your analysis reproducible: guarantee a datafile contains exactly -the same data as before by checking its hash. - -```python -datafile.hash_value ->>> 'mnG7TA==' -``` - -You can also check that any metadata is the same. - -```python -datafile.metadata_hash_value ->>> 'DIgCHg==' -``` - -### Immutable ID - -Each datafile has an immutable UUID: - -```python -datafile.id ->>> '9a1f9b26-6a48-4f2d-be80-468d3270d79b' -``` - -### Check a datafile's locality - -Is this datafile local or in the cloud? - -```python -datafile.exists_locally ->>> True - -datafile.exists_in_cloud ->>> False -``` - -A cloud datafile that has been downloaded will return `True` for both of -these properties. - -### Represent HDF5 files - -Support fast I/O processing and storage. - -!!! warning - - If you want to represent HDF5 files with a `Datafile`, you must include - the extra requirements provided by the `hdf5` key at installation i.e. - - ``` shell - pip install octue[hdf5] - ``` - - or - - ``` shell - poetry add octue -E hdf5 - ``` - -## Usage examples - -The `Datafile` class can be used functionally or as a context manager. -When used as a context manager, it is analogous with the [python -built-in open -function](https://docs.python.org/3/library/functions.html#open). On -exiting the context (the `with` block), it closes the datafile locally -and, if the datafile also exists in the cloud, updates the cloud object -with any data or metadata changes. - -![image](../images/datafile_use_cases.png) - -### Example A - -**Scenario:** Download a cloud object, calculate Octue metadata from its -contents, and add the new metadata to the cloud object - -**Starting point:** Object in cloud with or without Octue metadata - -**Goal:** Object in cloud with updated metadata - -```python -from octue.resources import Datafile - - -datafile = Datafile("gs://my-bucket/path/to/data.csv") - -with datafile.open() as f: - data = f.read() - new_metadata = metadata_calculating_function(data) - - datafile.timestamp = new_metadata["timestamp"] - datafile.tags = new_metadata["tags"] - datafile.labels = new_metadata["labels"] -``` - -### Example B - -**Scenario:** Add or update Octue metadata on an existing cloud object -_without downloading its content_ - -**Starting point:** A cloud object with or without Octue metadata - -**Goal:** Object in cloud with updated metadata - -```python -from datetime import datetime -from octue.resources import Datafile - - -datafile = Datafile("gs://my-bucket/path/to/data.csv") - -datafile.timestamp = datetime.now() -datafile.tags = {"manufacturer": "Vestas", "output": "1MW"} -datafile.labels = {"new"} - -datafile.upload(update_metadata=True) # Or, datafile.update_metadata() -``` - -### Example C - -**Scenario:** Read in the data and Octue metadata of an existing cloud -object without intent to update it in the cloud - -**Starting point:** A cloud object with Octue metadata - -**Goal:** Cloud object data (contents) and metadata held locally in -local variables - -```python -from octue.resources import Datafile - - -datafile = Datafile("gs://my-bucket/path/to/data.csv") - -with datafile.open() as f: - data = f.read() - -metadata = datafile.metadata() -``` - -### Example D - -**Scenario:** Create a new cloud object from local data, adding Octue -metadata - -**Starting point:** A file-like locally (or content data in local -variable) with Octue metadata stored in local variables - -**Goal:** A new object in the cloud with data and Octue metadata - -For creating new data in a new local file: - -```python -from octue.resources import Datafile - - -datafile = Datafile( - "path/to/local/file.dat", - tags={"cleaned": True, "type": "linear"}, - labels={"Vestas"} -) - -with datafile.open("w") as f: - f.write("This is some cleaned data.") - -datafile.upload("gs://my-bucket/path/to/data.dat") -``` - -For existing data in an existing local file: - -```python -from octue.resources import Datafile - - -tags = {"cleaned": True, "type": "linear"} -labels = {"Vestas"} - -datafile = Datafile(path="path/to/local/file.dat", tags=tags, labels=labels) -datafile.upload("gs://my-bucket/path/to/data.dat") -``` diff --git a/docs/data_containers/dataset.md b/docs/data_containers/dataset.md deleted file mode 100644 index 02442c875..000000000 --- a/docs/data_containers/dataset.md +++ /dev/null @@ -1,230 +0,0 @@ -# Dataset - -!!! info "Definitions" - - **Dataset** - - A set of related datafiles that exist in the same location, dataset metadata, and helper methods. - - **Locality** - - A dataset has one of these localities: - - **Cloud-based:** it exists only in the cloud - - **Local:** it exists only on your local filesystem - -!!! tip - - Use a dataset if you want to: - - - Group together a set of files that naturally relate to each other e.g. a timeseries that's been split into - multiple files. - - Add metadata to it for future sorting and filtering - - Include it in a manifest with other datasets and send them to a Twined service for processing - -## Key features - -### Work with local and cloud datasets - -Working with a dataset is the same whether it's local or cloud-based. - -```python -from octue.resources import Dataset - -# Local dataset -dataset = Dataset(path="path/to/dataset") - -# Dataset in cloud bucket -dataset = Dataset(path="gs://my-bucket/path/to/dataset") -``` - -!!! warning - - Datasets recurse all subdirectories by default unless `recursive=False` - is set. - -### Upload a dataset - -Back up and share your datasets for collaboration. - -```python -dataset.upload("gs://my-bucket/path/to/upload") -``` - -### Download a dataset - -Use a shared or public dataset or retrieve a backup. - -```python -dataset.download("path/to/download") -``` - -### Easy and expandable custom metadata - -Find the needle in the haystack by making your data searchable. You can -set the following metadata on a dataset: - -- Name -- Labels (a set of lowercase strings) -- Tags (a dictionary of key-value pairs) - -This metadata is stored locally in a `.octue` file in the same directory -as the dataset and is used during `Dataset` instantiation. It can be -accessed like this: - -```python -dataset.name ->>> "my-dataset" - -dataset.labels ->>> {"processed"} - -dataset.tags ->>> {"organisation": "octue", "energy": "renewable"} -``` - -You can update the metadata by setting it on the instance while inside -the `Dataset` context manager. - -```python -with dataset: - datafile.labels.add("updated") -``` - -You can do this outside the context manager too, but you then need to -call the update method: - -```python -dataset.labels.add("updated") -dataset.update_metadata() -``` - -### Get dataset and metadata hashes - -Make your analysis reproducible: guarantee a dataset contains exactly -the same data as before by checking its hash. - -```python -dataset.hash_value ->>> 'uvG7TA==' -``` - -!!! note - - A dataset's hash is a function of its datafiles' hashes. Datafile and - dataset metadata do not affect it. - -You can also check that dataset metadata is the same. - -```python -dataset.metadata_hash_value ->>> 'DIgCHg==' -``` - -### Immutable ID - -Each dataset has an immutable UUID: - -```python -dataset.id ->>> '9a1f9b26-6a48-4f2d-be80-468d3270d79c' -``` - -### Check a dataset's locality - -Is this dataset local or in the cloud? - -```python -dataset.exists_locally ->>> True - -dataset.exists_in_cloud ->>> False -``` - -A dataset can only return `True` for one of these at a time. - -### Filter datasets - -Narrow down a dataset to just the files you want to avoiding extra -downloading and processing. - -Datafiles in a dataset are stored in a -`FilterSet`, meaning they can be easily filtered by any attribute of the -datafiles contained e.g. name, extension, ID, timestamp, tags, labels, -size. The filtering syntax is similar to Django's i.e. - -```shell -# Get datafiles that have an attribute that satisfies the filter. -dataset.files.filter(__=) - -# Or, if your filter is a simple equality filter: -dataset.files.filter(=) -``` - -Here's an example: - -```python -# Make a dataset. -dataset = Dataset( - path="blah", - files=[ - Datafile(path="my_file.csv", labels=["one", "a", "b" "all"]), - Datafile(path="your_file.txt", labels=["two", "a", "b", "all"), - Datafile(path="another_file.csv", labels=["three", "all"]), - ] -) - -# Filter it! -dataset.files.filter(name__starts_with="my") ->>> })> - -dataset.files.filter(extension="csv") ->>> , })> - -dataset.files.filter(labels__contains="a") ->>> , })> -``` - -You can iterate through the filtered files: - -```python -for datafile in dataset.files.filter(labels__contains="a"): - print(datafile.name) ->>> 'my_file.csv' - 'your_file.txt' -``` - -If there's just one result, get it via the `FilterSet.one` method: - -```python -dataset.files.filter(name__starts_with="my").one() ->>> -``` - -You can also chain filters or specify them all at the same time - these -two examples produce the same result: - -```python -# Chaining multiple filters. -dataset.files.filter(extension="csv").filter(labels__contains="a") ->>> })> - -# Specifying multiple filters at once. -dataset.files.filter(extension="csv", labels__contains="a") ->>> })> -``` - -For the full list of available filters, [click here](/available_filters). - -### Order datasets - -A dataset can also be ordered by any of the attributes of its datafiles: - -```python -dataset.files.order_by("name") ->>> , , ])> -``` - -The ordering can also be carried out in reverse (i.e. descending order) -by passing `reverse=True` as a second argument to the -`FilterSet.order_by` method. diff --git a/docs/data_containers/index.md b/docs/data_containers/index.md deleted file mode 100644 index f257a2ae6..000000000 --- a/docs/data_containers/index.md +++ /dev/null @@ -1,10 +0,0 @@ -# Datafiles, datasets, and manifests - -One of the main features of Twined is making using, creating, and -sharing scientific datasets easy. There are three main data classes in -the that do this. - -- **Datafile** - [a single local or cloud file](/data_containers/datafile) and its metadata. -- **Dataset** - [a set of related datafiles](/data_containers/dataset) that exist in the same location, plus metadata. -- **Manifest** - [a set of related datasets](/data_containers/manifest) that exist anywhere, plus metadata. Typically produced by or for - one analysis. diff --git a/docs/data_containers/manifest.md b/docs/data_containers/manifest.md deleted file mode 100644 index e04310072..000000000 --- a/docs/data_containers/manifest.md +++ /dev/null @@ -1,114 +0,0 @@ -# Manifest - -!!! info "Definition" - - **Manifest** - - A set of related cloud and/or local [datasets](/dataset), metadata, and helper methods. Typically produced by or - needed for processing by a Twined service. - -!!! tip - - Use a manifest to send [datasets](/dataset) to a Twined service as a question (for processing) - the - service will send an output manifest back with its answer if the answer includes output datasets. - -## Key features - -### Group related datasets together - -Make a clear grouping of datasets needed for a particular analysis. - -```python -from octue.resources import Manifest - -manifest = Manifest( - datasets={ - "my_dataset_0": "gs://my-bucket/my_dataset_0", - "my_dataset_1": "gs://my-bucket/my_dataset_1", - "my_dataset_2": "gs://another-bucket/my_dataset_2", - } -) -``` - -### Send datasets to a service - -Get a Twined service to analyse data for you as part of a larger -analysis. - -```python -from octue.twined.resources import Child - -child = Child( - id="octue/wind-speed:2.1.0", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, -) - -answer, question_uuid = child.ask(input_manifest=manifest) -``` - -See [here](/asking_questions) for more information. - -### Receive datasets from a service - -Access output datasets from a Twined service from the cloud when you're -ready. - -```python -manifest = answer["output_manifest"] -manifest["an_output_dataset"].files ->>> , })> -``` - -!!! hint - - Datasets in an output manifest are stored in the cloud. You'll need to - keep a reference to where they are to access them - the output manifest - is this reference. You'll need to use it straight away or save it to - make use of it. - -### Download all datasets from a manifest - -Download all or a subset of datasets from a manifest. - -```python -manifest.download() ->>> { - "my_dataset": "/path/to/dataset" -} -``` - -!!! note - - Datasets are downloaded to a temporary directory if no paths are given. - -## Further information - -### Manifests of local datasets - -You can include local datasets in your manifest if you can guarantee all -services that need them can access them. A use case for this is, for -example, a supercomputer cluster running several Twined services -locally that process and transfer large amounts of data. It is much -faster to store and access the required datasets locally than upload -them to the cloud and then download them again for each service (as -would happen with cloud datasets). - -!!! warning - - If you want to ask a child a question that includes a manifest containing one or more local datasets, you must - include the `allow_local_files` parameter. For example, if you have an analysis object with a child called "wind_speed": - - ``` python - input_manifest = Manifest( - datasets={ - "my_dataset_0": "gs://my-bucket/my_dataset_0", - "my_dataset_1": "local/path/to/my_dataset_1", - } - ) - - answer, question_uuid = analysis.children["wind_speed"].ask( - input_values=analysis.input_values, - input_manifest=analysis.input_manifest, - allow_local_files=True, - ) - ``` diff --git a/docs/deploying_services.md b/docs/deploying_services.md deleted file mode 100644 index 91a658412..000000000 --- a/docs/deploying_services.md +++ /dev/null @@ -1,79 +0,0 @@ -# Deploying services (developer's guide) {#deploying_services_advanced} - -This is a guide for developers that want to deploy Twined services -themselves - it is not needed if Octue manages your services for you or -if you are only asking questions to existing Twined services. - -## What is deployment? - -Deploying a Twined service means the service: - -- Is a docker image that is spun up and down in a Kubernetes cluster on - demand -- Is ready at any time to answer questions from users and other Twined - services in the service network -- Can ask questions to any other Twined service in the service network -- Will automatically spin down after it has finished answering a - question -- Will automatically build and redeploy after a relevant code change - (e.g. on push or merge into `main`) - -We can split deployment into service deployment and infrastructure -deployment. - -## Deploying a service - -Assuming the service network infrastructure already exists, a service -can be deployed by building and pushing its docker image to the service -network's Artifact Registry repository. We recommend pushing a new -image for each release of the code e.g. on merge into the `main` branch. -Each new image is the deployment of a new service revision. This can be -done automatically: - -- Follow the - [instructions](https://github.com/octue/workflows#deploying-a-kuberneteskueue-octue-twined-service-revision) - to add the - [build-twined-service](https://github.com/octue/workflows/blob/main/.github/workflows/build-twined-service.yml) - GitHub Actions workflow to your service's GitHub repository. Set its - trigger to merge or push to `main` (see example below) -- This needs to be done **once for every service** you want to deploy -- A live example can be [found - here](https://github.com/octue/example-service-kueue/blob/main/.github/workflows/release.yml) - including automated pre-deployment testing and creation of a GitHub - release - -You can now [ask your service some questions](/asking_questions)! It will be available in the service network as -`/:` (e.g. `octue/example-service-kueue:0.1.1`). - -## Deploying the infrastructure - -### Prerequisites - -Twined services are currently deployable to Google Cloud Platform (GCP). -You must have "owner" level access to the GCP project you're -deploying to and billing must be set up for it. - -### Deploying step-by-step - -There are two steps to deploying the infrastructure: - -1. Deploy the core infrastructure (storage bucket, event store, IAM - service accounts and roles) -2. Deploy the Kubernetes cluster, event handler, service registry, and - Pub/Sub topic - -#### 1. Deploy core infrastructure - -- Follow [the - instructions](https://github.com/octue/terraform-octue-twined-core) to - deploy the resources in the `terraform-octue-twined-core` Terraform - module -- This only needs to be done once per service network - -#### 2. Deploy Kubernetes cluster - -- Follow the - [instructions](https://github.com/octue/terraform-octue-twined-cluster) - to deploy the resources in the `terraform-octue-twined-cluster` - Terraform module -- This only needs to be done once per service network diff --git a/docs/downloading_datafiles.md b/docs/downloading_datafiles.md deleted file mode 100644 index 7938653f6..000000000 --- a/docs/downloading_datafiles.md +++ /dev/null @@ -1,33 +0,0 @@ -# More information on downloading datafiles - -- To avoid unnecessary data transfer and costs, cloud datafiles are not - downloaded locally [until necessary](/data_containers/datafile/#automatic-lazy-downloading) -- When downloaded, they are downloaded by default to a temporary local - file that will exist at least as long as the python session is running -- Calling `Datafile.download` or using `Datafile.local_path` again will - not re-download the file -- Any changes made to the datafile via the `Datafile.open` method are - made to the local copy and then synced with the cloud object - -!!! warning - - External changes to cloud files will not be synced locally unless the - datafile is re-instantiated. - -- If you want a cloud datafile to be downloaded to a permanent location, - you can do one of: - - ```python - datafile.download(local_path="my/local/path.csv") - - datafile.local_path = "my/local/path.csv" - ``` - -- To pre-set a permanent download location on instantiation, run: - - ```python - datafile = Datafile( - "gs://my-bucket/path/to/file.dat", - local_path="my/local/path.csv", - ) - ``` diff --git a/docs/index.md b/docs/index.md index 15eca833a..adeffa7ae 100644 --- a/docs/index.md +++ b/docs/index.md @@ -40,7 +40,7 @@ The following command asks a question to the local example data service. === "CLI" ```shell - octue twined question ask remote example/service:latest --input-values='{"some": "data"}' + octue twined question ask example/service:latest --input-values='{"some": "data"}' ``` !!! tip @@ -48,7 +48,7 @@ The following command asks a question to the local example data service. To ask a question to a real data service, just specify its ID: ```shell - octue twined question ask remote some-org/a-service:1.2.0 --input-values='{"some": "data"}' + octue twined question ask some-org/a-service:1.2.0 --input-values='{"some": "data"}' ``` === "Python" @@ -101,10 +101,10 @@ The following command asks a question to the local example data service. ```shell # Format the result using the `jq` command line tool - octue twined question ask remote example/service:latest --input-values='{"some": "data"}' | jq + octue twined question ask example/service:latest --input-values='{"some": "data"}' | jq # Store the result in a file - octue twined question ask remote example/service:latest --input-values='{"some": "data"}' > result.json + octue twined question ask example/service:latest --input-values='{"some": "data"}' > result.json ``` === "Python" diff --git a/docs/inter_service_compatibility.md b/docs/inter_service_compatibility.md deleted file mode 100644 index 4079e0ad5..000000000 --- a/docs/inter_service_compatibility.md +++ /dev/null @@ -1,77 +0,0 @@ -# Inter-service compatibility - -Twined services acting as parents and children communicate with each -other according to the [services communication -schema](https://strands.octue.com/octue/service-communication). The -table below shows which `octue` versions parents can run (rows) to send -questions compatible with versions children are running (columns). Note -that this table does not display whether children's responses are -compatible with the parent, just that a child is able to accept a -question. - -**Key** - -- `0` = incompatible -- `1` = compatible - -| | 0.68.0 | 0.67.0 | 0.66.1 | 0.66.0 | 0.65.0 | 0.64.0 | 0.63.0 | 0.62.1 | 0.62.0 | 0.61.2 | 0.61.1 | 0.61.0 | 0.60.2 | 0.60.1 | 0.60.0 | 0.59.1 | 0.59.0 | 0.58.0 | 0.57.2 | 0.57.1 | 0.57.0 | 0.56.0 | 0.55.0 | 0.54.0 | 0.53.0 | 0.52.2 | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | -| :----- | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -| 0.68.0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.67.0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.66.1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.66.0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.65.0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.64.0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.63.0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.62.1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.62.0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.61.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.61.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.61.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.60.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.60.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.60.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.59.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.59.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.58.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.57.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.57.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.57.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.56.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.55.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.54.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.53.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.52.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.52.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.52.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.51.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.50.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.50.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.49.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.49.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.49.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.48.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.47.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.47.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.47.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.46.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.46.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.46.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.46.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.45.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.44.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.7 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.42.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.42.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.41.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.41.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.40.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.40.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.40.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | diff --git a/docs/logging.md b/docs/logging.md deleted file mode 100644 index db5ce7f88..000000000 --- a/docs/logging.md +++ /dev/null @@ -1,55 +0,0 @@ -# Logging - -By default, `octue` streams your logs to `stderr` in a nice, readable -format so your log messages are immediately visible when you start -developing without any extra configuration. If you prefer to use your -own handlers or formatters, simply set `USE_OCTUE_LOG_HANDLER=0` in the -environment running your app. - -## Readable logs - -Some advantages of the Octue log handler are: - -- Its readable format -- Its clear separation of log **context** from log **message**. - -Below, the context is on the left and includes: - -- The time -- Log level -- Module producing the log -- Octue analysis ID - -This is followed by the actual log message on the right: - -``` -[2021-07-10 20:03:12,713 | INFO | octue.runner | 102ee7d5-4b94-4f8a-9dcd-36dbd00662ec] Hello! The child services template app is running! -``` - -## Colourised services - -Another advantage to using the Octue log handler is that each Twined -service is coloured according to its position in the tree, making it -much easier to read log messages from multiple levels of children. - -![image](images/coloured_logs.png) - -In this example: - -- The log context is in blue -- Anything running in the root parent service's app is labeled with the - analysis ID in green -- Anything running in the immediate child services (`elevation` and - `wind_speed`) are labelled with the analysis ID in yellow -- Any children further down the tree (i.e. children of the child - services and so on) will have their own labels in other colours - consistent to their level - -## Add extra information - -You can add certain log record attributes to the logging context by also -providing the following environment variables: - -- `INCLUDE_LINE_NUMBER_IN_LOGS=1` - include the line number -- `INCLUDE_PROCESS_NAME_IN_LOGS=1` - include the process name -- `INCLUDE_THREAD_NAME_IN_LOGS=1` - include the thread name diff --git a/docs/running_services_locally.md b/docs/running_services_locally.md deleted file mode 100644 index e547b9ec3..000000000 --- a/docs/running_services_locally.md +++ /dev/null @@ -1,72 +0,0 @@ -# Running services locally - -Services can be operated locally (e.g. for testing or ad-hoc data -processing). You can: - -- Run your service once (i.e. run one analysis): - - Via the `octue twined` CLI - - By using the `octue` library in a python script -- Start your service as a child, allowing it to answer any number of - questions from any other Twined service: - - Via the CLI - -## Running a service once - -### Via the CLI - -1. Ensure you've created a valid - [`octue.yaml`](/creating_services/#octueyaml) - file for your service - -2. Run: - - > ```shell - > octue twined question ask local --input-values='{"some": "input"}' - > ``` - -The output values and/or manifest will be printed to `stdout` but are -also stored in the event store. - -### Via a python script - -Imagine we have a simple app that calculates the area of a square. It -could be run locally on a given height and width like this: - -```python -from octue.twined.runner import Runner - -runner = Runner(app_src="path/to/app.py", twine="path/to/twine.json") -analysis = runner.run(input_values={"height": 5, "width": 10}) - -analysis.output_values ->>> {"area": 50} - -analysis.output_manifest ->>> None -``` - -See the `Runner` API documentation for more advanced usage including providing configuration, children, and an input manifest. - -## Starting a service as a child - -### Via the CLI - -1. Ensure you've created a valid - [`octue.yaml`](/creating_services/#octueyaml) - file for your service - -2. Run: - - > ```shell - > octue twined service start - > ``` - -This will run the service as a child waiting for questions until you -press `Ctrl + C` or an error is encountered. The service will be -available to be questioned by other services at the service ID -`organisation/name` as specified in the `octue.yaml` file. - -!!! tip - - You can use the `--timeout` option to stop the service after a given - number of seconds. diff --git a/docs/services.md b/docs/services.md deleted file mode 100644 index 02eb209d9..000000000 --- a/docs/services.md +++ /dev/null @@ -1,125 +0,0 @@ -# Twined services - -There's a growing range of live [services](/) in the -Twined ecosystem that you can query, mostly related to wind energy and -other renewables. Here's a quick glossary of terms before we tell you -more: - -!!! info "Definitions" - - **Twined service** - - See [here](/). - - **Child** - - A Twined service that can be asked a question. This name reflects - the tree structure of services (specifically, [a - DAG](https://en.wikipedia.org/wiki/Directed_acyclic_graph)) formed - by the service asking the question (the parent), the child it asks - the question to, any children that the child asks questions to as - part of forming its answer, and so on. - - **Parent** - - A Twined service that asks a question to another Twined service (a - child). - - **Asking a question** - - Sending data (input values and/or an input manifest) to a child for - processing/analysis. - - **Receiving an answer** - - Receiving data (output values and/or an output manifest) from a - child you asked a question to. - - **Twined ecosystem** - - The set of services running Twined as their backend. These - services guarantee: - - - Defined input/output JSON schemas and validation - - An easy and consistent interface for asking them questions and - receiving their answers - - Logs, exceptions, and monitor messages forwarded to you - - High availability (if deployed in the cloud) - -## Service names - -Questions are always asked to a _revision_ of a service. Services -revisions are named in a similar way to docker images. They look like -`namespace/name:tag` where the tag is often a semantic version (but -doesn't have to be). - -!!! info "Definitions" - - **Service revision** - - A specific instance of a Twined service that can be individually - addressed. The revision could correspond to a version of the - service, a dynamic development branch for it, or a deliberate - duplication or variation of it. - - **Service revision unique identifier (SRUID)** - - The combination of a service revision's namespace, name, and - revision tag that uniquely identifies it. For example, - `octue/my-service:1.3.0` where the namespace is `octue`, the name is - `my-service`, and the revision tag is `1.3.0`. - - **Service namespace** - - The group to which the service belongs e.g. your name or your - organisation's name. If in doubt, use the GitHub handle of the user - or organisation publishing the services. - - Namespaces must be lower kebab case (i.e. they may contain the - letters \[a-z\], numbers \[0-9\], and hyphens \[-\]). They may not - begin or end with hyphens. - - **Service name** - - A name to uniquely identify the service within its namespace. This - usually corresponds to the name of the GitHub repository for the - service. Names must be lower kebab case (i.e. they may contain the - letters \[a-z\], numbers \[0-9\] and hyphens \[-\]). They may not - begin or end with hyphens. - - **Service revision tag** - - A tag that uniquely identifies a particular revision of a service. - The revision tag could be a: - - - Commit hash (e.g. `a3eb45`) - - Semantic version (e.g. `0.12.4`) - - Branch name (e.g. `development`) - - Particular environment the service is deployed in (e.g. - `production`) - - Combination of these (e.g. `0.12.4-production`) - - Tags may contain lowercase and uppercase letters, numbers, - underscores, periods, and hyphens, but can't start with a period or - a dash. They can contain a maximum of 128 characters. These - requirements are the same as the [Docker tag - format](https://docs.docker.com/engine/reference/commandline/tag/). - - **Service ID** - - The SRUID is a special case of a service ID. A service ID can be an - SRUID or just the service namespace and name. It can be used to ask - a question to a service without specifying a specific revision of - it. This enables asking questions to, for example, the service - `octue/my-service` and automatically having them routed to its - default (usually latest) revision. - [See here for more info](/asking_questions/#asking-a-question) - -## Service communication standard - -Twined services communicate according to the service communication -standard. The JSON schema defining this can be found -[here](https://strands.octue.com/octue/service-communication). Messages -received by services are validated against it and invalid messages are -rejected. The schema is in beta, so (rare) breaking changes are -reflected in the minor version number. diff --git a/docs/testing_services.md b/docs/testing_services.md deleted file mode 100644 index 32fe14408..000000000 --- a/docs/testing_services.md +++ /dev/null @@ -1,290 +0,0 @@ -# Testing services {#testing_services} - -We recommend writing automated tests for your service so anyone who -wants to use it can have confidence in its quality and reliability at a -glance. [Here's an example -test](https://github.com/octue/example-service-kueue/blob/main/tests/test_app.py) -for our example service. - -## Emulating children - -If your app has children, you should emulate them in your tests instead -of communicating with the real ones. This makes your tests: - -- **Independent** of anything external to your app code - i.e. - independent of the remote child, your internet connection, and - communication between your app and the child (Google Pub/Sub). -- **Much faster** - the emulation will complete in a few milliseconds as - opposed to the time it takes the real child to actually run an - analysis, which could be minutes, hours, or days. Tests for our [child - services template - app](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-child-services) - run around **900 times faster** when the children are emulated. - -### The Child Emulator - -We've written a child emulator that takes a list of events and returns -them to the parent for handling in the order given - without contacting -the real child or using Pub/Sub. Any events a real child can produce are -supported. `Child` instances can be mocked like-for-like by -`ChildEmulator` instances without the parent knowing. - -### Event kinds - -You can emulate any event that your app (the parent) can handle. The -table below shows what these are. - -| Event kind | Number of events supported | Example | -| -------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `delivery_acknowledgement` | One | `{"event": {"kind": "delivery_acknowledgement"}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}}` | -| `heartbeat` | Any number | `{"event": {"kind": "heartbeat"}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | -| `log_record` | Any number | `{"event": {"kind": "log_record": "log_record": {"msg": "Starting analysis."}}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | -| `monitor_message` | Any number | `{"event": {"kind": "monitor_message": "data": '{"progress": "35%"}'}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | -| `exception` | One | `{"event": {"kind": "exception", "exception_type": "ValueError", "exception_message": "x cannot be less than 10."}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | -| `result` | One | `{"event": {"kind": "result", "output_values": {"my": "results"}, "output_manifest": None}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | - -**Notes** - -- Event formats and contents must conform with the [service - communication - schema](https://strands.octue.com/octue/service-communication). -- Every event must be accompanied with the required event attributes -- The `log_record` key of a `log_record` event is any dictionary that - the `logging.makeLogRecord` function can convert into a log record. -- The `data` key of a `monitor_message` event must be a JSON-serialised - string -- Any events after a `result` or `exception` event won't be passed to - the parent because execution of the child emulator will have ended. - -### Instantiating a child emulator - -```python -events = [ - { - { - "event": { - "kind": "log_record", - "log_record": {"msg": "Starting analysis."}, - ... # Left out for brevity. - }, - "attributes": { - "datetime": "2024-04-11T10:46:48.236064", - "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", - "retry_count": 0, - "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", - "parent_question_uuid": "5776ad74-52a6-46f7-a526-90421d91b8b2", - "originator_question_uuid": "86dc55b2-4282-42bd-92d0-bd4991ae7356", - "parent": "octue/test-service:1.0.0", - "originator": "octue/test-service:1.0.0", - "sender": "octue/test-service:1.0.0", - "sender_type": "CHILD", - "sender_sdk_version": "0.51.0", - "recipient": "octue/another-service:3.2.1" - }, - }, - }, - { - "event": { - "kind": "monitor_message", - "data": '{"progress": "35%"}', - }, - "attributes": { - ... # Left out for brevity. - }, - }, - { - "event": { - "kind": "log_record", - "log_record": {"msg": "Finished analysis."}, - ... # Left out for brevity. - }, - "attributes": { - ... # Left out for brevity. - }, - }, - { - "event": { - "kind": "result", - "output_values": [1, 2, 3, 4, 5], - }, - "attributes": { - ... # Left out for brevity. - }, - }, -] - -child_emulator = ChildEmulator(events) - -def handle_monitor_message(message): - ... - -result, question_uuid = child_emulator.ask( - input_values={"hello": "world"}, - handle_monitor_message=handle_monitor_message, -) ->>> {"output_values": [1, 2, 3, 4, 5], "output_manifest": None} -``` - -### Using the child emulator - -To emulate your children in tests, patch the `Child` class with the `ChildEmulator` class. - -```python -from unittest.mock import patch - -from octue.twined.runner import Runner -from octue.twined.cloud.emulators import ChildEmulator - - -app_directory_path = "path/to/directory_containing_app" - -# You can explicitly specify your children here as shown or -# read the same information in from your service configuration file. -children = [ - { - "key": "my_child", - "id": "octue/my-child-service:2.1.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project" - } - }, -] - -runner = Runner( - app_src=app_directory_path, - twine=os.path.join(app_directory_path, "twine.json"), - children=children, - service_id="your-org/your-service:2.1.0", -) - -emulated_children = [ - ChildEmulator( - events=[ - { - "event": { - "kind": "result", - "output_values": [300], - }, - "attributes": { - "datetime": "2024-04-11T10:46:48.236064", - "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", - "retry_count": 0, - "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", - "parent_question_uuid": "5776ad74-52a6-46f7-a526-90421d91b8b2", - "originator_question_uuid": "86dc55b2-4282-42bd-92d0-bd4991ae7356", - "parent": "you/your-service:2.1.0", - "originator": "you/your-service:2.1.0", - "sender": "octue/my-child-service:2.1.0", - "sender_type": "CHILD", - "sender_sdk_version": "0.56.0", - "recipient": "you/your-service:2.1.0" - }, - }, - ], - ) -] - -with patch("octue.runner.Child", side_effect=emulated_children): - analysis = runner.run(input_values={"some": "input"}) - -analysis.output_values ->>> [300] - -analysis.output_manifest ->>> None -``` - -**Notes** - -- If your app uses more than one child, provide more child emulators in - the `emulated_children` list in the order they're asked questions in - your app. -- If a given child is asked more than one question, provide a child - emulator for each question asked in the same order the questions are - asked. - -## Creating a test fixture - -Since the child is _emulated_, it doesn't actually do any calculation - -if you change the inputs, the outputs won't change correspondingly (or -at all). So, it's up to you to define a set of realistic inputs and -corresponding outputs (the list of emulated events) to test your -service. These are called **test fixtures**. - -!!! note - - Unlike a real child, the **inputs** given to the emulator aren't - validated against the schema in the child's twine -this is because the - twine is only available to the real child. This is ok - you're testing - your service, not the child your service contacts. The events given to - the emulator are still validated against the service communication - schema, though. - -You can create test fixtures manually or by using the -`Child.received_events` property after questioning a real child. - -```python -import json -from octue.twined.resources import Child - - -child = Child( - id="octue/my-child:2.1.0", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, -) - -result, question_uuid = child.ask(input_values=[1, 2, 3, 4]) - -child.received_events ->>> [ - { - "event": { - 'kind': 'delivery_acknowledgement', - }, - "attributes": { - ... # Left out for brevity. - }, - }, - { - "event": { - 'kind': 'log_record', - 'log_record': { - 'msg': 'Finished analysis.', - 'args': None, - 'levelname': 'INFO', - ... # Left out for brevity. - }, - }, - "attributes": { - ... # Left out for brevity. - }, - }, - { - "event": { - 'kind': 'result', - 'output_values': {"some": "results"}, - }, - "attributes": { - ... # Left out for brevity. - }, - }, - ] -``` - -You can then feed these into a child emulator to emulate one possible -response of the child: - -```python -from octue.twined.cloud.emulators import ChildEmulator - - -child_emulator = ChildEmulator(events=child.received_events) -result, question_uuid = child_emulator.ask(input_values=[1, 2, 3, 4]) - -result ->>> {"some": "results"} -``` - -You can also create test fixtures from -[downloaded service crash diagnostics](/troubleshooting_services/#creating-test-fixtures-from-diagnostics) diff --git a/docs/troubleshooting_services.md b/docs/troubleshooting_services.md deleted file mode 100644 index f9de86687..000000000 --- a/docs/troubleshooting_services.md +++ /dev/null @@ -1,131 +0,0 @@ -# Troubleshooting services - -## Diagnostics - -Services save the following data to the cloud if they crash while -processing a question (the default), or when they finish processing a -question successfully if diagnostics are permanently turned on (not the -default): - -- Input values -- Input manifest and datasets -- Child configuration values -- Child configuration manifest and datasets -- Inputs to and events received in response to each question the service - asked its children (if it has any). These are stored in the order the - questions were asked. - -!!! important - - For this feature to be enabled, the child must have the - `diagnostics_cloud_path` field in its service configuration - ([`octue.yaml`](/creating_services/#octueyaml) file) set to a Google Cloud Storage path. - -## Accessing diagnostics - -If diagnostics are enabled, a service will upload the diagnostics and -send the upload path to the parent as a log message. A user with -credentials to access this path can use the `octue` CLI to retrieve the -diagnostics data: - -```shell -octue twined question diagnostics -``` - -More information on the command: - -``` ->>> octue twined question diagnostics -h - -Usage: octue twined question diagnostics [OPTIONS] CLOUD_PATH - - Download diagnostics for an analysis from the given directory in - Google Cloud Storage. The cloud path should end in the analysis ID. - - CLOUD_PATH: The path to the directory in Google Cloud Storage containing the - diagnostics data. - -Options: - --local-path DIRECTORY The path to a directory to store the directory of - diagnostics data in. Defaults to the current working - directory. - --download-datasets If provided, download any datasets from the - diagnostics and update their paths in their - manifests to the new local paths. - -h, --help Show this message and exit. -``` - -## Creating test fixtures from diagnostics {#test_fixtures_from_diagnostics} - -You can create test fixtures directly from diagnostics, allowing you to -recreate the exact conditions that caused your service to fail. - -```python -from unittest.mock import patch - -from octue.twined.runner import Runner -from octue.twined.utils.testing import load_test_fixture_from_diagnostics - - -( - configuration_values, - configuration_manifest, - input_values, - input_manifest, - child_emulators, -) = load_test_fixture_from_diagnostics(path="path/to/downloaded/diagnostics") - -# You can explicitly specify your children here as shown or -# read the same information in from your service configuration file. -children = [ - { - "key": "my_child", - "id": "octue/my-child-service:2.1.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project", - } - }, - { - "key": "another_child", - "id": "octue/another-child-service:2.1.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project", - } - } -] - -runner = Runner( - app_src="path/to/directory_containing_app", - twine="twine.json", - children=children, - configuration_values=configuration_values, - configuration_manifest=configuration_manifest, - service_id="your-org/your-service:2.1.0", -) - -with patch("octue.twined.runner.Child", side_effect=child_emulators): - analysis = runner.run(input_values=input_values, input_manifest=input_manifest) -``` - -## Disabling diagnostics - -When asking a question to a child, parents can disable diagnostics -upload in the child on a question-by-question basis by setting -`save_diagnostics` to `"SAVE_DIAGNOSTICS_OFF"` in `Child.ask`. For example: - -```python -from octue.twined.resources import Child - - -child = Child( - id="my-organisation/my-service:2.1.0", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, -) - -answer, question_uuid = child.ask( - input_values={"height": 32, "width": 3}, - save_diagnostics="SAVE_DIAGNOSTICS_OFF", -) -``` diff --git a/docs/twines/about/about_digital_twins.md b/docs/twines/about/about_digital_twins.md deleted file mode 100644 index ade802551..000000000 --- a/docs/twines/about/about_digital_twins.md +++ /dev/null @@ -1,53 +0,0 @@ -# Digital Twins {#digital_twins} - -A digital twin is a virtual representation of a real life being - a -physical asset like a wind turbine or car - or even a human. - -There are three reasons why you might want to create a digital twin: - -: - Monitoring - Prediction - Optimisation - -On its own, a digital twin can be quite useful. For example, a twin -might embody an AI-based analysis to predict power output of a turbine. - -
- -
A digital twin consists of some kind of analysis or -processing task, which could be run many times per second, or daily, -down to occasionally or sometimes only once (the same as a "normal" -analysis).
-
- -Coupling digital twins is generally even more useful. You might wish to -couple your turbine twin with a representation of the local power grid, -and a representation of a factory building to determine power demand\... -enabling you to optimise your factory plant for lowest energy cost -whilst intelligently selling surplus power to the grid. - -
- -
A hierarchy of digital twins. Each blue circle represents a -twin, coupled to its neighbours. Yellow nodes are where schema are used -to connect twins.
-
- -## Gemini Principles {#gemini_principles} - -The Gemini Principles have been derived by the [Centre for Digital Built -Britain -(CDBB)](https://www.cdbb.cam.ac.uk/system/files/documents/TheGeminiPrinciples.pdf). -We strongly recommend you give them a read if embarking on a digital -twins project. - -The aim of **twined** is to enable the following principles. In -particular: - -1. Openness (open-source project to create schema for twins that can be - run anywhere, anywhen) -2. Federation (encouraging a standardised way of connecting twins - together) -3. Security (making sure schemas and data can be read safely) -4. Public Good (see our nano-rant about climate change in - `reason_for_being`{.interpreted-text role="ref"}) diff --git a/docs/twines/about/about_introducing_json_schema.md b/docs/twines/about/about_introducing_json_schema.md deleted file mode 100644 index b75ab93e8..000000000 --- a/docs/twines/about/about_introducing_json_schema.md +++ /dev/null @@ -1,126 +0,0 @@ -# Introducing JSON Schema {#introducing_json_schema} - -`JSON` is a data interchange format that has rapidly taken over as the -defacto web-based data communication standard in recent years. - -`JSONSchema` is a way of specifying what a `JSON` document should -contain. The Schema are, themselves, written in `JSON`! - -Whilst schema can become extremely complicated in some scenarios, they -are best designed to be quite succinct. See below for the schema (and -matching `JSON`) for an integer and a string variable. - -**JSON:** - -```json -{ - "id": 1, - "name": "Tom" -} -``` - -**Schema:** - -```json -{ - "type": "object", - "title": "An id number and a name", - "properties": { - "id": { - "type": "integer", - "title": "An integer number", - "default": 0 - }, - "name": { - "type": "string", - "title": "A string name", - "default": "" - } - } -} -``` - -## Useful resources {#useful_resources} - -Link Resource - ---- - - Useful web tool for inferring schema from existing json - A powerful online editor for json, allowing manipulation of large documents better than most text editors - The JSON standard spec - The (draft standard) JSONSchema spec - A front end library for generating webforms directly from a schema - -## Human readability {#human_readbility} - -Back in our `requirements`{.interpreted-text role="ref"} section, we -noted it was important for humans to read and understand schema. - -The actual documents themselves are pretty easy to read by technical -users. But, for non technical users, readability can be enhanced even -further by the ability to turn `JSONSchema` into web forms -automatically. For our example above, we can autogenerate a web form -straight from the schema: - -
- -
Web form generated from the example schema -above.
-
- -Thus, we can take a schema (or a part of a schema) and use it to -generate a control form for a digital twin in a web interface without -writing a separate form component - great for ease and maintainability. - -## Why not XML? {#why_not_xml} - -In a truly excellent [three-part -blog](https://www.toptal.com/web/json-vs-xml-part-3), writer Seva Savris -takes us through the ups and downs of `JSON` versus `XML`; well worth a -read if wishing to understand the respective technologies better. - -In short, both `JSON` and `XML` are generalised data interchange -specifications and can both can do what we want here. We choose `JSON` -because: - -1. Textual representation is much more concise and easy to understand - (very important where non-developers like engineers and scientists - must be expected to interpret schema) -2. [Attack - vectors](https://www.opswat.com/blog/depth-look-xml-document-attack-vectors). - Because entities in `XML` are not necessarily primitives (unlike in - `JSON`), an `XML` document parser in its default state may leave a - system open to XXE injection attacks and DTD validation attacks, and - therefore requires hardening. `JSON` documents are similarly - afflicated (just like any kind of serialized data) but default - parsers, operating on the premise of only deserializing to primitive - types, are safe by default - it is only when nondefault parsering or - deserialization techniques (such as `JSONP`) are used that the - application becomes vulnerable. By utilising a default `JSON` parser - we can therefore significantly shrink the attack surface of the - system. See [this blog - post](https://blog.securityevaluators.com/xml-vs-json-security-risks-22e5320cf529) - for further discussion. -3. `XML` is powerful\... perhaps too powerful. The standard can be - adapted greatly, resulting in high encapsulation and a high - resilience to future unknowns. Both beneficial. However, this - requires developers of twins to maintain interfaces of very high - complexity, adaptable to a much wider variety of input. To enable - developers to progress, we suggest handling changes and future - unknowns through well-considered versioning, whilst keeping their - API simple. -4. `XML` allows baked-in validation of data and attributes. Whilst - advantageous in some situations, this is not a benefit here. We wish - validation to be one-sided: validation of data accepted/generated by - a digital twin should be occur within (at) the boundaries of that - twin. -5. Required validation capabilities, built into `XML` are achievable - with `JSONSchema` (otherwise missing from the pure `JSON` standard) -6. `JSON` is a more compact expression than XML, significantly reducing - memory and bandwidth requirements. Whilst not a major issue for most - modern PCS, sensors on the edge may have limited memory, and both - memory and bandwidth at scale are extremely expensive. Thus for - extremely large networks of interconnected systems there could be - significant speed and cost savings. diff --git a/docs/twines/about/about_other_considerations.md b/docs/twines/about/about_other_considerations.md deleted file mode 100644 index 03d4ddecb..000000000 --- a/docs/twines/about/about_other_considerations.md +++ /dev/null @@ -1,112 +0,0 @@ -# Other Considerations {#other_considerations} - -A variety of thoughts that arose whilst architecting **twined**. - -## Bash-style stdio {#bash_style_stdio} - -Some thought was given to using a very old-school-unix approach to -piping data between twins, via stdout. - -Whilst attractive (as being a wildly fast way of piping data between -twins on the same machine) it was felt this was insufficiently general, -eg: - -> - where twins don\'t exist on the same machine or container, making it -> cumbersome to engineer common iostreams -> - where slight differences between different shells might lead to -> incompatibilities or changes in behaviour - -And also unfriendly, eg: - -> - engineers or scientists unfamiliar with subtleties of bash shell -> scripting encounter difficulty piping data around -> - difficult to build friendly web based tools to introspect the data -> and configuration -> - bound to be headaches on windows platforms, even though windows now -> supports bash -> - easy to corrupt using third party libraries (e.g. which print to -> stdout) - -## Units {#Units} - -Being used (mostly) for engineering and scientific analysis, it was -tempting to add in a specified sub-schema for units. For example, -mandating that where values can be given in units, they be specified in -a certain way, like: - -```javascript -{ - "wind_speed": { - "value": 10.2, - "units": "mph" - } -} -``` - -or (more succinct): - -```javascript -{ - "wind_speed": 10.2, - "wind_speed_units": "mph" -} -``` - -It\'s still extremely tempting to provide this facility; or at least -provide some way of specifying in the schema what units a value should -be provided in. Thinking about it but don\'t have time right now. If -anybody wants to start crafting a PR with an extension or update to -**twined** that facilitates this; please raise an issue to start -progressing it. - -## Variable Style {#variable_style} - -A premptive stamp on the whinging\... - -Note that in the `JSON` descriptions above, all variables are named in -`snake_case` rather than `camelCase`. This decision, more likely than -even Brexit to divide opinions, is based on: - -- The languages we anticipate being most popular for building twins seem to trend toward snake case (eg - - : [python](https://www.python.org/dev/peps/pep-0008/), - [c++](https://google.github.io/styleguide/cppguide.html)) although - to be fair we might\'ve woefully misjudged which languages start - emerging. - -- The reservation of snake case for the schema spec has the subtle advantage that in future, we might be able to use - - : camelCase within the spec to denote class types in some useful - way, just like in python. Not sure yet; just mulling. - -- The `requirements`{.interpreted-text role="ref"} mention human-readability as a must; - - : [this - paper](https://ieeexplore.ieee.org/document/5521745?tp=&arnumber=5521745&url=http:%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D5521745) - suggests a 20% slower comprehension of camel case than snake, - although to be fair that\'s probably arguable. - -- We\'re starting in Python so are taking a lead from PEP8, which is bar none the most successful style guide on the - - : planet, because it got everybody on the same page really early on. - -If existing code that you\'re dropping in uses camelCase, please don\'t -file that as an issue\... converting property names automatically after -schema validation generation is trivial, there are tons of libraries -(like [humps](https://humps.readthedocs.io/en/latest/)) to do it. - -We\'d also consider a pull request for a built-in utility converting -[to](https://pypi.org/project/camelcase/) and -[from](https://pypi.org/project/snakecase/) that does this following -validation and prior to returning results. Suggest your proposed -approach on the [issues board](https://github.com/octue/twined). - -## Language Choice {#language_choice} - -**twined** is presently released in python only. It won\'t be too hard -to replicate functionality in other languages, and we\'re considering -other languages at present, so might be easily persuadable ;) - -If you require implementation of **twined** in a different language, and -are willing to consider sponsorship of development and maintenance of -that library, please [file an issue](https://github.com/octue/twined). diff --git a/docs/twines/about/about_requirements.md b/docs/twines/about/about_requirements.md deleted file mode 100644 index 9727e323d..000000000 --- a/docs/twines/about/about_requirements.md +++ /dev/null @@ -1,37 +0,0 @@ -# Requirements of the framework {#requirements} - -A _twine_ must describe a digital twin, and have multiple roles. It -must: - -1. Define what data is required by a digital twin, in order to run -2. Define what data will be returned by the twin following a successful - run -3. Define the formats of these data, in such a way that incoming data - can be validated -4. Define what other (1st or 3rd party) twins / services are required - by this one in order for it to run. - -If this weren\'t enough, the description: - -1. Must be trustable (i.e. a _twine_ from an untrusted, corrupt or - malicious third party should be safe to at least read) -2. Must be machine-readable _and machine-understandable_[^1] -3. Must be human-readable _and human-understandable_[^2] -4. Must be discoverable (that is, searchable/indexable) otherwise - people won\'t know it\'s there in orer to use it. - -Fortunately for digital twin developers, several of these requirements -have already been seen for data interchange formats developed for the -web. **twined** uses `JSON` and `JSONSchema` to help interchange data. - -If you\'re not already familiar with `JSONSchema` (or wish to know why -**twined** uses `JSON` over the seemingly more appropriate `XML` -standard), see `introducing_json_schema`{.interpreted-text role="ref"}. - -[^1]: - _Understandable_ essentially means that, once read, the machine or - human knows what it actually means and what to do with it. - -[^2]: - _Understandable_ essentially means that, once read, the machine or - human knows what it actually means and what to do with it. diff --git a/docs/twines/about/index.md b/docs/twines/about/index.md deleted file mode 100644 index fce02f56c..000000000 --- a/docs/twines/about/index.md +++ /dev/null @@ -1,16 +0,0 @@ -# About Twines {#about} - -**Twined** is a framework for describing a digital twin or data service. - -We call these descriptions \"twines\". To just get started building a -_twine_, check out the `quick_start`{.interpreted-text role="ref"}. To -get into the detail of what\'s in a _twine_, see -`anatomy`{.interpreted-text role="ref"}. - -Here, we look at requirements for the framework, our motivations and -background, and some of the decisions made while developing **twined**. - -::: {.toctree maxdepth="1"} -about_digital_twins about_requirements about_introducing_json_schema -about_other_considerations -::: diff --git a/docs/twines/anatomy.md b/docs/twines/anatomy.md deleted file mode 100644 index 96952b1c1..000000000 --- a/docs/twines/anatomy.md +++ /dev/null @@ -1,109 +0,0 @@ -# Anatomy Of The Twine File {#anatomy} - -The main point of **twined** is to enable engineers and scientists to -easily (and rigorously) define a digital twin or data service. - -This is done by adding a `twine.json` file to the repository containing -your code. Adding a _twine_ means you can: - -- communicate (to you or a colleague) what data is required by this - service -- communicate (to another service / machine) what data is required -- deploy services automatically with a provider like - [Octue](https://www.octue.com). - -To just get started building a _twine_, check out the -`quick_start`{.interpreted-text role="ref"}. To learn more about twines -in general, see `about`{.interpreted-text role="ref"}. Here, we describe -the parts of a _twine_ (\"strands\") and what they mean. - -## Strands - -A _twine_ has several sections, called _strands_. Each defines a -different kind of data required (or produced) by the twin. - ---- - -Strand Describes the twin\'s requirements for\... - ---- - -`Configuration Values `{.interpreted-text Data, in JSON form, used for configuration of the -role="ref"} twin/service. - -`Configuration Manifest `{.interpreted-text Files/datasets required by the twin at -role="ref"} configuration/startup - -`Input Values `{.interpreted-text Data, in JSON form, passed to the twin in order -role="ref"} to trigger an analysis - -`Input Manifest `{.interpreted-text role="ref"} Files/datasets passed with Input Values to -trigger an analysis - -`Output Values `{.interpreted-text Data, in JSON form, that will be produced by the -role="ref"} twin (in response to inputs) - -`Output Manifest `{.interpreted-text Files/datasets that will be produced by the twin -role="ref"} (in response to inputs) - -`Credentials `{.interpreted-text role="ref"} Credentials that are required by the twin in -order to access third party services - -`Children `{.interpreted-text role="ref"} Other twins, access to which are required for -this twin to function - -`Monitors `{.interpreted-text role="ref"} Visual and progress outputs from an analysis - ---- - -::: {.toctree maxdepth="1" hidden=""} -anatomy_values anatomy_manifest anatomy_credentials anatomy_monitors -anatomy_children -::: - -## Twine File Schema {#twine_file_schema} - -Because the `twine.json` file itself is in `JSON` format with a strict -structure, **twined** uses a schema to make that twine files are -correctly written (a \"schema-schema\", if you will, since a twine -already contains schema). Try not to think about it. But if you must, -the _twine_ schema is -[here](https://github.com/octue/twined/blob/master/twined/schema/twine_schema.json). - -The first thing **twined** always does is check that the `twine.json` -file itself is valid, and give you a descriptive error if it isn\'t. - -## Other External I/O {#other_external_io} - -A twin might: - -- GET/POST data from/to an external API, -- query/update a database, -- upload files to an object store, -- trigger events in another network, or -- perform pretty much any interaction you can think of with other - applications over the web. - -However, such data exchange may not be controllable by **twined** (which -is intended to operate at the boundaries of the twin) unless the -resulting data is returned from the twin (and must therefore be -compliant with the schema). - -So, there\'s nothing for **twined** to do here, and no need for a strand -in the _twine_ file. However, interacting with third party APIs or -databases might require some credentials. See -`credentials_strand`{.interpreted-text role="ref"} for help with that. - -:::: note -::: title -Note -::: - -This is actually a very common scenario. For example, the purpose of the -twin might be to fetch data (like a weather forecast) from some external -API then return it in the `output_values` for use in a network of -digital twins. But its the twin developer\'s job to do the fetchin\' and -make sure the resulting data is compliant with the -`output_values_schema` (see `values_based_strands`{.interpreted-text -role="ref"}). -:::: diff --git a/docs/twines/anatomy_children.md b/docs/twines/anatomy_children.md deleted file mode 100644 index 29e5fbb45..000000000 --- a/docs/twines/anatomy_children.md +++ /dev/null @@ -1,9 +0,0 @@ -# Children Strand {#children_strand} - -:::: attention -::: title -Attention -::: - -Coming Soon! -:::: diff --git a/docs/twines/anatomy_credentials.md b/docs/twines/anatomy_credentials.md deleted file mode 100644 index 8ed890dc8..000000000 --- a/docs/twines/anatomy_credentials.md +++ /dev/null @@ -1,83 +0,0 @@ -# Credentials Strand {#credentials_strand} - -In order to: - -- GET/POST data from/to an API, -- query a database, or -- connect to a socket (for receiving Values or emitting Values, Monitors - or Logs), - -A digital twin must have _access_ to it. API keys, database URIs, etc -must be supplied to the digital twin but treated with best practice with -respect to security considerations. The purpose of the `credentials` -strand is to dictate what credentials the twin requires in order to -function. - -## Defining the Credentials Strand {#defining_the_credentials_strand} - -This is the simplest of the strands, containing a list of credentials -(whose `NAMES_SHOULD_BE_SHOUTY_SNAKE_CASE`) with a reminder of the -purpose. - -```javascript -{ - "credentials": [ - { - "name": "SECRET_THE_FIRST", - "purpose": "Token for accessing a 3rd party API service" - }, - { - "name": "SECRET_THE_SECOND", - "purpose": "Token for accessing a 3rd party API service" - }, - { - "name": "SECRET_THE_THIRD", - "purpose": "Another secret, like a password for a sandbox or local database" - } - ] -} -``` - -## Supplying Credentials {#supplying_credentials} - -:::: attention -::: title -Attention -::: - -_Credentials should never be hard-coded into application code_ - -Do you trust the twin code? If you insert credentials to your own -database into a digital twin provided by a third party, you better be -very sure that twin isn\'t going to scrape all that data out then send -it elsewhere! - -Alternatively, if you\'re building a twin requiring such credentials, -it\'s your responsibility to give the end users confidence that you\'re -not abusing their access. - -There\'ll be a lot more discussion on these issues, but it\'s outside -the scope of **twined** - all we do here is make sure a twin has the -credentials it requires. -:::: - -Credentials should be securely managed by whatever system is managing -the twin, then made accessible to the twin in the form of environment -variables: - -```javascript -SERVICE_API_KEY = - someLongTokenTHatYouProbablyHaveToPayTheThirdPartyProviderLoadsOfMoneyFor; -``` - -Credentials may also reside in a `.env` file in the current directory, -either in the format above (with a new line for each variable) or, for -convenience, as bash exports like: - -```javascript -export SERVICE_API_KEY=someLongTokenTHatYouProbablyHaveToPayTheThirdPartyProviderLoadsOfMoneyFor -``` - -The `validate_credentials()` method of the `Twine` class checks for -their presence and, where contained in a `.env` file, ensures they are -loaded into the environment. diff --git a/docs/twines/anatomy_manifest.md b/docs/twines/anatomy_manifest.md deleted file mode 100644 index dfb3082cd..000000000 --- a/docs/twines/anatomy_manifest.md +++ /dev/null @@ -1,420 +0,0 @@ -# Manifest-based Strands {#manifest_strands} - -Frequently, twins operate on files containing some kind of data. These -files need to be made accessible to the code running in the twin, in -order that their contents can be read and processed. Conversely, a twin -might produce an output dataset which must be understood by users. - -The `configuration_manifest`, `input_manifest` and `output_manifest` -strands describe what kind of datasets (and associated files) are -required / produced. - -:::: note -::: title -Note -::: - -Files are always contained in datasets, even if there\'s only one file. -It\'s so that we can keep nitty-gritty file metadata separate from the -more meaningful, higher level metadata like what a dataset is for. -:::: - -:::::: tabs -::: group-tab -Configuration Manifest Strand - -This describes datasets/files that are required at startup of the twin / -service. They typically contain a resource that the twin might use -across many analyses. - -For example, a twin might predict failure for a particular component, -given an image. It will require a trained ML model (saved in a -`*.pickle` or `*.json`). While many thousands of predictions might be -done over the period that the twin is deployed, all predictions are done -using this version of the model - so the model file is supplied at -startup. -::: - -::: group-tab -Input Manifest Strand - -These files are made available for the twin to run a particular analysis -with. Each analysis will likely have different input datasets. - -For example, a twin might be passed a dataset of LiDAR `*.scn` files and -be expected to compute atmospheric flow properties as a timeseries -(which might be returned in the -`output values `{.interpreted-text role="ref"} for -onward processing and storage). -::: - -::: group-tab -Output Manifest Strand - -Files are created by the twin during an analysis, tagged and stored as -datasets for some onward purpose. This strand is not used for sourcing -data; it enables users or other services to understand appropriate -search terms to retrieve datasets produced. -::: -:::::: - -## Describing Manifests {#describing_manifests} - -Manifest-based strands are a **description of what files are needed**. -The purpose of the manifest strands is to provide a helper to a wider -system providing datafiles to digital twins. - -::::::::::::::::::::: tabs -:::::::: group-tab -Configuration Manifest Strand - -::::::: accordion -:::: accordion-row -Show twine containing this strand - -::: {.literalinclude language="javascript"} -../../octue/twined/examples/damage_classifier_service/twine.json -::: -:::: - -:::: accordion-row -Show a matching file manifest - -::: {.literalinclude language="javascript"} -../../octue/twined/examples/damage_classifier_service/data/configuration_manifest.json -::: -:::: -::::::: -:::::::: - -:::::::: group-tab -Input Manifest Strand - -Here we specify that two datasets (and all or some of the files -associated with them) are required, for a service that cross-checks -meteorological mast data and power output data for a wind farm. - -::::::: accordion -:::: accordion-row -Show twine containing this strand - -::: {.literalinclude language="javascript"} -../../octue/twined/examples/met_mast_scada_service/strands/input_manifest.json -::: -:::: - -:::: accordion-row -Show a matching file manifest - -::: {.literalinclude language="javascript"} -../../octue/twined/examples/met_mast_scada_service/data/input_manifest.json -::: -:::: -::::::: -:::::::: - -:::::::: group-tab -Output Manifest Strand - -::::::: accordion -:::: accordion-row -Show twine containing this strand - -::: {.literalinclude language="javascript"} -../../octue/twined/examples/met_mast_scada_service/strands/output_manifest.json -::: -:::: - -:::: accordion-row -Show a matching file manifest - -::: {.literalinclude language="javascript"} -../../octue/twined/examples/met_mast_scada_service/data/output_manifest.json -::: -:::: -::::::: -:::::::: -::::::::::::::::::::: - -## File tag templates {#file_tag_templates} - -Datafiles can be tagged with key-value pairs of relevant metadata that -can be used in analyses. Certain datasets might need one set of metadata -on each file, while others might need a different set. The required (or -optional) file tags can be specified in the twine in the -`file_tags_template` property of each dataset of any `manifest` strand. -Each file in the corresponding manifest strand is then validated against -its dataset\'s file tag template to ensure the required tags are -present. - -::::::::::: tabs -:::::: group-tab -Manifest strand with file tag template - -The example below is for an input manifest, but the format is the same -for configuration and output manifests. - -::::: accordion -::: accordion-row -Show twine containing a manifest strand with a file tag template - -```javascript -{ - "input_manifest": { - "datasets": [ - { - "key": "met_mast_data", - "purpose": "A dataset containing meteorological mast data", - "file_tags_template": { - "type": "object", - "properties": { - "manufacturer": {"type": "string"}, - "height": {"type": "number"}, - "is_recycled": {"type": "boolean"} - }, - "required": ["manufacturer", "height", "is_recycled"] - } - } - ] - } -} -``` - -::: - -::: accordion-row -Show a matching file manifest - -```javascript -{ - "id": "8ead7669-8162-4f64-8cd5-4abe92509e17", - "datasets": [ - { - "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", - "name": "met_mast_data", - "tags": {}, - "labels": ["met", "mast", "wind"], - "files": [ - { - "path": "input/datasets/7ead7669/file_1.csv", - "cluster": 0, - "sequence": 0, - "extension": "csv", - "labels": ["mykeyword1", "mykeyword2"], - "tags": { - "manufacturer": "vestas", - "height": 500, - "is_recycled": true - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "file_1.csv" - }, - { - "path": "input/datasets/7ead7669/file_1.csv", - "cluster": 0, - "sequence": 1, - "extension": "csv", - "labels": [], - "tags": { - "manufacturer": "vestas", - "height": 500, - "is_recycled": true - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "file_1.csv" - } - ] - } - ] -} -``` - -::: -::::: -:::::: - -:::::: group-tab -Manifest strand with a remote file tag template - -A remote reference can also be given for a file tag template. If the tag -template somewhere public, this is useful for sharing the template -between one or more teams working on the same type of data. - -The example below is for an input manifest, but the format is the same -for configuration and output manifests. It also shows two different tag -templates being specified for two different types of dataset required by -the manifest. - -::::: accordion -::: accordion-row -Show twine using a remote tag template - -```javascript -{ - "input_manifest": { - "datasets": [ - { - "key": "met_mast_data", - "purpose": "A dataset containing meteorological mast data", - "file_tags_template": { - "$ref": "https://refs.schema.octue.com/octue/my-file-type-tag-template/0.0.0.json" - } - }, - { - "key": "some_other_kind_of_dataset", - "purpose": "A dataset containing something else", - "file_tags_template": { - "$ref": "https://refs.schema.octue.com/octue/another-file-type-tag-template/0.0.0.json" - } - } - ] - } -} -``` - -::: - -::: accordion-row -Show a matching file manifest - -```javascript -{ - "id": "8ead7669-8162-4f64-8cd5-4abe92509e17", - "datasets": [ - { - "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", - "name": "met_mast_data", - "tags": {}, - "labels": ["met", "mast", "wind"], - "files": [ - { - "path": "input/datasets/7ead7669/file_1.csv", - "cluster": 0, - "sequence": 0, - "extension": "csv", - "labels": ["mykeyword1", "mykeyword2"], - "tags": { - "manufacturer": "vestas", - "height": 500, - "is_recycled": true - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "file_1.csv" - }, - { - "path": "input/datasets/7ead7669/file_1.csv", - "cluster": 0, - "sequence": 1, - "extension": "csv", - "labels": [], - "tags": { - "manufacturer": "vestas", - "height": 500, - "is_recycled": true - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "file_1.csv" - } - ] - }, - { - "id": "7ead7669-8162-4f64-8cd5-4abe92509e29", - "name": "some_other_kind_of_dataset", - "tags": {}, - "labels": ["my-label"], - "files": [ - { - "path": "input/datasets/7eadpp9/interesting_file.dat", - "cluster": 0, - "sequence": 0, - "extension": "dat", - "labels": [], - "tags": { - "length": 864, - "orientation_angle": 85 - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae9071", - "name": "interesting_file.csv" - }, - } - ] -} -``` - -::: -::::: -:::::: -::::::::::: - -TODO - clean up or remove this section - -## How Filtering Works {#how_filtering_works} - -It\'s the job of **twined** to make sure of two things: - -1. make sure the _twine_ file itself is valid, - - > **File data (input, output)** - > - > Files are not streamed directly to the digital twin (this would - > require extreme bandwidth in whatever system is orchestrating all - > the twins). Instead, files should be made available on the local - > storage system; i.e. a volume mounted to whatever container or VM - > the digital twin runs in. - > - > Groups of files are described by a `manifest`, where a manifest is - > (in essence) a catalogue of files in a dataset. - > - > A digital twin might receive multiple manifests, if it uses - > multiple datasets. For example, it could use a 3D point cloud - > LiDAR dataset, and a meteorological dataset. - > - > ```javascript - > { - > "manifests": [ - > { - > "type": "dataset", - > "id": "3c15c2ba-6a32-87e0-11e9-3baa66a632fe", // UUID of the manifest - > "files": [ - > { - > "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", // UUID of that file - > "sha1": "askjnkdfoisdnfkjnkjsnd" // for quality control to check correctness of file contents - > "name": "Lidar - 4 to 10 Dec.csv", - > "path": "local/file/path/to/folder/containing/it/", - > "type": "csv", - > "metadata": { - > }, - > "size_bytes": 59684813, - > "tags": {"special_number": 1}, - > "labels": ["lidar", "helpful", "information", "like"], // Searchable, parsable and filterable - > }, - > { - > "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - > "name": "Lidar - 11 to 18 Dec.csv", - > "path": "local/file/path/to/folder/containing/it/", - > "type": "csv", - > "metadata": { - > }, - > "size_bytes": 59684813, - > "tags": {"special_number": 2}, - > "labels": ["lidar", "helpful", "information", "like"] // Searchable, parsable and filterable - > }, - > { - > "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - > "name": "Lidar report.pdf", - > "path": "local/file/path/to/folder/containing/it/", - > "type": "pdf", - > "metadata": { - > }, - > "size_bytes": 484813, - > "tags": {}, - > "labels": ["report"] // Searchable, parsable and filterable - > } - > ] - > }, - > { - > // ... another dataset manifest ... - > } - > ] - > } - > ``` diff --git a/docs/twines/anatomy_monitors.md b/docs/twines/anatomy_monitors.md deleted file mode 100644 index ee8f0a7e1..000000000 --- a/docs/twines/anatomy_monitors.md +++ /dev/null @@ -1,57 +0,0 @@ -# Monitor Message Strand {#monitors_strand} - -The `monitor_message_schema` strand is _values-based_ meaning the data -that matches the strand is in JSON form. It is a _json schema_ which -describes a monitor message. - -:::: tabs -::: group-tab -Monitors Strand - -There are two kinds of monitoring data required from a digital twin. - -**Monitor data (output)** - -Values for health and progress monitoring of the twin, for example -percentage progress, iteration number and status - perhaps even -residuals graphs for a converging calculation. Broadly speaking, this -should be user-facing information. - -_This kind of monitoring data can be in a suitable form for display on a -dashboard_ - -**Log data (output)** - -Logged statements, typically in iostream form, produced by the twin -(e.g. via python\'s `logging` module) must be capturable as an output -for debugging and monitoring purposes. Broadly speaking, this should be -developer-facing information. -::: -:::: - -Let\'s look at basic examples for twines containing each of these -strands: - -:::: tabs -::: group-tab -Monitors Strand - -**Monitor data (output)** - -```javascript -{ - "monitor_message_schema": { - "type": "object", - "properties": { - "my_property": { - "type": "number" - } - }, - "required": ["my_property"] - } -} -``` - -**Log data (output)** -::: -:::: diff --git a/docs/twines/anatomy_values.md b/docs/twines/anatomy_values.md deleted file mode 100644 index 2a2c14b9b..000000000 --- a/docs/twines/anatomy_values.md +++ /dev/null @@ -1,157 +0,0 @@ -# Values-based Strands {#values_based_strands} - -The `configuration_values_schema`, `input_values_schema` and -`output_values_schema` strands are _values-based_, meaning the data that -matches these strands is in JSON form. - -Each of these strands is a _json schema_ which describes that data. - -:::::::: tabs -::: group-tab -Configuration Values Strand - -This strand is a `configuration_values_schema`, that is used to check -validity of any `configuration_values` data supplied to the twin at -startup. - -The Configuration Values Strand is generally used to define control -parameters relating to what the twin should do, or how it should -operate. - -For example, should it produce output images as low resolution PNGs or -as SVGs? How many iterations of a fluid flow solver should be used? What -is the acceptable error level on an classifier algorithm? -::: - -::::: group-tab -Input Values Strand - -This strand is an `input_values_schema`, that is used to check validity -of `input_values` data supplied to the twin at the beginning of an -analysis task. - -The Input Values Strand is generally used to define actual data which -will be processed by the twin. Sometimes, it may be used to define -control parameters specific to an analysis. - -For example, if a twin cleans and detects anomalies in a 10-minute -timeseries of 1Hz data, the `input_values` might contain an array of -data and a list of corresponding timestamps. It may also contain a -control parameter specifying which algorithm is used to do the -detection. - -:::: note -::: title -Note -::: - -Depending on the way the twin is deployed (see -`deployment`{.interpreted-text role="ref"}), the `input_values` might -come in from a web request, over a websocket or called directly from the -command line or another library. - -However they come, if the new `input_values` validate against the -`input_values_schema` strand, then analysis can proceed. -:::: -::::: - -::: group-tab -Output Values Strand - -This strand is an `output_values_schema`, that is used to check results -(`output_values`) computed during an analysis. This ensures that the -application wrapped up within the _twine_ is operating correctly, and -enables other twins/services or the end users to see what outputs they -will get. - -For example,if a twin cleans and detects anomalies in a 10-minute -timeseries of 1Hz data, the `output_values` might contain an array of -data interpolated onto regular timestamps, with missing values filled in -and a list of warnings where anomalies were found. -::: -:::::::: - -Let\'s look at basic examples for twines containing each of these -strands: - -:::::: tabs -::: group-tab -Configuration Values Strand - -This _twine_ contains an example `configuration_values_schema` with one -control parameter. - -[Many more detailed and specialised examples are available in the GitHub -repository](https://github.com/octue/twined/tree/main/examples) - -```javascript -{ - "configuration_values_schema": { - "title": "The example configuration form", - "description": "The Configuration Values Strand of an example twine", - "type": "object", - "properties": { - "n_iterations": { - "description": "An example of an integer configuration variable, called 'n_iterations'.", - "type": "integer", - "minimum": 1, - "maximum": 10, - "default": 5 - } - } - } -} -``` - -Matching `configuration_values` data could look like this: - -```javascript -{ - "n_iterations": 8, -} -``` - -::: - -::: group-tab -Input Values Strand - -This _twine_ contains an example `input_values_schema` with one input -value, which marked as required. - -Many more detailed and specialised examples are available in -`examples`{.interpreted-text role="ref"}. - -```javascript -{ - "input_values_schema": { - "title": "Input Values", - "description": "The input values strand of an example twine, with a required height value", - "type": "object", - "properties": { - "height": { - "description": "An example of an integer value called 'height'", - "type": "integer", - "minimum": 2 - } - }, - "required": ["height"] - }, -``` - -Matching `input_values` data could look like this: - -```javascript -{ - "height": 13, -} -``` - -::: - -::: group-tab -Output Values Strand - -Stuff -::: -:::::: diff --git a/docs/twines/examples.md b/docs/twines/examples.md deleted file mode 100644 index 974c05706..000000000 --- a/docs/twines/examples.md +++ /dev/null @@ -1,200 +0,0 @@ -# Examples - -Here, we look at example use cases for the library, and show how to use -it in python. - -It\'s also well worth looking at the unit test cases copied straight -from the unit test cases, so you can always check there to see how -everything hooks up. - -## \[Simple\] Equipment installation cost {#example_equipment_installation_cost} - -::::: tabs -::: group-tab -Scenario - -You need to provide your team with an estimate for installation cost of -an equipment foundation. - -It\'s a straightforward calculation for you, but the Logistics Team -keeps changing the installation position, to try and optimise the -overall project logistics. - -Each time the locations change, the GIS team gives you an updated -embedment depth, which is what you use (along with steel cost and -foundation type), to calculate cost and report it back. - -This twine allows you to define to create a wrapper around your scripts -that communicates to the GIS team what you need as an input, communicate -to the logistics team what they can expect as an output. - -When deployed as a digital twin, the calculation gets automatically -updated, leaving you free to get on with all the other work! -::: - -::: group-tab -Twine - -We specify the `steel_cost` and `foundation_type` as `configuration` -values, which you can set on startup of the twin. - -Once the twin is running, it requires the `embedment_depth` as an -`input_value` from the GIS team. A member of the GIS team can use your -twin to get `foundation_cost` directly. - -```javascript -{ - "title": "Foundation Cost Model", - "description": "This twine helps compute the cost of an installed foundation.", - "children": [ - ], - "configuration_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Foundation cost twin configuration", - "description": "Set config parameters and constants at startup of the twin.", - "type": "object", - "properties": { - "steel_cost": { - "description": "The cost of steel in GBP/m^3. To get a better predictive model, you could add an economic twin that forecasts the cost of steel using the project timetable.", - "type": "number", - "minimum": 0, - "default": 3000 - }, - "foundation_type": { - "description": "The type of foundation being used.", - "type": "string", - "pattern": "^(monopile|twisted-jacket)$", - "default": "monopile" - } - } - }, - "input_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Input Values schema for the foundation cost twin", - "description": "These values are supplied to the twin asynchronously over a web socket. So as these values change, the twin can reply with an update.", - "type": "object", - "properties": { - "embedment_depth": { - "description": "Embedment depth in metres", - "type": "number", - "minimum": 10, - "maximum": 500 - } - } - }, - "output_manifest": { - "datasets": [] - }, - "output_values_schema": { - "title": "Output Values schema for the foundation cost twin", - "description": "The response supplied to a change in input values will always conform to this schema.", - "type": "object", - "properties": { - "foundation_cost": { - "description": "The foundation cost.", - "type": "integer", - "minimum": 2 - } - } - } -} -``` - -::: -::::: - -## \[Simple\] Site weather conditions {#example_site_weather_conditions} - -::::: tabs -::: group-tab -Scenario - -You need to be able to get characteristic weather conditions at a -specific location, for a range of reasons including assessing extreme -design loads. The values you need are computed in a script, which calls -a Weather API (provided by a third party), but also needs a dataset of -\"Wind Resource\" files. -::: - -::: group-tab -Twine - -```javascript -{ - "title": "Weather Service Digital Twin", - "description": "Provides a model for design extreme weather conditions given a location", - "notes": "Easily extendable with children to add forecast and historical data of different types.", - "credentials": [ - { - "name": "WEATHER_API_SECRET_KEY", - "purpose": "Token for accessing a 3rd party weather API service" - } - ], - "input_manifest": { - "datasets": [ - { - "key": "wind_resource_data", - "purpose": "A dataset containing Wind Resource Grid files" - } - ] - }, - "input_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Input Values for the weather service twin", - "description": "This is a simple example for getting metocean conditions at a single location", - "type": "object", - "properties": { - "location": { - "description": "Location", - "type": "object", - "properties": { - "latitude": { - "type": "number", - "minimum": -90, - "maximum": 90 - }, - "longitude": { - "type": "number", - "minimum": -180, - "maximum": 180 - }, - "srid": { - "description": "The Spatial Reference System ID for the coordinate. Default is 4326 (WGS84)", - "type": "integer", - "default": 4326 - } - } - } - } - }, - "output_manifest": { - "datasets": [ - { - "key": "production_data", - "purpose": "A dataset containing production data", - "tags": {"cleaned": true}, - "labels": ["production", "wind"] - } - ] - }, - "output_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Output Values for the metocean service twin", - "description": "The output values strand of an example twine", - "type": "object", - "properties": { - "water_depth": { - "description": "Design water depth for use in concept calculations", - "type": "number" - }, - "extreme_wind_speed": { - "description": "Extreme wind speed value for use in concept calculations", - "type": "number" - } - } - } -} -``` - -::: -::::: diff --git a/docs/twines/index.md b/docs/twines/index.md deleted file mode 100644 index 6722bf27e..000000000 --- a/docs/twines/index.md +++ /dev/null @@ -1,109 +0,0 @@ -:::: attention -::: title -Attention -::: - -This library is in very early stages. Like the idea of it? Please [star -us on GitHub](https://github.com/octue/twined) and contribute via the -[issues board](https://github.com/octue/twined/issues) and -[roadmap](https://github.com/octue/twined/projects/1). -:::: - -# Twined - -**twined** is a library to help create and connect -`digital_twins`{.interpreted-text role="ref"} and data services. - -> - -A digital twin is a virtual representation of a real life being - a -physical asset like a wind turbine or car - or even a human. Like real -things, digital twins need to interact, so can be connected together, -but need a common communication framework to do so. - -**twined** helps you to define a single file, a \"twine\", that defines -a digital twin / data service. It specifies specifying its data -interfaces, connections to other twins, and other requirements. - -Any person, or any computer, can read a twine and understand -_what-goes-in_ and _what-comes-out_. That makes it easy to collaborate -with other teams, since everybody is crystal clear about what\'s needed. - -
- -
Digital twins / data services connected in a hierarchy. Each -blue circle represents a twin, coupled to its neighbours. Yellow nodes -are where schema are used to connect twins.
-
- -## Aims - -**twined** provides a toolkit to help create and validate \"twines\" - -descriptions of a digital twin, what data it requires, what it does and -how it works. - -The goals of this **twined** library are as follows: - -: - Provide a clear framework for what a _twine_ can and/or must -contain - Provide functions to validate incoming data against a known -_twine_ - Provide functions to check that a _twine_ itself is valid - Provide (or direct you to) tools to create _twines_ describing -what you require - -In `anatomy`{.interpreted-text role="ref"}, we describe the different -parts of a twine (examining how digital twins connect and interact\... -building them together in hierarchies and networks). But you may prefer -to dive straight in with the `quick_start`{.interpreted-text role="ref"} -guide. - -The scope of **twined** is not large. Many other libraries will deal -with hosting and deploying digital twins, still more will deal with the -actual analyses done within them. **twined** purely deals with parsing -and checking the information exchanged. - -## Raison d\'etre {#reason_for_being} - -Octue believes that a lynchpin of solving climate change is the ability -for all engineering, manufacturing, supply chain and infrastructure -plant to be connected together, enabling strong optimisation and -efficient use of these systems. - -To enable engineers and scientists to build, connect and run digital -twins in large networks (or even in small teams!) it is necessary for -everyone to be on the same page - the -`gemini_principles`{.interpreted-text role="ref"} are a great way to -start with that, which is why we\'ve released this part of our -technology stack as open source, to support those principles and help -develop a wider ecosystem. - -The main goal is to **help engineers and scientists focus on doing -engineering and science** - instead of apis, data cleaning/management, -and all this cloud-pipeline-devops-test-ci-ml BS that takes up 90% of a -scientist\'s time, when they should be spending their valuable time -researching migratory patterns of birds, or cell structures, or wind -turbine performance, or whatever excites them. - -## Uses - -At [Octue](https://www.octue.com), **twined** is used as a core part of -our application creation process: - -> - As a format to communicate requirements to our partners in research -> projects -> - As a tool to validate incoming data to digital twins -> - As a framework to help establish schema when designing digital twins -> - As a source of information on digital twins in our network, to help -> map and connect twins together - -We\'d like to hear about your use case. Please get in touch! - -We use the [GitHub Issue Tracker](https://github.com/octue/twined) to -manage bug reports and feature requests. Please note, this is not a -\"general help\" forum; we recommend Stack Overflow for such questions. -For really gnarly issues or for help designing digital twin schema, -Octue is able to provide application support services for those building -digital twins using **twined**. - -::: {.toctree maxdepth="2"} -self quick_start anatomy about deployment license version_history -::: diff --git a/docs/twines/lifecycle.md b/docs/twines/lifecycle.md deleted file mode 100644 index 133e7f096..000000000 --- a/docs/twines/lifecycle.md +++ /dev/null @@ -1,34 +0,0 @@ -> Data matching the `configuration_values_schema` is supplied to the -> digital twin / data service at startup. -> -> It\'s generally used to define control parameters relating to what the -> service should do, or how it should operate. For example, should it -> produce output images as low resolution PNGs or as SVGs? How many -> iterations of a fluid flow solver should be used? What is the -> acceptable error level on an classifier algorithm? -> -> Input Values -> -> Once configuration data supplied to a service has been validated, it -> can accept inputs and run analyses using them. -> -> Depending on the way it\'s deployed (see -> `deployment`{.interpreted-text role="ref"}), the `input_values` might -> come in from a web request, over a websocket or called directly from -> the command line or another library. -> -> However it comes, new `input_values`, which are in `JSON` format, are -> checked against the `input_values_schema` strand of the twine. If they -> match, then analysis can proceed. -> -> Output Values -> -> Once a service has Data matching the `output_values_schema` is -> supplied to the service while it\'s running. Depending on the way -> it\'s deployed, the values might come in from a web request, over a -> websocket or called directly from another library -> -> Input For example current rotor speed, or forecast wind direction. -> -> Values might be passed at instantiation of a twin (typical -> application-like process) or via a socket. diff --git a/docs/twines/quick_start.md b/docs/twines/quick_start.md deleted file mode 100644 index c8bee73f0..000000000 --- a/docs/twines/quick_start.md +++ /dev/null @@ -1,5 +0,0 @@ -# Quick Start {#quick_start} - -::: {.toctree maxdepth="2"} -quick_start_installation quick_start_create_your_first_twine -::: diff --git a/docs/twines/quick_start_create_your_first_twine.md b/docs/twines/quick_start_create_your_first_twine.md deleted file mode 100644 index 0d0f997a0..000000000 --- a/docs/twines/quick_start_create_your_first_twine.md +++ /dev/null @@ -1,112 +0,0 @@ -# Create your first twine {#create_your_first_twine} - -Let\'s say we want a digital twin that accepts two values, uses them to -make a calculation, then gives the result. Anyone connecting to the twin -will need to know what values it requires, and what it responds with. - -First, create a blank text file, call it [twine.json]{.title-ref}. -We\'ll give the twin a title and description. Paste in the following: - -```javascript -{ - "title": "My first digital twin... of an atomising discombobulator", - "description": "A simple example... estimates the `foz` value of an atomising discombobulator." -} -``` - -Now, let\'s define an input values strand, to specify what values are -required by the twin. For this we use a json schema (you can read more -about them in `introducing_json_schema`{.interpreted-text role="ref"}). -Add the `input_values` field, so your twine looks like this: - -```javascript -{ - "title": "My first digital twin", - "description": "A simple example to build on..." - "input_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Input Values schema for my first digital twin", - "description": "These values are supplied to the twin by another program (often over a websocket, depending on your integration provider). So as these values change, the twin can reply with an update.", - "type": "object", - "properties": { - "foo": { - "description": "The foo value... speed of the discombobulator's input bobulation module, in m/s", - "type": "number", - "minimum": 10, - "maximum": 500 - }, - "baz": { - "description": "The baz value... period of the discombobulator's recombulation unit, in s", - "type": "number", - "minimum": 0, - "maximum": 1000 - } - } - } -} -``` - -Finally, let\'s define an output values strand, to define what kind of -data is returned by the twin: - -```javascript -"output_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Output Values schema for my first digital twin", - "description": "The twin will output data that matches this schema", - "type": "object", - "properties": { - "foz": { - "description": "Estimate of the foz value... efficiency of the discombobulator in %", - "type": "number", - "minimum": 10, - "maximum": 500 - } - } -} -``` - -# Load the twine {#load_the_twine} - -**twined** provides a [Twine()]{.title-ref} class to load a twine (from -a file or a json string). The loading process checks the twine itself is -valid. It\'s as simple as: - -```py -from octue.twined import Twine - -my_twine = Twine(source='twine.json') -``` - -# Validate some inputs {#validate_some_inputs} - -Say we have some json that we want to parse and validate, to make sure -it matches what\'s required for input values. - -```py -my_input_values = my_twine.validate_input_values(json='{"foo": 30, "baz": 500}') -``` - -You can read the values from a file too. Paste the following into a file -named `input_values.json`: - -```javascript -{ - "foo": 30, - "baz": 500 -} -``` - -Then parse and validate directly from the file: - -```py -my_input_values = my_twine.validate_input_values(source="input_values.json") -``` - -:::: attention -::: title -Attention -::: - -LIBRARY IS UNDER CONSTRUCTION! WATCH THIS SPACE FOR MORE! -:::: diff --git a/docs/updating_services.md b/docs/updating_services.md deleted file mode 100644 index f6ca83a63..000000000 --- a/docs/updating_services.md +++ /dev/null @@ -1,109 +0,0 @@ -# Updating a Twined service - -This page describes how to update an existing, deployed Twined service - -in other words, how to deploy a new Twined service revision. - -We assume that: - -- Your service's repository is on GitHub and you have push access to it -- The [standard Twined service deployment GitHub Actions - workflow](https://github.com/octue/workflows/blob/main/.github/workflows/build-twined-service.yml) - is set up in the repository and being used to build and push the - service image to the artifact registry on merge of a pull request into - the `main` branch (see an example - [here](https://github.com/octue/example-service-kueue/blob/main/.github/workflows/release.yml)) -- A release workflow is set up that will tag and release the new service - revision on GitHub (see an example - [here](https://github.com/octue/example-service-kueue/blob/main/.github/workflows/release.yml)) - -## Instructions - -1. Check out and pull the `main` branch to make sure you're up to date - with the latest changes - - ```shell - git checkout main - git pull - ``` - -2. Install your service locally so you can run the tests and your - development environment can lint the code etc.: - - ```shell - poetry install - ``` - -3. Set up [pre-commit](https://pre-commit.com/) to enforce code - quality: - - ```shell - pre-commit install && pre-commit install -t commit-msg - ``` - -4. Check out a new branch so you can work independently of any other - work on the code happening at the same time - - ```shell - git checkout -b my-new-feature - ``` - -5. Add and make changes to your app's code as needed, committing each - self-contained change. Use the [Conventional - Commits](https://www.conventionalcommits.org/en/v1.0.0/) commit - message format so the new version for your service can be - automatically calculated. - - ```shell - git add a-new-file another-new-file - git commit -m "Your commit message" - ...repeat... - ``` - - Push your commits frequently so your work is backed up on GitHub - - ```shell - git push - ``` - -6. Write any new tests you need to verify your code works and update - any old tests as needed - -7. Run the tests locally using `pytest` and fix anything that makes - them fail - - ![image](images/updating_services/pytest.png) - -8. Update the [semantic version](https://semver.org/) of your app. This - communicates to anyone updating from a previous version of the - service whether they can use it as before or if there might be - changes they need to make to their own code or data first. - - - `poetry version patch` for a bug fix or small non-code change - - `poetry version minor` for a new feature - - `poetry version major` for a breaking change - - Don't forget to commit this change, too. - -9. When you're ready to review the changes, head to GitHub and open a - pull request of your branch into `main`. This makes it easy for you - and anyone else to see what's changed. Check the "Files Changed" - tab to make sure everything's there and consistent (it's easy to - forget to push a commit). Ask your colleagues to review the code if - required. - - ![image](images/updating_services/diff.png) - -10. When you're ready to release the new version of your service, check - that the GitHub checks have passed. These ensure code quality, that - the tests pass, and that the new version number is correct. - -> ![image](images/updating_services/checks.png) - -11. Merge the pull request into `main`. This will run the deployment - workflow (usually called `cd` - continuous deployment), making the - new version of the service available to everyone. -12. Check that the deployment workflow has run successfully (this can - take a few minutes). You can check the progress in the "Actions" - tab of the GitHub repository - -> ![image](images/updating_services/deployment.png) From d1fc9834fd27ad6a3a758ea0b35121fb50af556d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 2 Oct 2025 17:26:55 -0400 Subject: [PATCH 44/48] DOC: Add glossary skipci --- docs/glossary.md | 18 ++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 19 insertions(+) create mode 100644 docs/glossary.md diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 000000000..f6c0ed668 --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,18 @@ +| Term | Definition | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Asking a question | Asking a question is sending data (input values and/or an input manifest) to a service for processing/analysis. | +| Child | A child is a service that can be asked a question. This name reflects the tree structure of services (specifically, a directed acyclic graph, or DAG) formed by the service asking the question (the parent), the child it asks the question to, any children that the child asks questions to as part of forming its answer, and so on. | +| Datafile | A datafile is a single local or cloud file and its metadata. | +| Dataset | A dataset is a set of related datafiles that exist in the same location and some metadata. | +| Locality | Locality is the type of location of a datafile or dataset. Possible types are: cloud-based (it exists only in the cloud); local (it exists only on your local filesystem); and hybrid (it's cloud-based but has been downloaded for low-latency reading/writing) | +| Manifest | A manifest is a set of related cloud and/or local datasets and some metadata. Typically produced by or needed for processing by a Twined service. | +| Parent | A parent is a service that asks a question to another service (a child). | +| Receiving an answer | Receiving data (output values and/or an output manifest) from a service you asked a question to. | +| Service | A service (also known as a Twined service) is a data service or digital twin built with the Twined framework that can be asked questions (input data), process them, and return answers (output data). Twined services can communicate with each other with minimal extra setup. | +| Service ID | A service ID can be an SRUID or just the service namespace and name. It can be used to ask a question to a service without specifying a specific revision of it. This enables asking questions to, for example, the service octue/my-service and automatically having them routed to its default (usually latest) revision. An SRUID is a special case of a service ID. | +| Service name | A name to uniquely identify the service within its namespace. This usually corresponds to the name of the GitHub repository for the service. Names must be lower kebab case (i.e. they may contain the letters [a-z], numbers [0-9] and hyphens [-]). They may not begin or end with hyphens. | +| Service namespace | A service namespace is the group to which the service belongs e.g. your name or your organisation's name. If in doubt, use the GitHub handle of the user or organisation publishing the services. Namespaces must be lower kebab case (i.e. they may contain the letters [a-z], numbers [0-9], and hyphens [-]). They may not begin or end with hyphens. | +| Service revision | A service revision is a specific instance of a service that can be individually addressed. The revision could correspond to a version of the service, a dynamic development branch for it, or a deliberate duplication or variation of it. | +| Service revision tag | A service revision tag is a short string of text that uniquely identifies a particular revision of a service. The revision tag could be a commit hash (e.g. a3eb45), a semantic version (e.g. 0.12.4), a branch name (e.g. development), a particular environment the service is deployed in (e.g. production), or a combination of these (e.g. 0.12.4-production). Tags may contain lowercase and uppercase letters, numbers, underscores, periods, and hyphens, but can't start with a period or a dash. They can contain a maximum of 128 characters. These requirements are the same as the [Docker tag format](https://docs.docker.com/engine/reference/commandline/tag/). | +| Service revision unique identifier (SRUID) | An SRUID is the combination of a service revision's namespace, name, and revision tag that uniquely identifies it. For example, `octue/my-service:1.3.0` where the namespace is `octue`, the name is `my-service`, and the revision tag is `1.3.0`. | +| Twined ecosystem | The Twined ecosystem is the set of services running Twined as their backend. These services guarantee: defined input/output JSON schemas and validation; an easy and consistent interface for asking them questions and receiving their answers; logs, exceptions, and monitor messages forwarded to you; and high availability (if deployed in the cloud) | diff --git a/mkdocs.yml b/mkdocs.yml index b7eb8e0a8..999c4e134 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,6 +8,7 @@ nav: - license.md - version_history.md - support.md + - glossary.md theme: name: material From 243e84cf342a7a434199c761245477889e259e81 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 3 Oct 2025 15:38:20 -0400 Subject: [PATCH 45/48] DOC: Add outline "creating services" doc skipci --- docs/creating.md | 14 ++++++++++++++ docs/index.md | 2 +- docs/infrastructure.md | 2 +- mkdocs.yml | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 docs/creating.md diff --git a/docs/creating.md b/docs/creating.md new file mode 100644 index 000000000..58546c25b --- /dev/null +++ b/docs/creating.md @@ -0,0 +1,14 @@ +Blah + +# Prerequisites + +Before you begin, ensure you: + + + +- Are familiar with Python and the command line +- Have the following tools installed: + - Python >= 3.10 + - The `octue` python library / CLI (see [installation instructions](installation.md)) + + diff --git a/docs/index.md b/docs/index.md index adeffa7ae..a452731b9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -124,7 +124,7 @@ The following command asks a question to the local example data service. Congratulations on running your first analysis! For additional information, check out the following resources: - [Set up infrastructure for a data service(s) using Terraform](infrastructure.md) -- Create a data service +- [Create a data service](creating.md) - Run a data service locally - See the library and CLI reference - [Get support](support.md) diff --git a/docs/infrastructure.md b/docs/infrastructure.md index 92d69eb11..28b9ca1cb 100644 --- a/docs/infrastructure.md +++ b/docs/infrastructure.md @@ -27,4 +27,4 @@ Follow the instructions [here](https://github.com/octue/terraform-octue-twined-c ## Next steps -Congratulations on setting up a Twined services network! Next up, create your first data service. +Congratulations on setting up a Twined services network! Next up, [create your first data service](creating.md). diff --git a/mkdocs.yml b/mkdocs.yml index 999c4e134..8e62258ac 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,6 +5,7 @@ nav: - installation.md - index.md - Setting up infrastructure: infrastructure.md + - Creating a service: creating.md - license.md - version_history.md - support.md From 6b704d66c0d9126514b80f8fdf4f2c5496276d06 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 3 Oct 2025 15:44:47 -0400 Subject: [PATCH 46/48] DOC: Add overview page and move getting started guide skipci --- docs/getting_started.md | 130 +++++++++++++++++++++++++++++++++++++++ docs/index.md | 133 +++------------------------------------- mkdocs.yml | 2 +- 3 files changed, 140 insertions(+), 125 deletions(-) create mode 100644 docs/getting_started.md diff --git a/docs/getting_started.md b/docs/getting_started.md new file mode 100644 index 000000000..a452731b9 --- /dev/null +++ b/docs/getting_started.md @@ -0,0 +1,130 @@ +# Getting started + +Twined is a python framework for creating and using data services and digital twins. + +This guide walks you through using an example Twined service locally. The process for using a real one (deployed locally +or in the cloud) is almost identical. + +By the end, you will be able to use the Twined CLI to run an analysis on a data service, sending it input data and +receiving output data. + +## Prerequisites + +Before you begin, ensure you: + + + +- Are familiar with Python and/or the command line +- Have the following tools installed: + - Python >= 3.10 + - The `octue` python library / CLI (see [installation instructions](installation.md)) + + + +## Authentication + +No authentication is needed to run the example data service. To authenticate for real data services, see +[authentication instructions](authentication.md). + +## Run your first analysis + +!!! info + + In Twined, sending input data to a service is called "asking a question". The service will run an analysis on the + question and send back any output data - this is called called "receiving an answer". + +### Ask a question + +The following command asks a question to the local example data service. + +=== "CLI" + + ```shell + octue twined question ask example/service:latest --input-values='{"some": "data"}' + ``` + + !!! tip + + To ask a question to a real data service, just specify its ID: + + ```shell + octue twined question ask some-org/a-service:1.2.0 --input-values='{"some": "data"}' + ``` + +=== "Python" + + ```python + from octue.twined.resources import Child + + child = Child( + id="example/service:latest", + backend={ + "name": "GCPPubSubBackend", + "project_id": "example", + }, + ) + + answer, question_uuid = child.ask(input_values={"some": "data"}) + ``` + + !!! info + + A child is a Twined service you ask a question to, in the sense of child and parent nodes in a tree. This only + becomes important when services use other Twined services as part of their analysis, forming a tree of services. + + !!! tip + + To ask a question to a real data service, specify its ID and project ID e.g. `some-org/real-service:1.2.0` + instead of `example/service:latest`. + +### Receive an answer + +=== "CLI" + + The output is automatically written to the command line. It contains log messages followed by the answer as + [JSON](https://en.wikipedia.org/wiki/JSON): + + ```text + [2025-09-23 18:20:13,513 | WARNING | octue.resources.dataset] is empty at instantiation time (path 'cleaned_met_mast_data'). + [2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Starting clean up of files in + [2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Averaging window set to 600s + [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis didn't produce output values. + [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis produced an output manifest. + [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] No output location was set in the service configuration - can't upload output datasets. + [2025-09-23 18:20:13,726 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] Validated outputs against the twine. + {"kind": "result", "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}} + ``` + + !!! tip + + You can pipe the output JSON into other CLI tools or redirect it to a file: + + ```shell + # Format the result using the `jq` command line tool + octue twined question ask example/service:latest --input-values='{"some": "data"}' | jq + + # Store the result in a file + octue twined question ask example/service:latest --input-values='{"some": "data"}' > result.json + ``` + +=== "Python" + + ```python + answer + + >>> { + "kind": "result", + "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, + "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}, + } + ``` + +## Next steps + +Congratulations on running your first analysis! For additional information, check out the following resources: + +- [Set up infrastructure for a data service(s) using Terraform](infrastructure.md) +- [Create a data service](creating.md) +- Run a data service locally +- See the library and CLI reference +- [Get support](support.md) diff --git a/docs/index.md b/docs/index.md index a452731b9..8dfb86f7a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,130 +1,15 @@ -# Getting started +Twined helps scientists and engineers focus on their analyses instead of wrestling with infrastructure. Whether you need +to **run published analyses out of the box**, **share your own models and code as reusable services**, or **deploy +everything securely in your own environment**, Twined streamlines the process so you can spend more time on the science. -Twined is a python framework for creating and using data services and digital twins. +## Using services -This guide walks you through using an example Twined service locally. The process for using a real one (deployed locally -or in the cloud) is almost identical. +Access ready-made scientific analyses instantly, with no coding, cloud setup, or DevOps required. Run them once or thousands of times, and connect them directly into your research workflows and pipelines. -By the end, you will be able to use the Twined CLI to run an analysis on a data service, sending it input data and -receiving output data. +## Creating services -## Prerequisites +Package your analysis code or models as production-ready services, without needing infrastructure expertise. Share them with colleagues, collaborators, or the wider community. -Before you begin, ensure you: +## Managing infrastructure - - -- Are familiar with Python and/or the command line -- Have the following tools installed: - - Python >= 3.10 - - The `octue` python library / CLI (see [installation instructions](installation.md)) - - - -## Authentication - -No authentication is needed to run the example data service. To authenticate for real data services, see -[authentication instructions](authentication.md). - -## Run your first analysis - -!!! info - - In Twined, sending input data to a service is called "asking a question". The service will run an analysis on the - question and send back any output data - this is called called "receiving an answer". - -### Ask a question - -The following command asks a question to the local example data service. - -=== "CLI" - - ```shell - octue twined question ask example/service:latest --input-values='{"some": "data"}' - ``` - - !!! tip - - To ask a question to a real data service, just specify its ID: - - ```shell - octue twined question ask some-org/a-service:1.2.0 --input-values='{"some": "data"}' - ``` - -=== "Python" - - ```python - from octue.twined.resources import Child - - child = Child( - id="example/service:latest", - backend={ - "name": "GCPPubSubBackend", - "project_id": "example", - }, - ) - - answer, question_uuid = child.ask(input_values={"some": "data"}) - ``` - - !!! info - - A child is a Twined service you ask a question to, in the sense of child and parent nodes in a tree. This only - becomes important when services use other Twined services as part of their analysis, forming a tree of services. - - !!! tip - - To ask a question to a real data service, specify its ID and project ID e.g. `some-org/real-service:1.2.0` - instead of `example/service:latest`. - -### Receive an answer - -=== "CLI" - - The output is automatically written to the command line. It contains log messages followed by the answer as - [JSON](https://en.wikipedia.org/wiki/JSON): - - ```text - [2025-09-23 18:20:13,513 | WARNING | octue.resources.dataset] is empty at instantiation time (path 'cleaned_met_mast_data'). - [2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Starting clean up of files in - [2025-09-23 18:20:13,649 | INFO | app] [fb85f10d-4428-4239-9543-f7650104b43c] Averaging window set to 600s - [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis didn't produce output values. - [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] The analysis produced an output manifest. - [2025-09-23 18:20:13,673 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] No output location was set in the service configuration - can't upload output datasets. - [2025-09-23 18:20:13,726 | INFO | octue.twined.resources.analysis] [fb85f10d-4428-4239-9543-f7650104b43c] Validated outputs against the twine. - {"kind": "result", "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}} - ``` - - !!! tip - - You can pipe the output JSON into other CLI tools or redirect it to a file: - - ```shell - # Format the result using the `jq` command line tool - octue twined question ask example/service:latest --input-values='{"some": "data"}' | jq - - # Store the result in a file - octue twined question ask example/service:latest --input-values='{"some": "data"}' > result.json - ``` - -=== "Python" - - ```python - answer - - >>> { - "kind": "result", - "output_values": {"some": "output", "heights": [1, 2, 3, 4, 5]}, - "output_manifest": {"id": "2e1fb3e4-2f86-4eb2-9c2f-5785d36c6df9", "name": null, "datasets": {"cleaned_met_mast_data": "/var/folders/9p/25hhsy8j4wv66ck3yylyz97c0000gn/T/tmps_qcb4yw"}}, - } - ``` - -## Next steps - -Congratulations on running your first analysis! For additional information, check out the following resources: - -- [Set up infrastructure for a data service(s) using Terraform](infrastructure.md) -- [Create a data service](creating.md) -- Run a data service locally -- See the library and CLI reference -- [Get support](support.md) +Deploy Twined services in your private cloud with just two pre-configured Terraform modules. Control costs, manage access, and integrate with your existing scientific data and digital tools. diff --git a/mkdocs.yml b/mkdocs.yml index 8e62258ac..fc466f88b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -2,8 +2,8 @@ site_name: Octue Twined site_url: https://twined.octue.com nav: + - Overview: index.md - installation.md - - index.md - Setting up infrastructure: infrastructure.md - Creating a service: creating.md - license.md From b86c7f5050f9a20b691477f3db0c9be0c509f8fa Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 3 Oct 2025 15:55:09 -0400 Subject: [PATCH 47/48] DOC: Add more info to overview --- docs/index.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 8dfb86f7a..afe0d3e08 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,14 +2,33 @@ Twined helps scientists and engineers focus on their analyses instead of wrestli to **run published analyses out of the box**, **share your own models and code as reusable services**, or **deploy everything securely in your own environment**, Twined streamlines the process so you can spend more time on the science. -## Using services +## Where to start + +### Using services Access ready-made scientific analyses instantly, with no coding, cloud setup, or DevOps required. Run them once or thousands of times, and connect them directly into your research workflows and pipelines. -## Creating services +### Creating services Package your analysis code or models as production-ready services, without needing infrastructure expertise. Share them with colleagues, collaborators, or the wider community. -## Managing infrastructure +### Managing infrastructure Deploy Twined services in your private cloud with just two pre-configured Terraform modules. Control costs, manage access, and integrate with your existing scientific data and digital tools. + +## Stats + +- 250,000+ analyses run +- 100s of services built +- 6 years in development +- 12 countries - users across the world + +## Twined in the wild + +- WindPioneers - [WindQuest](https://wind-pioneers.com/what-we-do/windquest-smarter-tools/): Twined powers the simulation infrastructure behind the WindQuest application, supporting design of hundreds of wind farms worldwide with over 250,000 Energy Yield Assessment (EYA) scenarios run to date. +- OST Zurich - [Aerosense](https://rtdt.ai/) (now part of RTDT Labs): The Twined framework was used to underpin digital twin and data infrastructure, enabling a novel turbine blade sensing technology to reach commercial readiness, now spun out and deployed internationally. + +## Twined by example + +- [Elevation service](https://github.com/octue/windeurope72hours-elevations-api) +- [Example service](https://github.com/octue/example-service-kueue) From ab93f24ddd974465506d497cfda506e2adafeede Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 3 Oct 2025 16:04:42 -0400 Subject: [PATCH 48/48] DOC: Split docs into three main sections skipci --- .../getting_started.md} | 2 +- .../getting_started.md} | 2 +- docs/{ => using_services}/getting_started.md | 8 ++++---- mkdocs.yml | 10 +++++++--- 4 files changed, 13 insertions(+), 9 deletions(-) rename docs/{creating.md => creating_services/getting_started.md} (90%) rename docs/{infrastructure.md => managing_infrastructure/getting_started.md} (94%) rename docs/{ => using_services}/getting_started.md (96%) diff --git a/docs/creating.md b/docs/creating_services/getting_started.md similarity index 90% rename from docs/creating.md rename to docs/creating_services/getting_started.md index 58546c25b..e40271d73 100644 --- a/docs/creating.md +++ b/docs/creating_services/getting_started.md @@ -9,6 +9,6 @@ Before you begin, ensure you: - Are familiar with Python and the command line - Have the following tools installed: - Python >= 3.10 - - The `octue` python library / CLI (see [installation instructions](installation.md)) + - The `octue` python library / CLI (see [installation instructions](../installation.md)) diff --git a/docs/infrastructure.md b/docs/managing_infrastructure/getting_started.md similarity index 94% rename from docs/infrastructure.md rename to docs/managing_infrastructure/getting_started.md index 28b9ca1cb..524500366 100644 --- a/docs/infrastructure.md +++ b/docs/managing_infrastructure/getting_started.md @@ -27,4 +27,4 @@ Follow the instructions [here](https://github.com/octue/terraform-octue-twined-c ## Next steps -Congratulations on setting up a Twined services network! Next up, [create your first data service](creating.md). +Congratulations on setting up a Twined services network! Next up, [create your first data service](../creating_services/getting_started.md). diff --git a/docs/getting_started.md b/docs/using_services/getting_started.md similarity index 96% rename from docs/getting_started.md rename to docs/using_services/getting_started.md index a452731b9..ad1323eb2 100644 --- a/docs/getting_started.md +++ b/docs/using_services/getting_started.md @@ -17,7 +17,7 @@ Before you begin, ensure you: - Are familiar with Python and/or the command line - Have the following tools installed: - Python >= 3.10 - - The `octue` python library / CLI (see [installation instructions](installation.md)) + - The `octue` python library / CLI (see [installation instructions](../installation.md)) @@ -123,8 +123,8 @@ The following command asks a question to the local example data service. Congratulations on running your first analysis! For additional information, check out the following resources: -- [Set up infrastructure for a data service(s) using Terraform](infrastructure.md) -- [Create a data service](creating.md) +- [Set up infrastructure for a data service(s) using Terraform](../managing_infrastructure/getting_started.md) +- [Create a data service](../creating_services/getting_started.md) - Run a data service locally - See the library and CLI reference -- [Get support](support.md) +- [Get support](../support.md) diff --git a/mkdocs.yml b/mkdocs.yml index fc466f88b..dbad8fc7f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,12 +4,16 @@ site_url: https://twined.octue.com nav: - Overview: index.md - installation.md - - Setting up infrastructure: infrastructure.md - - Creating a service: creating.md + - Using services: + - using_services/getting_started.md + - Creating services: + - creating_services/getting_started.md + - Managing infrastructure: + - managing_infrastructure/getting_started.md + - glossary.md - license.md - version_history.md - support.md - - glossary.md theme: name: material