diff --git a/.env b/.env index d58fe532a..0ea91c30b 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ -FAABRIC_VERSION=0.1.1 -FAABRIC_CLI_IMAGE=faasm/faabric:0.1.1 +FAABRIC_VERSION=0.1.2 +FAABRIC_CLI_IMAGE=faasm/faabric:0.1.2 COMPOSE_PROJECT_NAME=faabric-dev diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 58b8536ba..dc7c97fe7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm/faabric:0.1.1 + image: faasm/faabric:0.1.2 defaults: run: working-directory: /code/faabric @@ -45,7 +45,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm/faabric:0.1.1 + image: faasm/faabric:0.1.2 defaults: run: working-directory: /code/faabric @@ -96,7 +96,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm/faabric:0.1.1 + image: faasm/faabric:0.1.2 defaults: run: working-directory: /code/faabric diff --git a/README.md b/README.md index c47fed256..9ecd28044 100644 --- a/README.md +++ b/README.md @@ -1,122 +1,17 @@ -# Faabric [![Faabric tests](https://github.com/faasm/faabric/workflows/Tests/badge.svg?branch=master)](https://github.com/faasm/faabric/actions) [![License](https://img.shields.io/github/license/faasm/faabric.svg)](https://github.com/faasm/faabric/blob/master/LICENSE.md) +# Faabric [![Faabric tests](https://github.com/faasm/faabric/workflows/Tests/badge.svg?branch=master)](https://github.com/faasm/faabric/actions) [![License](https://img.shields.io/github/license/faasm/faabric.svg)](https://github.com/faasm/faabric/blob/master/LICENSE.md) -Faabric is a messaging and state layer for serverless applications. +Faabric is a library that provides scheduling, messaging and state for +distributed serverless runtimes. -## Building and Development +It began life as part of [Faasm](https://github.com/faasm/faasm), but is now a +stand-alone library that can be used as a base for other distributed runtimes. -You can build Faabric natively or using the containerised environment. +## Usage -### Containerised (recommended) +You can see a simple example in the [examples](examples) directory. -Run the following: +## Development -```bash -./bin/cli.sh -``` +See the [development docs](docs/development.md) for more information on +developing Faabric. -This container has everything you need, and the script will also mount your -current checkout of the code. This means you can just run the following: - -```bash -# Set up the build -inv dev.cmake - -# Build the tests -inv dev.cc faabric_tests - -# Run the tests -faabric_tests -``` - -To stop the `faabric`-related containers run: -```bash -docker-compose down -``` - -### Native - -Most external dependencies are installed through CMake in -[cmake/ExternalProjects.cmake](./cmake/ExternalProjects.cmake). -The remaining installed packages can be inspected in the [`faabric-base`]( -./docker/faabric-base.dockerfile) dockerfile. - -Use of Clang and Ninja is recommended. From the root of this project you can -run: - -```bash -mkdir build -cd build - -cmake \ - -GNinja \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_BUILD_TYPE=Release \ - .. - -ninja -``` - -You can also run the CLI with: - -```bash -# Set up env -export FAABRIC_BUILD_DIR= -source bin/workon.sh - -# Install requirements -pip install -r requirements.txt - -# Build Faabric -inv dev.cc faabric -``` - -## Releasing - -Create a new branch, then find and replace the current version with the relevant -bumped version. Currently it's held in `VERSION`, `.env`, -`mpi-native/mpi-native.env` and the Github Actions configuration. - -Once done, commit and push, then run: - -```bash -source bin/workon.sh -inv git.tag -``` - -This will trigger the release build in Github Actions which will build all the -containers. Once that's complete, create a PR from your branch and make sure the -tests pass as normal. - -If you want to overwrite a tag, you can run: - -```bash -inv git.tag --force -``` - -### Building images manually - -Containers are built with Github Actions, so you should only need to build them -yourself when diagnosing issues. - -To build the main container, run: - -```bash -source bin/workon.sh - -# Build -inv container.build - -# Push -inv container.push - -# Build and push -inv container.build --push -``` - -## Additional documentation - -More detail on some key features and implementations can be found below: - -- [Native MPI builds](docs/mpi_native.md) - run native applications against -Faabric's MPI library. diff --git a/VERSION b/VERSION index 17e51c385..d917d3e26 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.1 +0.1.2 diff --git a/docs/development.md b/docs/development.md index 6e0aceef2..1a2012c0e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,5 +1,73 @@ # Developing Faabric +## Development environment + +### Containerised (recommended) + +Run the following: + +```bash +./bin/cli.sh +``` + +This container has everything you need, and the script will also mount your +current checkout of the code. This means you can just run the following: + +```bash +# Set up the build +inv dev.cmake + +# Build the tests +inv dev.cc faabric_tests + +# Run the tests +faabric_tests +``` + +To stop the `faabric`-related containers run: + +```bash +docker-compose down +``` + +### Native + +Most external dependencies are installed through CMake in +[cmake/ExternalProjects.cmake](./cmake/ExternalProjects.cmake). +The remaining installed packages can be inspected in the [`faabric-base`]( +./docker/faabric-base.dockerfile) dockerfile. + +Use of Clang and Ninja is recommended. From the root of this project you can +run: + +```bash +mkdir build +cd build + +cmake \ + -GNinja \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_BUILD_TYPE=Release \ + .. + +ninja +``` + +You can also run the CLI with: + +```bash +# Set up env +export FAABRIC_BUILD_DIR= +source bin/workon.sh + +# Install requirements +pip install -r requirements.txt + +# Build Faabric +inv dev.cc faabric +``` + ## Testing We have some standard tests using [Catch2](https://github.com/catchorg/Catch2) @@ -52,3 +120,47 @@ docker-compose rm # Run once through ./dist-test/run.sh ``` + +## Releasing + +Create a new branch, then find and replace the current version with the relevant +bumped version. Currently it's held in `VERSION`, `.env`, +`mpi-native/mpi-native.env` and the Github Actions configuration. + +Once done, commit and push, then run: + +```bash +source bin/workon.sh +inv git.tag +``` + +This will trigger the release build in Github Actions which will build all the +containers. Once that's complete, create a PR from your branch and make sure the +tests pass as normal. + +If you want to overwrite a tag, you can run: + +```bash +inv git.tag --force +``` + +### Building images manually + +Containers are built with Github Actions, so you should only need to build them +yourself when diagnosing issues. + +To build the main container, run: + +```bash +source bin/workon.sh + +# Build +inv container.build + +# Push +inv container.push + +# Build and push +inv container.build --push +``` + diff --git a/tasks/container.py b/tasks/container.py index bb340b503..d7a001386 100644 --- a/tasks/container.py +++ b/tasks/container.py @@ -50,7 +50,7 @@ def _do_push(name): run(cmd, shell=True, check=True) -@task +@task(default=True) def build(ctx, nocache=False, push=False): """ Build current version of faabric container diff --git a/tests/dist/scheduler/test_threads.cpp b/tests/dist/scheduler/test_threads.cpp index b91ff3bf6..23eb52b79 100644 --- a/tests/dist/scheduler/test_threads.cpp +++ b/tests/dist/scheduler/test_threads.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include #include diff --git a/tests/test/endpoint/test_endpoint_api.cpp b/tests/test/endpoint/test_endpoint_api.cpp index 10f6a40cb..9d3f7c024 100644 --- a/tests/test/endpoint/test_endpoint_api.cpp +++ b/tests/test/endpoint/test_endpoint_api.cpp @@ -45,6 +45,11 @@ class EndpointApiTestExecutor final : public Executor returnVal = 1; msg.set_outputdata(fmt::format( "Endpoint API returning {} for {}", returnVal, msg.id())); + } else if (msg.isasync()) { + returnVal = 0; + SLEEP_MS(3000); + msg.set_outputdata( + fmt::format("Finished async message {}", msg.id())); } else { throw std::runtime_error("Endpoint API error"); } @@ -138,4 +143,59 @@ TEST_CASE_METHOD(EndpointApiTestFixture, serverThread.join(); } } + +TEST_CASE_METHOD(EndpointApiTestFixture, + "Test status requests to endpoint", + "[endpoint]") +{ + port++; + faabric::endpoint::FaabricEndpoint endpoint(port, 2); + + std::thread serverThread([&endpoint]() { endpoint.start(false); }); + + // Wait for the server to start + SLEEP_MS(2000); + + // Make the initial invocation + faabric::Message msg = faabric::util::messageFactory("foo", "blah"); + msg.set_isasync(true); + std::string body = faabric::util::messageToJson(msg); + + std::pair result = + submitGetRequestToUrl(LOCALHOST, port, body); + + REQUIRE(result.first == 200); + REQUIRE(result.second == std::to_string(msg.id())); + + // Make a status request, should still be running + faabric::Message statusMsg; + statusMsg.set_user("foo"); + statusMsg.set_function("blah"); + statusMsg.set_id(msg.id()); + statusMsg.set_isstatusrequest(true); + + std::string statusBody = faabric::util::messageToJson(statusMsg); + + std::pair statusResult = + submitGetRequestToUrl(LOCALHOST, port, statusBody); + + REQUIRE(statusResult.first == 200); + REQUIRE(statusResult.second == "RUNNING"); + + // Wait for the function to finish + SLEEP_MS(4000); + + std::pair statusResultAfter = + submitGetRequestToUrl(LOCALHOST, port, statusBody); + + REQUIRE(statusResultAfter.first == 200); + REQUIRE(statusResultAfter.second == + fmt::format("SUCCESS: Finished async message {}", msg.id())); + + endpoint.stop(); + + if (serverThread.joinable()) { + serverThread.join(); + } +} } diff --git a/tests/utils/http_utils.cpp b/tests/utils/http_utils.cpp index b592e57b3..7d52b3e82 100644 --- a/tests/utils/http_utils.cpp +++ b/tests/utils/http_utils.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -24,7 +23,6 @@ namespace tests { std::pair submitGetRequestToUrl(const std::string& host, int port, const std::string& body) - { Http::Client client; client.init();