diff --git a/.github/ci-config.yml b/.github/ci-config.yml new file mode 100644 index 0000000..26971bc --- /dev/null +++ b/.github/ci-config.yml @@ -0,0 +1,9 @@ +dependencies: | + ecmwf/ecbuild + MathisRosenhauer/libaec@master + ecmwf/eccodes + ecmwf/eckit + ecmwf/metkit + ecmwf/fdb +dependency_branch: develop +parallelism_factor: 8 diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 0000000..8135394 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,16 @@ +build: + modules: + - ninja + - aec + dependencies: + - ecmwf/ecbuild@develop + - ecmwf/eccodes@develop + - ecmwf/eckit@develop + - ecmwf/metkit@develop + - ecmwf/fdb@develop + dependency_cmake_options: + - "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'" + parallel: 64 + env: + - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples + - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..ef99a21 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,11 @@ +name: cd + +on: + push: + tags: + - '**' + +jobs: + deploy: + uses: ecmwf-actions/reusable-workflows/.github/workflows/create-package.yml@v2 + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..14cb745 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ +name: ci + +on: + # Trigger the workflow on push to master or develop, except tag creation + push: + branches: + - 'master' + - 'develop' + tags-ignore: + - '**' + + # Trigger the workflow on pull request + pull_request: ~ + + # Trigger the workflow manually + workflow_dispatch: ~ + + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] + +jobs: + # Run CI including downstream packages on self-hosted runners + downstream-ci: + name: downstream-ci + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + with: + gribjump: ecmwf/gribjump@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true + secrets: inherit + + # Run CI of private downstream packages on self-hosted runners + private-downstream-ci: + name: private-downstream-ci + needs: [downstream-ci] + if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Dispatch private downstream CI + uses: ecmwf-actions/dispatch-private-downstream-ci@v1 + with: + token: ${{ secrets.GH_REPO_READ_TOKEN }} + owner: ecmwf-actions + repository: private-downstream-ci + event_type: downstream-ci + payload: '{"gribjump": "ecmwf/gribjump@${{ github.event.pull_request.head.sha || github.sha }}"}' + + # Build downstream packages on HPC + downstream-ci-hpc: + name: downstream-ci-hpc + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + with: + gribjump: ecmwf/gribjump@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit + + # Run CI of private downstream packages on HPC + private-downstream-ci-hpc: + name: private-downstream-ci-hpc + needs: [downstream-ci-hpc] + if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Dispatch private downstream CI + uses: ecmwf-actions/dispatch-private-downstream-ci@v1 + with: + token: ${{ secrets.GH_REPO_READ_TOKEN }} + owner: ecmwf-actions + repository: private-downstream-ci + event_type: downstream-ci-hpc + payload: '{"gribjump": "ecmwf/gribjump@${{ github.event.pull_request.head.sha || github.sha }}"}' diff --git a/.github/workflows/label-public-pr.yml b/.github/workflows/label-public-pr.yml new file mode 100644 index 0000000..59b2bfa --- /dev/null +++ b/.github/workflows/label-public-pr.yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: label-public-pr + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index c7b1555..3da6347 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,10 @@ if (HAVE_GRIBJUMP_LOCAL_EXTRACT) message(FATAL_ERROR "AEC version is too old. Minimum supported version is 1.1.1. Found version: ${AEC_VERSION}") endif() + # Optional dependency: dhskit + ecbuild_find_package( NAME dhskit VERSION 0.8.6 ) + set(GRIBJUMP_HAVE_DHSKIT ${dhskit_FOUND}) + endif() ######################################################################################################################## @@ -66,13 +70,15 @@ if (HAVE_GRIBJUMP_LOCAL_EXTRACT) add_subdirectory( tests ) endif() +######################################################################################################################## +# PLUGIN # Write and install the manifest file for the plugin set( gribjump_plugin_file share/plugins/gribjump-plugin.yml ) file( WRITE ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} "plugin:\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} " name: gribjump-plugin\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} " namespace: int.ecmwf\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} " url: http://www.ecmwf.int\n" ) -file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} " version: ${${PROJECT_NAME}_VERSION}\n" ) # TODO set version +file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} " version: ${${PROJECT_NAME}_VERSION}\n" ) file( APPEND ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} " library: gribjump\n" ) install( FILES ${CMAKE_BINARY_DIR}/${gribjump_plugin_file} DESTINATION share/plugins ) diff --git a/VERSION b/VERSION index 8f0916f..4b9fcbe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0 +0.5.1 diff --git a/src/gribjump/CMakeLists.txt b/src/gribjump/CMakeLists.txt index fc4ec3a..35d6049 100644 --- a/src/gribjump/CMakeLists.txt +++ b/src/gribjump/CMakeLists.txt @@ -125,6 +125,10 @@ if( HAVE_GRIBJUMP_LOCAL_EXTRACT ) endif() +if (GRIBJUMP_HAVE_DHSKIT) +list( APPEND SERVER_LIBS dhskit ) +endif() + ecbuild_add_library( TARGET gribjump diff --git a/src/gribjump/gribjump_config.h.in b/src/gribjump/gribjump_config.h.in index df7324b..4e933ef 100644 --- a/src/gribjump/gribjump_config.h.in +++ b/src/gribjump/gribjump_config.h.in @@ -7,5 +7,6 @@ // features #define GRIBJUMP_HAVE_FDB @GRIBJUMP_HAVE_FDB@ +#cmakedefine GRIBJUMP_HAVE_DHSKIT #endif // gribjump_gribjump_config_h diff --git a/src/gribjump/remote/WorkQueue.cc b/src/gribjump/remote/WorkQueue.cc index abb6d20..5a5dd36 100644 --- a/src/gribjump/remote/WorkQueue.cc +++ b/src/gribjump/remote/WorkQueue.cc @@ -33,8 +33,8 @@ WorkQueue::~WorkQueue() { } -WorkQueue::WorkQueue() : queue_(eckit::Resource("$GRIBJUMP_QUEUESIZE", 1024)) { - int nthreads = eckit::Resource("$GRIBJUMP_THREADS", LibGribJump::instance().config().getInt("threads", 1)); +WorkQueue::WorkQueue() : queue_(eckit::Resource("$GRIBJUMP_QUEUESIZE;gribjumpQueueSize", 1024)) { + int nthreads = eckit::Resource("$GRIBJUMP_THREADS;gribjumpThreads", LibGribJump::instance().config().getInt("threads", 1)); eckit::Log::info() << "Starting " << eckit::Plural(nthreads, "thread") << std::endl; for (int i = 0; i < nthreads; ++i) { workers_.emplace_back([this] { diff --git a/src/tools/gribjump-server.cc b/src/tools/gribjump-server.cc index 80b5102..a57c311 100644 --- a/src/tools/gribjump-server.cc +++ b/src/tools/gribjump-server.cc @@ -8,25 +8,38 @@ * does it submit to any jurisdiction. */ -#include "gribjump/remote/GribJumpServer.h" -#include "eckit/runtime/Application.h" #include #include #include "eckit/config/Resource.h" #include "eckit/net/Port.h" +#include "eckit/runtime/Application.h" -namespace gribjump { +#include "gribjump/gribjump_config.h" +#include "gribjump/remote/GribJumpServer.h" +#ifdef GRIBJUMP_HAVE_DHSKIT +#include "dhskit/runtime/DHSApplication.h" +#endif + +namespace { +#ifdef GRIBJUMP_HAVE_DHSKIT +using BaseApp = dhskit::DHSApplication; +#else +using BaseApp = eckit::Application; +#endif +} // namespace + + +namespace gribjump { -class GribJumpServerApp : public eckit::Application, public GribJumpServer { +class GribJumpServerApp : public BaseApp, public GribJumpServer { public: GribJumpServerApp(int argc, char** argv) : - Application(argc, argv), - GribJumpServer(eckit::net::Port( - "gribJumpServer", - eckit::Resource("$GRIBJUMP_SERVER_PORT", LibGribJump::instance().config().getInt("server.port", 9777) + BaseApp(argc, argv), + GribJumpServer(eckit::net::Port( // gribjumpServerPort + "gribjumpServer", eckit::Resource("$GRIBJUMP_SERVER_PORT", LibGribJump::instance().config().getInt("server.port", 9777) ))) {} @@ -48,7 +61,6 @@ class GribJumpServerApp : public eckit::Application, public GribJumpServer { int main(int argc, char** argv) { - eckit::Log::info() << "Starting gribjump server" << std::endl; gribjump::GribJumpServerApp app(argc, argv); app.start(); return 0; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 55e3b3d..7fd8cfa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -95,7 +95,7 @@ ecbuild_add_test( SOURCES "test_engine.cc" INCLUDES "${ECKIT_INCLUDE_DIRS}" ENVIRONMENT "${gribjump_env}" - TEST_DEPENDS gribjump_test_O1280_data_files + TEST_DEPENDS gribjump_test_data_files NO_AS_NEEDED LIBS gribjump )