Merge branch 'develop' into feature/backend-s3

.clang-tidy

@@ -0,0 +1,13 @@
+---
+UseColor: On
+Checks: >-
+  -*,
+  readability-delete-null-pointer,
+  bugprone-parent-virtual-call,
+
+WarningsAsErrors:             false
+HeaderFileExtensions:         ['', 'h','hh','hpp','hxx']
+ImplementationFileExtensions: ['c','cc','cpp','cxx']
+HeaderFilterRegex:            '.*'
+FormatStyle:                  none
+InheritParentConfig:          false

.github/workflows/ci.yml

@@ -74,3 +74,21 @@ jobs:
           repository: private-downstream-ci
           event_type: downstream-ci-hpc
           payload: '{"fdb": "ecmwf/fdb@${{ github.event.pull_request.head.sha || github.sha }}"}'
+
+  notify:
+    runs-on: ubuntu-latest
+    needs:
+      - downstream-ci
+      - private-downstream-ci
+      - downstream-ci-hpc
+      - private-downstream-ci-hpc
+    if: ${{ always() && !github.event.pull_request.head.repo.fork && (github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
+    steps:
+      - name: Trigger Teams notification
+        if: failure() && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop')
+        uses: ecmwf-actions/notify-teams@v1
+        with:
+          incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}
+          needs_context: ${{ toJSON(needs) }}
+          notify_on: |
+            failure

.github/workflows/notify-issues.yml

@@ -0,0 +1,14 @@
+name: Notify Issues
+
+on:
+  issues:
+    types: [opened, reopened]
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Issues
+        uses: ecmwf-actions/notify-teams-issue@v1
+        with:
+          incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}

.github/workflows/notify-pull-request.yml

@@ -0,0 +1,14 @@
+name: Notify Pull Request
+
+on:
+  pull_request_target:
+    types: [opened, reopened]
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Pull Request
+        uses: ecmwf-actions/notify-teams-pr@v1
+        with:
+          incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}

.github/workflows/sync.yml

@@ -0,0 +1,26 @@
+name: sync
+
+# Controls when the workflow will run
+on:
+
+  # Trigger the workflow on all pushes
+  push:
+    branches:
+    - '**'
+    tags:
+    - '**'
+
+  # Trigger the workflow when a branch or tag is deleted
+  delete: ~
+
+jobs:
+
+  # Calls a reusable CI workflow to sync the current with a remote repository.
+  #   It will correctly handle addition of any new and removal of existing Git objects.
+  sync:
+    name: sync
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/sync.yml@v2
+    secrets:
+      target_repository: mars/fdb5
+      target_username: ClonedDuck
+      target_token: ${{ secrets.BITBUCKET_PAT }}

.gitignore

@@ -1 +1,2 @@
 .vscode
+build

CMakeLists.txt

@@ -20,7 +20,7 @@ endif()
 
 set( PERSISTENT_NAMESPACE "eckit" CACHE INTERNAL "" ) # needed for generating .b files for persistent support
 
-ecbuild_find_package( NAME eckit  VERSION 1.24.4 REQUIRED )
+ecbuild_find_package( NAME eckit  VERSION 1.28.3 REQUIRED )
 
 ### GRIB support
 
@@ -31,7 +31,7 @@ ecbuild_add_option( FEATURE GRIB
                     CONDITION eccodes_FOUND
                     DESCRIPTION "Support for GRIB via eccodes")
 
-ecbuild_find_package( NAME metkit VERSION 1.5 REQUIRED )
+ecbuild_find_package( NAME metkit VERSION 1.11.22 REQUIRED )
 
 ### FDB backend in persistent memory, i.e. pmem (NVRAM)
 
@@ -76,6 +76,45 @@ ecbuild_add_option( FEATURE FDB_REMOTE
                     DEFAULT ON
                     DESCRIPTION "Support for FDB remote access" )
 
+find_package(UUID QUIET)
+
+find_package(DAOS QUIET)
+
+set(_default_dummy_daos ON)
+if(DAOS_FOUND)
+    set(_default_dummy_daos OFF)
+endif()
+
+ecbuild_add_option( FEATURE DUMMY_DAOS
+                    DEFAULT ${_default_dummy_daos}
+                    CONDITION UUID_FOUND
+                    DESCRIPTION "Use dummy DAOS library emulating DAOS with a file system" )
+
+if(HAVE_DUMMY_DAOS)
+    set(DAOS_LIBRARIES dummy_daos)
+    set(DAOS_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/src/dummy_daos)
+    set(DAOS_FOUND TRUE)
+    set(DAOS_TESTS_LIBRARIES dummy_daos_tests)
+    set(DAOS_TESTS_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/src/dummy_daos)
+    set(DAOS_TESTS_FOUND TRUE)
+endif()
+
+ecbuild_add_option( FEATURE DAOSFDB
+                    CONDITION DAOS_FOUND AND UUID_FOUND 
+                    DEFAULT ON
+                    DESCRIPTION "DAOS support for FDB Store" )
+
+ecbuild_add_option( FEATURE DAOS_ADMIN
+                    DEFAULT ${_default_dummy_daos}
+                    CONDITION HAVE_DAOSFDB AND DAOS_TESTS_FOUND
+                    DESCRIPTION "Add features for DAOS pool management. Removes need to manually create a pool for DAOS unit tests" )
+
+# DAOS_TESTS is a daos admin library, usually not present
+if(NOT HAVE_DAOS_ADMIN)
+    set( DAOS_TESTS_LIBRARIES "" )
+    set( DAOS_TESTS_INCLUDE_DIRS "" )
+endif()
+
 ecbuild_add_option( FEATURE EXPERIMENTAL
                     DEFAULT OFF
                     DESCRIPTION "Experimental features" )

VERSION

@@ -1 +1 @@
-5.11.29
+5.13.4

cmake/FindDAOS.cmake

@@ -0,0 +1,164 @@
+# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+
+include(FindPackageHandleStandardArgs)
+
+# daos
+
+find_path(DAOS_INCLUDE_DIR
+    NAMES daos.h
+    HINTS
+        ${DAOS_ROOT}
+        ${DAOS_DIR}
+        ${DAOS_PATH}
+        ENV DAOS_ROOT
+        ENV DAOS_DIR
+        ENV DAOS_PATH
+    PATH_SUFFIXES include
+)
+
+find_library(DAOS_LIBRARY
+    NAMES daos
+    HINTS
+        ${DAOS_ROOT}
+        ${DAOS_DIR}
+        ${DAOS_PATH}
+        ENV DAOS_ROOT
+        ENV DAOS_DIR
+        ENV DAOS_PATH
+    PATH_SUFFIXES lib lib64
+)
+
+# daos_common
+
+find_library(DAOS_COMMON_LIBRARY
+    NAMES daos_common
+    HINTS
+        ${DAOS_ROOT}
+        ${DAOS_DIR}
+        ${DAOS_PATH}
+        ENV DAOS_ROOT
+        ENV DAOS_DIR
+        ENV DAOS_PATH
+    PATH_SUFFIXES lib lib64
+)
+
+# gurt
+
+find_library(GURT_LIBRARY
+    NAMES gurt
+    HINTS
+        ${DAOS_ROOT}
+        ${DAOS_DIR}
+        ${DAOS_PATH}
+        ENV DAOS_ROOT
+        ENV DAOS_DIR
+        ENV DAOS_PATH
+    PATH_SUFFIXES lib lib64
+)
+
+# daos tests
+
+# if not using dummy DAOS, then the daos_tests lib should be installed
+# from daos RPMs into the corresponding system directories. However its
+# headers (daos/tests_lib.h) are not provided in any of the RPMs and 
+# need to be copied from source. tests_lib.h could be copied into standard 
+# system directories together with other DAOS headers, but the tests_lib.h
+# file includes many other DAOS headers not provided by RPMs and other 
+# external library headers, e.g. boost, and are required when compiling a 
+# program which includes tests_lib.h (e.g. fdb5's DAOS backend with 
+# DAOS_ADMIN enabled for unit testing). To avoid having to install all 
+# these headers, tests_lib.h can be copied in a user directory and modified
+# to only declare the necessary functions (pool create and destroy) and 
+# remove most of the includes. All this explains why a DAOS_TESTS_INCLUDE_ROOT
+# environment variable is used here to find tests_lib.h rather than DAOS_ROOT.
+
+find_path(DAOS_TESTS_INCLUDE_DIR
+    NAMES daos/tests_lib.h
+    HINTS
+        ${DAOS_TESTS_INCLUDE_ROOT}
+        ${DAOS_TESTS_DIR}
+        ${DAOS_TESTS_PATH}
+        ENV DAOS_TESTS_INCLUDE_ROOT
+        ENV DAOS_TESTS_DIR
+        ENV DAOS_TESTS_PATH
+    PATH_SUFFIXES include
+)
+
+find_library(DAOS_TESTS_LIBRARY
+    NAMES daos_tests
+    HINTS
+        ${DAOS_ROOT}
+        ${DAOS_DIR}
+        ${DAOS_PATH}
+        ENV DAOS_ROOT
+        ENV DAOS_DIR
+        ENV DAOS_PATH
+    PATH_SUFFIXES lib lib64
+)
+
+find_package_handle_standard_args(
+    DAOS
+    DEFAULT_MSG
+    DAOS_LIBRARY
+    DAOS_INCLUDE_DIR
+    DAOS_COMMON_LIBRARY
+    GURT_LIBRARY )
+
+mark_as_advanced(DAOS_INCLUDE_DIR DAOS_LIBRARY DAOS_COMMON_LIBRARY GURT_LIBRARY)
+
+if(DAOS_FOUND)
+    add_library(daos UNKNOWN IMPORTED GLOBAL)
+    set_target_properties(daos PROPERTIES
+        IMPORTED_LOCATION ${DAOS_LIBRARY}
+        INTERFACE_INCLUDE_DIRECTORIES ${DAOS_INCLUDE_DIR}
+    )
+    add_library(daos_common UNKNOWN IMPORTED GLOBAL)
+    set_target_properties(daos_common PROPERTIES
+        IMPORTED_LOCATION ${DAOS_COMMON_LIBRARY}
+        INTERFACE_INCLUDE_DIRECTORIES ${DAOS_INCLUDE_DIR}
+    )
+    add_library(gurt UNKNOWN IMPORTED GLOBAL)
+    set_target_properties(gurt PROPERTIES
+        IMPORTED_LOCATION ${GURT_LIBRARY}
+        INTERFACE_INCLUDE_DIRECTORIES ${DAOS_INCLUDE_DIR}
+    )
+    set(DAOS_INCLUDE_DIRS ${DAOS_INCLUDE_DIR})
+    list(APPEND DAOS_LIBRARIES daos daos_common gurt)
+endif()
+
+find_package_handle_standard_args(
+    DAOS_TESTS
+    DEFAULT_MSG
+    DAOS_TESTS_LIBRARY
+    DAOS_TESTS_INCLUDE_DIR )
+
+mark_as_advanced(DAOS_TESTS_INCLUDE_DIR DAOS_TESTS_LIBRARY)
+
+if(DAOS_TESTS_FOUND)
+    add_library(daos_tests UNKNOWN IMPORTED GLOBAL)
+    set_target_properties(daos_tests PROPERTIES
+        IMPORTED_LOCATION ${DAOS_TESTS_LIBRARY}
+        INTERFACE_INCLUDE_DIRECTORIES "${DAOS_TESTS_INCLUDE_DIR};${DAOS_INCLUDE_DIR}"
+    )
+    list(APPEND DAOS_TESTS_INCLUDE_DIRS 
+        ${DAOS_TESTS_INCLUDE_DIR}
+        ${DAOS_INCLUDE_DIR}
+    )
+    set(DAOS_TESTS_LIBRARIES daos_tests)
+endif()

cmake/FindUUID.cmake

@@ -0,0 +1,60 @@
+# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+
+include(FindPackageHandleStandardArgs)
+
+# uuid
+
+find_path(UUID_INCLUDE_DIR
+    NAMES uuid/uuid.h
+    HINTS
+        ${UUID_ROOT}
+        ${UUID_DIR}
+        ${UUID_PATH}
+        ENV UUID_ROOT
+        ENV UUID_DIR
+        ENV UUID_PATH
+    PATH_SUFFIXES include include/uuid
+    NO_DEFAULT_PATH
+)
+
+find_library(UUID_LIBRARY
+    NAMES uuid
+    HINTS
+        ${UUID_ROOT}
+        ${UUID_DIR}
+        ${UUID_PATH}
+        ENV UUID_ROOT
+        ENV UUID_DIR
+        ENV UUID_PATH
+    PATH_SUFFIXES lib lib64
+)
+
+find_package_handle_standard_args(UUID DEFAULT_MSG UUID_LIBRARY UUID_INCLUDE_DIR)
+
+mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY)
+
+if(UUID_FOUND)
+    add_library(uuid UNKNOWN IMPORTED GLOBAL)
+    set_target_properties(uuid PROPERTIES
+        IMPORTED_LOCATION ${UUID_LIBRARY}
+        INTERFACE_INCLUDE_DIRECTORIES ${UUID_INCLUDE_DIR}
+    )
+    set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR})
+    set(UUID_LIBRARIES uuid)
+endif()