From ed369436ef7cf3a16d4405a469edf82d5615466e Mon Sep 17 00:00:00 2001 From: Bruno Vavala Date: Tue, 22 Oct 2024 16:11:48 +0000 Subject: [PATCH 1/4] upgrade sgx and ssl Signed-off-by: Bruno Vavala --- docker/pdo_services_base.dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/pdo_services_base.dockerfile b/docker/pdo_services_base.dockerfile index a039fe6f..f4357384 100644 --- a/docker/pdo_services_base.dockerfile +++ b/docker/pdo_services_base.dockerfile @@ -20,9 +20,9 @@ FROM pdo_base:${PDO_VERSION} ARG UBUNTU_VERSION=22.04 ARG UBUNTU_NAME=jammy -ARG SGX=2.22 -ARG OPENSSL=3.0.12 -ARG SGXSSL=3.0_Rev1 +ARG SGX=2.25 +ARG OPENSSL=3.0.14 +ARG SGXSSL=3.0_Rev4 RUN echo "deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu ${UBUNTU_NAME} main" >> /etc/apt/sources.list \ && wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add - \ From 832f89f6c40343ebb7586f7ef50faed8d4a50144 Mon Sep 17 00:00:00 2001 From: Mic Bowman Date: Wed, 30 Oct 2024 08:10:57 -0700 Subject: [PATCH 2/4] Updates to github workflows for PDO (#504) * Updates to github workflows for PDO *EXPERIMENTAL* -- new github workflows will only be visible and runnable when they are attached to the main branch. this commit will enable further testing. Expand the github workflows associated with the PDO repository. The old "ci" workflow has been moved to a workflow called "full_test" and is used as a condition for merge of PRs. It will no longer be run on every push but only on PR creation. Add a "configured_test" workflow that can be dispatched on demand. This workflow allows for different build and run parameters to be set interactively. Add a "build_docker" workflow that will attempt to build docker images pdo_client, pdo_services and pdo_ccf when a PR is merged successfully. This workflow can also be dispatched interactively. Signed-off-by: Mic Bowman * add documentation to workflows Signed-off-by: Mic Bowman --------- Signed-off-by: Mic Bowman --- .github/workflows/configured_test.yml | 71 ++++++++++++++++++++ .github/workflows/docker.yml | 63 +++++++++++++++++ .github/workflows/{ci.yaml => full_test.yml} | 32 ++++----- 3 files changed, 146 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/configured_test.yml create mode 100644 .github/workflows/docker.yml rename .github/workflows/{ci.yaml => full_test.yml} (57%) diff --git a/.github/workflows/configured_test.yml b/.github/workflows/configured_test.yml new file mode 100644 index 00000000..9bea032c --- /dev/null +++ b/.github/workflows/configured_test.yml @@ -0,0 +1,71 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This workflow is intended to provide an interactive way of configuring +# PDO tests. Common configuration variables can be set interactively to +# debug differences between local and github. + +name: Run specific PDO tests +on: + workflow_dispatch: + inputs: + interpreter: + description: 'Interpreter' + required: true + default: 'wawaka' + type: choice + options: + - wawaka + - wawaka-opt + logLevel: + description: 'Log level' + required: true + default: 'warning' + type: choice + options: + - debug + - info + - warning + memoryConfiguration: + description: 'Interpreter memory configuration' + required: false + default: MEDIUM + type: choice + options: + - SMALL + - MEDIUM + - LARGE + +jobs: + pdo_specific_tests: + name: Run specific PDO tests + runs-on: ubuntu-22.04 + + steps: + - name: Check out repo + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - name: Display branch name + run: | + echo "Building branch $GITHUB_HEAD_REF" + echo PDO VERSION is $(bin/get_version) + + - name: Build and run tests + env: + PDO_INTERPRETER: ${{ inputs.interpreter }} + PDO_LOG_LEVEL: ${{ inputs.logLevel }} + PDO_MEMORY_CONFIG: ${{ inputs.memoryConfiguration }} + PDO_DEBUG_BUILD: 1 + run: | + # The creation of a dummy branch is necessary for the CI tests + # to work on PRs. Based on empirical results, in the absence of + # this command, CI tests work on the main branch and on local + # branches. However, they fail as a PR is created. + git checkout -b ci-test-branch + . build/common-config.sh + make -C docker test diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..e3eaf1b7 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,63 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +name: Build and Push PDO Docker Images + +on: + workflow_dispatch: + + pull_request: + types: [closed] + branches: [main] + +jobs: + + docker_build: + + if: > + github.event.name == 'workflow_dispatch' || + github.event.name == 'pull_request' && github.event.pull_request.merged == true + name: Build PDO Images + runs-on: ubuntu-22.04 + + steps: + - name: Check out repo + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - name: Display branch name + run: | + echo "Building branch images for $GITHUB_HEAD_REF" + echo PDO VERSION is $(bin/get_version) + echo "PDO_VERSION=$(bin/get_version)" >> $GITHUB_ENV + echo "EVENT NAME: ${{ github.event.name }}" + echo "MERGED: ${{ github.event.pull_request.merged }}" + + - name: Build Docker Images + env: + PDO_INTERPRETER: wawaka + PDO_LOG_LEVEL: warning + run: | + git checkout -b ci-test-branch + . build/common-config.sh + make -C docker + + - name: Login to the ghcr.io Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tag and push the images + run: | + for image in pdo_services pdo_ccf pdo_client + do + docker image tag ghcr.io/{{ github.repository_owner }}/$image:$PDO_VERSION + docker image tag ghcr.io/{{ github.repository_owner }}/$image:latest + docker image push --all-tags ghcr.io/{{ github.repository_owner }}/$image + done diff --git a/.github/workflows/ci.yaml b/.github/workflows/full_test.yml similarity index 57% rename from .github/workflows/ci.yaml rename to .github/workflows/full_test.yml index c6faf01f..f947d2fe 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/full_test.yml @@ -2,12 +2,19 @@ # SPDX-License-Identifier: Apache-2.0 # -name: PDO CI -on: [pull_request, push] +# This workflow is intended to be used as a validity test for any +# pull request. That is, this is a minimal functionality that must +# be successfully executed prior to merging a pull request. Note +# that this can be overridden by adding '[skip ci]' in the commit +# name. This should not be done on the main PDO branch. + +name: Run full PDO tests +on: [ pull_request ] + jobs: - pdo_ci: + pdo_full_tests: if: "!contains(github.event.commits[0].message, '[skip ci]')" - name: PDO CI Job + name: Run full PDO tests runs-on: ubuntu-22.04 strategy: @@ -25,11 +32,10 @@ jobs: - name: Display branch name run: | - echo "Building branch $GITHUB_HEAD_REF" echo PDO VERSION is $(bin/get_version) + echo "BRANCH is $GITHUB_HEAD_REF" - name: Build and run tests - if: "!contains(github.event.commits[0].message, '[debug]')" env: PDO_INTERPRETER: ${{ matrix.interpreter }} PDO_LOG_LEVEL: warning @@ -41,17 +47,3 @@ jobs: git checkout -b ci-test-branch . build/common-config.sh make -C docker test - - - name: Build and run tests (DEBUG MODE) - if: "contains(github.event.commits[0].message, '[debug]')" - env: - PDO_INTERPRETER: ${{ matrix.interpreter }} - PDO_LOG_LEVEL: debug - run: | - # The creation of a dummy branch is necessary for the CI tests - # to work on PRs. Based on empirical results, in the absence of - # this command, CI tests work on the main branch and on local - # branches. However, they fail as a PR is created. - git checkout -b ci-test-branch - . build/common-config.sh - make -C docker test From 5e487ceaf3912da19fca728138e3188e7ec995ba Mon Sep 17 00:00:00 2001 From: Mic Bowman Date: Wed, 30 Oct 2024 15:11:54 -0700 Subject: [PATCH 3/4] Fix the docker workflow (#505) Two issues... first, moved the docker workflow to be triggered exclusively through interactive request. The policy for automated triggering can be determined at a later time. Second, fixed a couple bugs in the docker workflow. Now that the workflow is a part of the main branch, debugging is far easier. Note that this is been tested on a personal repository but not on an organizational repository such as hyperledger. Signed-off-by: Mic Bowman --- .github/workflows/docker.yml | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e3eaf1b7..81dd80c7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,22 +2,32 @@ # SPDX-License-Identifier: Apache-2.0 # +# This workflow will generate docker images for the +# current branch and push those images into the +# repository owners resources. + name: Build and Push PDO Docker Images on: workflow_dispatch: - pull_request: - types: [closed] - branches: [main] + # This section is commented out for the moment until a + # reasonable policy is determined for automated generation. + # The conditional execution must be evaluated as well. These + # are left here to serve as potential documentation for how + # the policy may be implemented. + + # pull_request: + # types: [closed] + # branches: [main] jobs: docker_build: - if: > - github.event.name == 'workflow_dispatch' || - github.event.name == 'pull_request' && github.event.pull_request.merged == true + # if: > + # github.event.name == 'workflow_dispatch' || + # github.event.name == 'pull_request' && github.event.pull_request.merged == true name: Build PDO Images runs-on: ubuntu-22.04 @@ -54,10 +64,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tag and push the images + env: + OWNER: ${{ github.repository_owner }} run: | + echo "Push images to ghcr.io/$OWNER" for image in pdo_services pdo_ccf pdo_client do - docker image tag ghcr.io/{{ github.repository_owner }}/$image:$PDO_VERSION - docker image tag ghcr.io/{{ github.repository_owner }}/$image:latest - docker image push --all-tags ghcr.io/{{ github.repository_owner }}/$image + docker image tag $image:$PDO_VERSION ghcr.io/$OWNER/$image:$PDO_VERSION + docker image tag $image:$PDO_VERSION ghcr.io/$OWNER/$image:latest + docker image push --all-tags ghcr.io/$OWNER/$image done From 4f4438405572bf5e2289589d97a3d59fa507f871 Mon Sep 17 00:00:00 2001 From: Mic Bowman Date: Fri, 1 Nov 2024 11:43:34 -0600 Subject: [PATCH 4/4] Add a lock around the block manager and key value start Signed-off-by: Mic Bowman --- python/pdo/common/block_store_manager.py | 6 ++++++ python/pdo/common/key_value.py | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/python/pdo/common/block_store_manager.py b/python/pdo/common/block_store_manager.py index c705edb9..8865fa66 100644 --- a/python/pdo/common/block_store_manager.py +++ b/python/pdo/common/block_store_manager.py @@ -33,14 +33,20 @@ # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +import threading +__block_manager_lock__ = threading.Lock() __local_block_manager__ = None def local_block_manager() : global __local_block_manager__ + + __block_manager_lock__.acquire() if __local_block_manager__ is None : block_store_file = pconfig.shared_configuration(['StorageService','BlockStore'], "./blockstore.mdb") __local_block_manager__ = BlockStoreManager(block_store_file, True) + __block_manager_lock__.release() + return __local_block_manager__ # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX diff --git a/python/pdo/common/key_value.py b/python/pdo/common/key_value.py index 6d4787cf..b556a14e 100644 --- a/python/pdo/common/key_value.py +++ b/python/pdo/common/key_value.py @@ -25,12 +25,19 @@ import logging logger = logging.getLogger(__name__) +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +import threading +__block_store_lock__ = threading.Lock() __block_store_initialized__ = False # ----------------------------------------------------------------- # ----------------------------------------------------------------- def KeyValueInitialize(block_store_file = None) : global __block_store_initialized__ + + __block_store_lock__.acquire() + if __block_store_initialized__ : raise Exception("duplicate block store initialization") @@ -41,6 +48,8 @@ def KeyValueInitialize(block_store_file = None) : kvs.block_store_open(block_store_file) __block_store_initialized__ = True + __block_store_lock__.release() + # ----------------------------------------------------------------- # ----------------------------------------------------------------- def KeyValueTerminate() :