diff --git a/.env b/.env index 968097cd..64c3f7aa 100644 --- a/.env +++ b/.env @@ -57,7 +57,7 @@ SPDK_URL="https://spdk.io" SPDK_PKGDEP_ARGS="--rbd" # check spdk/configure --help -SPDK_CONFIGURE_ARGS="--with-rbd --disable-tests --disable-unit-tests --disable-examples --enable-debug" +SPDK_CONFIGURE_ARGS="--with-idxd --with-rbd --disable-tests --disable-unit-tests --disable-examples --enable-debug" SPDK_TARGET_ARCH="x86-64-v2" SPDK_MAKEFLAGS= SPDK_CENTOS_BASE="https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/" diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index f40d9f8f..345df6ee 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -86,47 +86,47 @@ jobs: path: | ceph.tar - build-arm64: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Build container images - spdk - run: make build SVC=spdk TARGET_ARCH=arm64 - - - name: Build container images - bdevperf - run: make build SVC=bdevperf TARGET_ARCH=arm64 - - - name: Build container images - nvmeof - run: make build SVC=nvmeof TARGET_ARCH=arm64 - - - name: Build container images - nvmeof-cli - run: make build SVC=nvmeof-cli TARGET_ARCH=arm64 - - - name: Build container images - ceph - run: make build SVC=ceph TARGET_ARCH=arm64 - - - name: Save container images - run: | - . .env - docker tag $QUAY_NVMEOF:$NVMEOF_VERSION $QUAY_NVMEOF:$NVMEOF_VERSION-arm64 - docker tag $QUAY_NVMEOFCLI:$NVMEOF_VERSION $QUAY_NVMEOFCLI:$NVMEOF_VERSION-arm64 - docker save $QUAY_NVMEOF:$NVMEOF_VERSION-arm64 > nvmeof-arm64.tar - docker save $QUAY_NVMEOFCLI:$NVMEOF_VERSION-arm64 > nvmeof-cli-arm64.tar - - - name: Upload nvmeof-arm64 container images - uses: actions/upload-artifact@v4 - with: - name: container_images_nvmeof_arm64 - path: | - nvmeof-arm64.tar - nvmeof-cli-arm64.tar +# build-arm64: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 +# with: +# submodules: recursive + +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v3 + +# - name: Build container images - spdk +# run: make build SVC=spdk TARGET_ARCH=arm64 + +# - name: Build container images - bdevperf +# run: make build SVC=bdevperf TARGET_ARCH=arm64 + +# - name: Build container images - nvmeof +# run: make build SVC=nvmeof TARGET_ARCH=arm64 + +# - name: Build container images - nvmeof-cli +# run: make build SVC=nvmeof-cli TARGET_ARCH=arm64 + +# - name: Build container images - ceph +# run: make build SVC=ceph TARGET_ARCH=arm64 + +# - name: Save container images +# run: | +# . .env +# docker tag $QUAY_NVMEOF:$NVMEOF_VERSION $QUAY_NVMEOF:$NVMEOF_VERSION-arm64 +# docker tag $QUAY_NVMEOFCLI:$NVMEOF_VERSION $QUAY_NVMEOFCLI:$NVMEOF_VERSION-arm64 +# docker save $QUAY_NVMEOF:$NVMEOF_VERSION-arm64 > nvmeof-arm64.tar +# docker save $QUAY_NVMEOFCLI:$NVMEOF_VERSION-arm64 > nvmeof-cli-arm64.tar + +# - name: Upload nvmeof-arm64 container images +# uses: actions/upload-artifact@v4 +# with: +# name: container_images_nvmeof_arm64 +# path: | +# nvmeof-arm64.tar +# nvmeof-cli-arm64.tar pytest: needs: [build, build-ceph] @@ -684,7 +684,7 @@ jobs: push-images-to-ceph-registry: if: github.event_name == 'release' - needs: [pytest, demo, discovery, ha, atom, build-arm64] + needs: [pytest, demo, discovery, ha, atom] runs-on: ubuntu-latest steps: @@ -701,8 +701,8 @@ jobs: run: | docker load < nvmeof.tar docker load < nvmeof-cli.tar - docker load < nvmeof-arm64.tar - docker load < nvmeof-cli-arm64.tar + #docker load < nvmeof-arm64.tar + #docker load < nvmeof-cli-arm64.tar - name: Login to quay.io uses: docker/login-action@v2 @@ -714,7 +714,7 @@ jobs: - name: Publish nvmeof containers when release/tag is created run: | make push - make push TAG_SUFFIX="-arm64" + #make push TAG_SUFFIX="-arm64" make push-manifest-list push-devel-image-to-ceph-registry: diff --git a/control/server.py b/control/server.py index 51bfa69c..4e9493b3 100644 --- a/control/server.py +++ b/control/server.py @@ -23,6 +23,7 @@ import spdk.rpc import spdk.rpc.client as rpc_client import spdk.rpc.nvmf as rpc_nvmf +import spdk.rpc.dsa as rpc_dsa from .proto import gateway_pb2 as pb2 from .proto import gateway_pb2_grpc as pb2_grpc @@ -475,7 +476,7 @@ def _start_spdk(self, omap_state): self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}") spdk_tgt_cmd_extra_args = self.config.get_with_default( "spdk", "tgt_cmd_extra_args", "") - cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path] + cmd = [spdk_tgt_path, "--wait-for-rpc", "-u", "-r", self.spdk_rpc_socket_path] # Add extra args from the conf file if spdk_tgt_cmd_extra_args: @@ -558,6 +559,9 @@ def _start_spdk(self, omap_state): log_level=protocol_log_level, conn_retries=conn_retries, ) + # Set config and enable dsa accel module offload. + self._probe_dsa() + except Exception: self.logger.exception(f"Unable to initialize SPDK") raise @@ -653,6 +657,16 @@ def _stop_discovery(self): self.discovery_pid = None + def _probe_dsa(self): + """Initializes dsa accel module offload.""" + try: + res = rpc_dsa.dsa_scan_accel_module(self.spdk_rpc_client) + spdk.rpc.framework_start_init(self.spdk_rpc_client) + except Exception: + self.logger.exception(f"Failed to probe dsa accel module offload") + raise + self.logger.debug(f"dsa_scan_accel_module: {res=}") + def _create_transport(self, trtype): """Initializes a transport type.""" args = {'trtype': trtype}