draft: add a recipe for creating the morpheus_core conda package

ci/conda/recipes/morpheus_core/conda_build_config.yaml

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+c_compiler_version:
+  - 11.2
+
+cxx_compiler_version:
+  - 11.2
+
+cuda_compiler:
+  - cuda-nvcc
+
+cuda_compiler_version:
+  - 12.1
+
+python:
+  - 3.10
+
+rapids_version:
+  - 24.02

ci/conda/recipes/morpheus_core/meta.yaml

@@ -0,0 +1,140 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
+{% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
+
+package:
+  name: morpheus_core
+  version: {{ version }}
+
+source:
+  git_url: ../../../..
+
+outputs:
+
+  - name: morpheus_core
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+      script_env:
+        # Note: Setting a default value here will overwrite the value from the
+        # shell! Defaults should be set in the run_conda_build.sh script
+        - CMAKE_CUDA_ARCHITECTURES
+        - MORPHEUS_CACHE_DIR
+        - MORPHEUS_PYTHON_BUILD_STUBS
+        - MORPHEUS_SUPPORT_DOCA
+        - MORPHEUS_BUILD_MORPHEUS_LLM
+        - PARALLEL_LEVEL
+      run_exports:
+        - {{ pin_subpackage("morpheus_core", max_pin="x.x") }}
+    script: morpheus_core_build.sh
+
+    requirements:
+      build:
+        - {{ compiler("c") }}
+        - {{ compiler("cuda") }}
+        - {{ compiler("cxx") }}
+        - automake =1.16.5 # Needed for DOCA build
+        - ccache
+        - cmake =3.27
+        - cuda-cudart-dev {{ cuda_compiler_version }}.* # Needed by CMake to compile a test application
+        - cuda-version {{ cuda_compiler_version }}.*
+        - libtool # Needed for DOCA build
+        - ninja =1.11
+        - pkg-config =0.29 # for mrc cmake
+        - sysroot_linux-64 =2.17
+      host:
+        # CUDA dependencies
+        - cuda-cudart-dev {{ cuda_compiler_version }}.*
+        - cuda-libraries-dev {{ cuda_compiler_version }}.* # required by matx for libcublas-dev
+        - cuda-nvml-dev {{ cuda_compiler_version }}.*
+        - cuda-nvrtc-dev {{ cuda_compiler_version }}.*
+        - cuda-nvtx-dev {{ cuda_compiler_version }}.*
+        - cuda-version {{ cuda_compiler_version }}.*
+        # Non-CUDA dependencies
+        - cudf {{ rapids_version }}
+        - cython 3.0.*
+        - glog 0.6.*
+        - libcudf {{ rapids_version }}
+        - librdkafka >=1.9.2,<1.10.0a0
+        - mrc {{ minor_version }}
+        - nlohmann_json 3.11.*
+        - pip
+        - pybind11-stubgen 0.10.5
+        - python {{ python }}
+        - rapidjson 1.1.0
+        - scikit-build 0.17.6
+        - versioneer-518
+        - zlib 1.2.13 # required to build triton client
+      run:
+        # Runtime only requirements. This + setup.py is the definitive runtime requirement list
+        # This should be synced with `runtime` in dependencies.yaml
+        - appdirs
+        - beautifulsoup4
+        - click >=8
+        - cuda-version {{ cuda_compiler_version }}.*
+        - cudf
+        - cupy # Version determined from cudf
+        - datacompy =0.10.*
+        - dill =0.3.7
+        - docker-py =5.0.*
+        - elasticsearch ==8.9.0
+        - feedparser =6.0.*
+        - grpcio =1.59.*
+        - mlflow>=2.10.0,<3
+        - mrc
+        - networkx=2.8.8
+        - numpydoc =1.5.*
+        - pydantic
+        - pluggy =1.3.*
+        - python
+        - python-confluent-kafka >=1.9.2,<1.10.0a0
+        - python-graphviz
+        - pytorch-cuda
+        - pytorch * *cuda*
+        - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed
+        - requests
+        - requests-cache =1.1.*
+        - scikit-learn =1.3.2.*
+        - sqlalchemy <2.0 # 2.0 is incompatible with pandas=1.3
+        - tqdm =4.*
+        - tritonclient =2.34.*
+        - typing_utils =0.1.*
+        - watchdog =3.0.*
+        - websockets
+    test:
+      requires:
+        # test that cuml can be installed in the env
+        - cuml {{ rapids_version }}.*
+        - pynvml
+        - pytest-asyncio
+        - pytest-benchmark
+        - pytest-cov
+        - pytest =7.4.4
+      source_files:
+        - pyproject.toml
+        - scripts/fetch_data.py
+      imports:
+        - morpheus
+      commands:
+          - echo # make sure test requirements get installed
+
+about:
+  home: https://github.com/nv-morpheus/Morpheus
+  license: Apache-2.0
+  license_family: Apache
+  license_file: LICENSE
+  summary: Morpheus Cybersecurity Library

ci/conda/recipes/morpheus_core/morpheus_core_build.sh

@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# It is assumed that this script is executed from the root of the repo directory by conda-build
+# (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake)
+
+# Need to ensure this value is set before checking it in the if block
+MORPHEUS_SUPPORT_DOCA=-OFF
+MORPHEUS_BUILD_MORPHEUS_LLM=-OFF
+
+# This will store all of the cmake args. Make sure to prepend args to allow
+# incoming values to overwrite them
+CMAKE_ARGS=${CMAKE_ARGS:-""}
+
+export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
+export USE_SCCACHE=${USE_SCCACHE:-""}
+
+# Check for some mrc environment variables. Append to front of args to allow users to overwrite them
+if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
+   # Set the cache variable, then set the Staging prefix to allow for host searching
+   CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
+
+   # Double check that the cache dir has been created
+   mkdir -p ${MORPHEUS_CACHE_DIR}
+fi
+
+CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
+CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS=-"ON"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_WHEEL=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"RAPIDS"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11
+CMAKE_ARGS="--log-level=VERBOSE ${CMAKE_ARGS}"
+
+if [[ "${USE_SCCACHE}" == "1" ]]; then
+   CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}"
+fi
+
+echo "CC          : ${CC}"
+echo "CXX         : ${CXX}"
+echo "CUDAHOSTCXX : ${CUDAHOSTCXX}"
+echo "CUDA        : ${CUDA}"
+echo "CMAKE_ARGS  : ${CMAKE_ARGS}"
+
+echo "========Begin Env========"
+env
+echo "========End Env========"
+
+BUILD_DIR="build-conda"
+
+# Check if the build directory already exists. And if so, delete the
+# CMakeCache.txt and CMakeFiles to ensure a clean configuration
+if [[ -d "./${BUILD_DIR}" ]]; then
+   echo "Deleting old CMake files at ./${BUILD_DIR}"
+   rm -rf "./${BUILD_DIR}/CMakeCache.txt"
+   rm -rf "./${BUILD_DIR}/CMakeFiles"
+fi
+
+# Run configure
+cmake -B ${BUILD_DIR} \
+   ${CMAKE_ARGS} \
+   --log-level=verbose \
+   .
+
+# Build the components
+cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
+
+# Install just the mprpheus core python wheel components
+${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl