Benchmark updates/fixes

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -88,6 +88,7 @@ dependencies:
 - pydantic
 - pylibcudf=24.10
 - pylint=3.0.3
+- pynvml=11.4
 - pypdf=3.17.4
 - pypdfium2=4.30
 - pytest-asyncio
@@ -107,7 +108,6 @@ dependencies:
 - s3fs
 - scikit-build=0.17.6
 - scikit-learn=1.3.2
-- sentence-transformers=2.7
 - sphinx
 - sphinx_rtd_theme
 - sqlalchemy<2.0
@@ -140,5 +140,6 @@ dependencies:
   - nemollm==0.3.5
   - pymilvus==2.3.6
   - pytest-kafka==0.6.0
+  - sentence-transformers[onnx-gpu]==2.7
   - torch==2.4.0+cu124
 name: all_cuda-125_arch-x86_64

conda/environments/dev_cuda-125_arch-x86_64.yaml

@@ -73,6 +73,7 @@ dependencies:
 - pydantic
 - pylibcudf=24.10
 - pylint=3.0.3
+- pynvml=11.4
 - pypdfium2=4.30
 - pytest-asyncio
 - pytest-benchmark=4.0

conda/environments/examples_cuda-125_arch-x86_64.yaml

@@ -55,7 +55,6 @@ dependencies:
 - requests-toolbelt=1.0
 - s3fs
 - scikit-learn=1.3.2
-- sentence-transformers=2.7
 - sqlalchemy<2.0
 - tqdm=4
 - transformers=4.36.2
@@ -78,5 +77,6 @@ dependencies:
   - milvus==2.3.5
   - nemollm==0.3.5
   - pymilvus==2.3.6
+  - sentence-transformers[onnx-gpu]==2.7
   - torch==2.4.0+cu124
 name: examples_cuda-125_arch-x86_64

dependencies.yaml

@@ -311,6 +311,7 @@ dependencies:
           - include-what-you-use=0.20
           - isort
           - pylint=3.0.3
+          - pynvml=11.4
           - vale=3.7
           - vale-styles-microsoft
           - vale-styles-write-good
@@ -466,14 +467,14 @@ dependencies:
           - *pypdfium2
           - *python-docx
           - requests-toolbelt=1.0 # Transitive dep needed by nemollm, specified here to ensure we get a compatible version
-          - sentence-transformers=2.7
           - pip
           - pip:
             - langchain==0.1.16
             - langchain-nvidia-ai-endpoints==0.0.11
             - faiss-cpu
             - google-search-results==2.4
             - nemollm==0.3.5
+            - sentence-transformers[onnx-gpu]==2.7
 
   model-training-tuning:
     common:

examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py

@@ -40,7 +40,7 @@
 from morpheus.stages.postprocess.serialize_stage import SerializeStage
 from morpheus.utils.column_info import DataFrameInputSchema
 from morpheus.utils.file_utils import date_extractor
-from morpheus.utils.logger import configure_logging
+from morpheus.utils.logger import set_log_level
 from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
 from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
 from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage
@@ -105,7 +105,7 @@ def dfp_training_pipeline_stages(pipe_config: Config,
                                  filenames: typing.List[str],
                                  reuse_cache=False):
 
-    configure_logging(log_level=logger.level)
+    set_log_level(log_level=logger.level)
 
     pipeline = LinearPipeline(pipe_config)
     pipeline.set_source(MultiFileSource(pipe_config, filenames=filenames))
@@ -157,7 +157,7 @@ def dfp_inference_pipeline_stages(pipe_config: Config,
                                   output_filepath: str,
                                   reuse_cache=False):
 
-    configure_logging(log_level=logger.level)
+    set_log_level(log_level=logger.level)
 
     pipeline = LinearPipeline(pipe_config)
     pipeline.set_source(MultiFileSource(pipe_config, filenames=filenames))

tests/benchmarks/README.md

@@ -46,36 +46,14 @@ Once Triton server finishes starting up, it will display the status of all loade
 +--------------------+---------+--------+
 ```
 
-### Set up Morpheus Dev Container
+### Build Morpheus Conda environment
 
-If you don't already have the Morpheus Dev container, run the following to build it:
-```bash
-./docker/build_container_dev.sh
-```
-
-Now run the container:
-```bash
-./docker/run_container_dev.sh
-```
-
-Note that Morpheus containers are tagged by date. By default, `run_container_dev.sh` will try to use current date as tag. Therefore, if you are trying to run a container that was not built on the current date, you must set the `DOCKER_IMAGE_TAG` environment variable. For example,
-```bash
-DOCKER_IMAGE_TAG=dev-221003 ./docker/run_container_dev.sh
-```
-
-In the `/workspace` directory of the container, run the following to compile Morpheus:
-```bash
-./scripts/compile.sh
-```
-
-Now install Morpheus:
-```bash
-pip install -e /workspace
-```
+Follow the instructions [here](../../docs/source/developer_guide/contributing.md#build-in-a-conda-environment) to build your Morpheus Conda environment.
 
-Fetch input data for benchmarks:
+In step 3 of the instructions, choose the command to install all dependencies.
 ```bash
-./scripts/fetch_data.py fetch validation
+conda env create --solver=libmamba -n morpheus --file conda/environments/all_cuda-125_arch-x86_64.yaml
+conda activate morpheus
 ```
 
 
@@ -124,7 +102,6 @@ The `test_bench_e2e_pipelines.py` script contains several benchmarks within it.
 - `test_sid_nlp_e2e`
 - `test_abp_fil_e2e`
 - `test_phishing_nlp_e2e`
-- `test_cloudtrail_ae_e2e`
 
 For example, to run E2E benchmarks on the SID NLP workflow:
 ```bash
@@ -138,11 +115,10 @@ pytest -s --run_benchmark --benchmark-enable --benchmark-warmup=on --benchmark-w
 
 The console output should look like this:
 ```
---------------------------------------------------------------------------------- benchmark: 4 tests --------------------------------------------------------------------------------
+--------------------------------------------------------------------------------- benchmark: 3 tests --------------------------------------------------------------------------------
 Name (time in s)              Min               Max              Mean            StdDev            Median               IQR            Outliers     OPS            Rounds  Iterations
 -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 test_sid_nlp_e2e           1.8907 (1.0)      1.9817 (1.0)      1.9400 (1.0)      0.0325 (2.12)     1.9438 (1.0)      0.0297 (1.21)          2;0  0.5155 (1.0)           5           1
-test_cloudtrail_ae_e2e     3.3403 (1.77)     3.3769 (1.70)     3.3626 (1.73)     0.0153 (1.0)      3.3668 (1.73)     0.0245 (1.0)           1;0  0.2974 (0.58)          5           1
 test_abp_fil_e2e           5.1271 (2.71)     5.3044 (2.68)     5.2083 (2.68)     0.0856 (5.59)     5.1862 (2.67)     0.1653 (6.75)          1;0  0.1920 (0.37)          5           1
 test_phishing_nlp_e2e      5.6629 (3.00)     6.0987 (3.08)     5.8835 (3.03)     0.1697 (11.08)    5.8988 (3.03)     0.2584 (10.55)         2;0  0.1700 (0.33)          5           1
 -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -193,7 +169,7 @@ Additional benchmark stats for each workflow:
 
 ### Production DFP E2E Benchmarks
 
-Note that the `test_cloudtrail_ae_e2e` benchmarks measure performance of a pipeline built using [Starter DFP](../../examples/digital_fingerprinting/starter/README.md) stages. Separate benchmark tests are also provided to measure performance of the example [Production DFP](../../examples/digital_fingerprinting/production/README.md) pipelines. More information about running those benchmarks can be found [here](../../examples/digital_fingerprinting/production/morpheus/benchmarks/README.md).
+Separate benchmark tests are provided to measure performance of the example [Production DFP](../../examples/digital_fingerprinting/production/README.md) pipelines. More information about running those benchmarks can be found [here](../../examples/digital_fingerprinting/production/morpheus/benchmarks/README.md).
 
 You can use the same Dev container created here to run the Production DFP benchmarks. You would just need to install additional dependencies as follows:
 

tests/benchmarks/e2e_test_configs.json

@@ -1,5 +1,5 @@
 {
-    "triton_server_url": "localhost:8001",
+    "triton_server_url": "localhost:8000",
     "test_sid_nlp_e2e": {
         "file_path": "../../models/datasets/validation-data/sid-validation-data.csv",
         "repeat": 10,

tests/benchmarks/test_bench_completion_pipeline.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import collections.abc
+import os
 import typing
 
 import pytest
@@ -82,4 +83,6 @@ def test_completion_pipe(benchmark: collections.abc.Callable[[collections.abc.Ca
                          config: Config,
                          dataset: DatasetManager,
                          llm_service_cls: type[LLMService]):
+    if llm_service_cls == OpenAIChatService:
+        os.environ.update({"OPENAI_API_KEY": "test_api_key"})
     benchmark(_run_pipeline, config, llm_service_cls, source_df=dataset["countries.csv"])

tests/benchmarks/test_bench_e2e_pipelines.py

@@ -21,27 +21,21 @@
 
 from _utils import TEST_DIRS
 from morpheus.config import Config
-from morpheus.config import ConfigAutoEncoder
 from morpheus.config import ConfigFIL
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
 from morpheus.pipeline.linear_pipeline import LinearPipeline
 from morpheus.stages.general.monitor_stage import MonitorStage
-from morpheus.stages.inference.auto_encoder_inference_stage import AutoEncoderInferenceStage
 from morpheus.stages.inference.triton_inference_stage import TritonInferenceStage
-from morpheus.stages.input.cloud_trail_source_stage import CloudTrailSourceStage
 from morpheus.stages.input.file_source_stage import FileSourceStage
 from morpheus.stages.output.write_to_file_stage import WriteToFileStage
 from morpheus.stages.postprocess.add_classifications_stage import AddClassificationsStage
-from morpheus.stages.postprocess.add_scores_stage import AddScoresStage
 from morpheus.stages.postprocess.serialize_stage import SerializeStage
 from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
-from morpheus.stages.preprocess.preprocess_ae_stage import PreprocessAEStage
 from morpheus.stages.preprocess.preprocess_fil_stage import PreprocessFILStage
 from morpheus.stages.preprocess.preprocess_nlp_stage import PreprocessNLPStage
-from morpheus.stages.preprocess.train_ae_stage import TrainAEStage
 from morpheus.utils.file_utils import load_labels_file
-from morpheus.utils.logger import configure_logging
+from morpheus.utils.logger import set_log_level
 
 E2E_CONFIG_FILE = os.path.join(TEST_DIRS.morpheus_root, "tests/benchmarks/e2e_test_configs.json")
 with open(E2E_CONFIG_FILE, 'r', encoding='UTF-8') as f:
@@ -50,7 +44,7 @@
 
 def nlp_pipeline(config: Config, input_file, repeat, vocab_hash_file, output_file, model_name):
 
-    configure_logging(log_level=logging.INFO)
+    set_log_level(log_level=logging.DEBUG)
 
     pipeline = LinearPipeline(config)
     pipeline.set_source(FileSourceStage(config, filename=input_file, repeat=repeat))
@@ -77,7 +71,7 @@ def nlp_pipeline(config: Config, input_file, repeat, vocab_hash_file, output_fil
 
 def fil_pipeline(config: Config, input_file, repeat, output_file, model_name):
 
-    configure_logging(log_level=logging.INFO)
+    set_log_level(log_level=logging.DEBUG)
 
     pipeline = LinearPipeline(config)
     pipeline.set_source(FileSourceStage(config, filename=input_file, repeat=repeat))
@@ -97,28 +91,6 @@ def fil_pipeline(config: Config, input_file, repeat, output_file, model_name):
     pipeline.run()
 
 
-def ae_pipeline(config: Config, input_glob, repeat, train_data_glob, output_file):
-
-    configure_logging(log_level=logging.INFO)
-    pipeline = LinearPipeline(config)
-    pipeline.set_source(CloudTrailSourceStage(config, input_glob=input_glob, max_files=200, repeat=repeat))
-    pipeline.add_stage(
-        TrainAEStage(config,
-                     train_data_glob=train_data_glob,
-                     source_stage_class="morpheus.stages.input.cloud_trail_source_stage.CloudTrailSourceStage",
-                     seed=42,
-                     sort_glob=True))
-    pipeline.add_stage(PreprocessAEStage(config))
-    pipeline.add_stage(AutoEncoderInferenceStage(config))
-    pipeline.add_stage(AddScoresStage(config))
-    pipeline.add_stage(MonitorStage(config, log_level=logging.INFO))
-    pipeline.add_stage(SerializeStage(config))
-    pipeline.add_stage(WriteToFileStage(config, filename=output_file, overwrite=True))
-
-    pipeline.build()
-    pipeline.run()
-
-
 @pytest.mark.benchmark
 def test_sid_nlp_e2e(benchmark, tmp_path):
 
@@ -196,30 +168,3 @@ def test_phishing_nlp_e2e(benchmark, tmp_path):
     model_name = "phishing-bert-onnx"
 
     benchmark(nlp_pipeline, config, input_filepath, repeat, vocab_filepath, output_filepath, model_name)
-
-
-@pytest.mark.benchmark
-def test_cloudtrail_ae_e2e(benchmark, tmp_path):
-
-    config = Config()
-    config.mode = PipelineModes.AE
-    config.num_threads = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["num_threads"]
-    config.pipeline_batch_size = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["pipeline_batch_size"]
-    config.model_max_batch_size = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["model_max_batch_size"]
-    config.feature_length = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["feature_length"]
-    config.edge_buffer_size = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["edge_buffer_size"]
-    config.class_labels = ["reconstruct_loss", "zscore"]
-
-    config.ae = ConfigAutoEncoder()
-    config.ae.userid_column_name = "userIdentityaccountId"
-    config.ae.userid_filter = "Account-123456789"
-    ae_cols_filepath = os.path.join(TEST_DIRS.data_dir, 'columns_ae_cloudtrail.txt')
-    config.ae.feature_columns = load_labels_file(ae_cols_filepath)
-    CppConfig.set_should_use_cpp(False)
-
-    input_glob = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["input_glob_path"]
-    repeat = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["repeat"]
-    train_glob = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["train_glob_path"]
-    output_filepath = os.path.join(tmp_path, "cloudtrail_ae_e2e_output.csv")
-
-    benchmark(ae_pipeline, config, input_glob, repeat, train_glob, output_filepath)

tests/benchmarks/test_bench_rag_standalone_pipeline.py

@@ -135,10 +135,12 @@ def test_rag_standalone_pipe(benchmark: collections.abc.Callable[[collections.ab
                              repeat_count: int,
                              import_mod: types.ModuleType,
                              llm_service_name: str):
+    if llm_service_name == "openai":
+        os.environ.update({"OPENAI_API_KEY": "test_api_key"})
     collection_name = f"test_bench_rag_standalone_pipe_{llm_service_name}"
     populate_milvus(milvus_server_uri=milvus_server_uri,
                     collection_name=collection_name,
-                    resource_kwargs=import_mod.build_milvus_config(embedding_size=EMBEDDING_SIZE),
+                    resource_kwargs=import_mod.build_default_milvus_config(embedding_size=EMBEDDING_SIZE),
                     df=dataset["service/milvus_rss_data.json"],
                     overwrite=True)
 

tests/benchmarks/test_bench_vdb_upload_pipeline.py

@@ -79,7 +79,7 @@ def _run_pipeline(config: Config,
     pipe.add_stage(
         WriteToVectorDBStage(config,
                              resource_name=collection_name,
-                             resource_kwargs=utils_mod.build_milvus_config(embedding_size=EMBEDDING_SIZE),
+                             resource_kwargs=utils_mod.build_default_milvus_config(embedding_size=EMBEDDING_SIZE),
                              recreate=True,
                              service="milvus",
                              uri=milvus_server_uri))
@@ -92,7 +92,7 @@ def _run_pipeline(config: Config,
 @pytest.mark.benchmark
 @pytest.mark.import_mod([
     os.path.join(TEST_DIRS.examples_dir, 'llm/common/utils.py'),
-    os.path.join(TEST_DIRS.examples_dir, 'llm/common/web_scraper_stage.py'),
+    os.path.join(TEST_DIRS.examples_dir, 'llm/vdb_upload/module/web_scraper_stage.py'),
 ])
 @mock.patch('feedparser.http.get')
 @mock.patch('requests.Session')