benchmark updates/fixes

conda/environments/dev_cuda-125_arch-x86_64.yaml

@@ -73,6 +73,7 @@ dependencies:
 - pydantic
 - pylibcudf=24.10
 - pylint=3.0.3
+- pynvml=11.4.1
 - pypdfium2=4.30
 - pytest-asyncio
 - pytest-benchmark=4.0

conda/environments/examples_cuda-125_arch-x86_64.yaml

@@ -54,7 +54,6 @@ dependencies:
 - requests-toolbelt=1.0
 - s3fs
 - scikit-learn=1.3.2
-- sentence-transformers=2.7
 - sqlalchemy<2.0
 - tqdm=4
 - transformers=4.36.2
@@ -77,5 +76,6 @@ dependencies:
   - milvus==2.3.5
   - nemollm==0.3.5
   - pymilvus==2.3.6
+  - sentence-transformers==2.7
   - torch==2.4.0+cu124
 name: examples_cuda-125_arch-x86_64

examples/digital_fingerprinting/production/morpheus/benchmarks/test_bench_e2e_dfp_pipeline.py

@@ -40,7 +40,7 @@
 from morpheus.stages.postprocess.serialize_stage import SerializeStage
 from morpheus.utils.column_info import DataFrameInputSchema
 from morpheus.utils.file_utils import date_extractor
-from morpheus.utils.logger import configure_logging
+from morpheus.utils.logger import set_log_level
 from morpheus_dfp.stages.dfp_file_batcher_stage import DFPFileBatcherStage
 from morpheus_dfp.stages.dfp_file_to_df import DFPFileToDataFrameStage
 from morpheus_dfp.stages.dfp_inference_stage import DFPInferenceStage
@@ -105,7 +105,7 @@ def dfp_training_pipeline_stages(pipe_config: Config,
                                  filenames: typing.List[str],
                                  reuse_cache=False):
 
-    configure_logging(log_level=logger.level)
+    set_log_level(log_level=logger.level)
 
     pipeline = LinearPipeline(pipe_config)
     pipeline.set_source(MultiFileSource(pipe_config, filenames=filenames))
@@ -157,7 +157,7 @@ def dfp_inference_pipeline_stages(pipe_config: Config,
                                   output_filepath: str,
                                   reuse_cache=False):
 
-    configure_logging(log_level=logger.level)
+    set_log_level(log_level=logger.level)
 
     pipeline = LinearPipeline(pipe_config)
     pipeline.set_source(MultiFileSource(pipe_config, filenames=filenames))

tests/benchmarks/README.md

@@ -48,32 +48,20 @@ Once Triton server finishes starting up, it will display the status of all loade
 
 ### Set up Morpheus Dev Container
 
-If you don't already have the Morpheus Dev container, run the following to build it:
-```bash
-./docker/build_container_dev.sh
-```
-
-Now run the container:
-```bash
-./docker/run_container_dev.sh
-```
+Follow the instructions [here](../../docs/source/developer_guide/contributing.md#build-in-docker-container) to set up your development environment in either a Docker container or Conda environment.
 
-Note that Morpheus containers are tagged by date. By default, `run_container_dev.sh` will try to use current date as tag. Therefore, if you are trying to run a container that was not built on the current date, you must set the `DOCKER_IMAGE_TAG` environment variable. For example,
-```bash
-DOCKER_IMAGE_TAG=dev-221003 ./docker/run_container_dev.sh
-```
+##### Install the required dependencies
 
-In the `/workspace` directory of the container, run the following to compile Morpheus:
+Run the following in the `/workspace` directory of your dev container:
 ```bash
-./scripts/compile.sh
+mamba env update \
+  -n ${CONDA_DEFAULT_ENV} \
+  --file ./conda/environments/examples_cuda-125_arch-x86_64.yaml
 ```
 
-Now install Morpheus:
-```bash
-pip install -e /workspace
-```
+##### Fetch input data for benchmarks
 
-Fetch input data for benchmarks:
+Run the following in the `/workspace` directory of your dev container:
 ```bash
 ./scripts/fetch_data.py fetch validation
 ```
@@ -124,7 +112,6 @@ The `test_bench_e2e_pipelines.py` script contains several benchmarks within it.
 - `test_sid_nlp_e2e`
 - `test_abp_fil_e2e`
 - `test_phishing_nlp_e2e`
-- `test_cloudtrail_ae_e2e`
 
 For example, to run E2E benchmarks on the SID NLP workflow:
 ```bash
@@ -138,11 +125,10 @@ pytest -s --run_benchmark --benchmark-enable --benchmark-warmup=on --benchmark-w
 
 The console output should look like this:
 ```
---------------------------------------------------------------------------------- benchmark: 4 tests --------------------------------------------------------------------------------
+--------------------------------------------------------------------------------- benchmark: 3 tests --------------------------------------------------------------------------------
 Name (time in s)              Min               Max              Mean            StdDev            Median               IQR            Outliers     OPS            Rounds  Iterations
 -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 test_sid_nlp_e2e           1.8907 (1.0)      1.9817 (1.0)      1.9400 (1.0)      0.0325 (2.12)     1.9438 (1.0)      0.0297 (1.21)          2;0  0.5155 (1.0)           5           1
-test_cloudtrail_ae_e2e     3.3403 (1.77)     3.3769 (1.70)     3.3626 (1.73)     0.0153 (1.0)      3.3668 (1.73)     0.0245 (1.0)           1;0  0.2974 (0.58)          5           1
 test_abp_fil_e2e           5.1271 (2.71)     5.3044 (2.68)     5.2083 (2.68)     0.0856 (5.59)     5.1862 (2.67)     0.1653 (6.75)          1;0  0.1920 (0.37)          5           1
 test_phishing_nlp_e2e      5.6629 (3.00)     6.0987 (3.08)     5.8835 (3.03)     0.1697 (11.08)    5.8988 (3.03)     0.2584 (10.55)         2;0  0.1700 (0.33)          5           1
 -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -193,7 +179,7 @@ Additional benchmark stats for each workflow:
 
 ### Production DFP E2E Benchmarks
 
-Note that the `test_cloudtrail_ae_e2e` benchmarks measure performance of a pipeline built using [Starter DFP](../../examples/digital_fingerprinting/starter/README.md) stages. Separate benchmark tests are also provided to measure performance of the example [Production DFP](../../examples/digital_fingerprinting/production/README.md) pipelines. More information about running those benchmarks can be found [here](../../examples/digital_fingerprinting/production/morpheus/benchmarks/README.md).
+Separate benchmark tests are provided to measure performance of the example [Production DFP](../../examples/digital_fingerprinting/production/README.md) pipelines. More information about running those benchmarks can be found [here](../../examples/digital_fingerprinting/production/morpheus/benchmarks/README.md).
 
 You can use the same Dev container created here to run the Production DFP benchmarks. You would just need to install additional dependencies as follows:
 

tests/benchmarks/e2e_test_configs.json

@@ -1,5 +1,5 @@
 {
-    "triton_server_url": "localhost:8001",
+    "triton_server_url": "localhost:8000",
     "test_sid_nlp_e2e": {
         "file_path": "../../models/datasets/validation-data/sid-validation-data.csv",
         "repeat": 10,

tests/benchmarks/test_bench_completion_pipeline.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import collections.abc
+import os
 import typing
 
 import pytest
@@ -82,4 +83,6 @@ def test_completion_pipe(benchmark: collections.abc.Callable[[collections.abc.Ca
                          config: Config,
                          dataset: DatasetManager,
                          llm_service_cls: type[LLMService]):
+    if llm_service_cls == OpenAIChatService:
+        os.environ.update({"OPENAI_API_KEY": "test_api_key"})
     benchmark(_run_pipeline, config, llm_service_cls, source_df=dataset["countries.csv"])

tests/benchmarks/test_bench_e2e_pipelines.py

@@ -21,7 +21,6 @@
 
 from _utils import TEST_DIRS
 from morpheus.config import Config
-from morpheus.config import ConfigAutoEncoder
 from morpheus.config import ConfigFIL
 from morpheus.config import CppConfig
 from morpheus.config import PipelineModes
@@ -41,7 +40,7 @@
 from morpheus.stages.preprocess.preprocess_nlp_stage import PreprocessNLPStage
 from morpheus.stages.preprocess.train_ae_stage import TrainAEStage
 from morpheus.utils.file_utils import load_labels_file
-from morpheus.utils.logger import configure_logging
+from morpheus.utils.logger import set_log_level
 
 E2E_CONFIG_FILE = os.path.join(TEST_DIRS.morpheus_root, "tests/benchmarks/e2e_test_configs.json")
 with open(E2E_CONFIG_FILE, 'r', encoding='UTF-8') as f:
@@ -50,7 +49,7 @@
 
 def nlp_pipeline(config: Config, input_file, repeat, vocab_hash_file, output_file, model_name):
 
-    configure_logging(log_level=logging.INFO)
+    set_log_level(log_level=logging.DEBUG)
 
     pipeline = LinearPipeline(config)
     pipeline.set_source(FileSourceStage(config, filename=input_file, repeat=repeat))
@@ -77,7 +76,7 @@ def nlp_pipeline(config: Config, input_file, repeat, vocab_hash_file, output_fil
 
 def fil_pipeline(config: Config, input_file, repeat, output_file, model_name):
 
-    configure_logging(log_level=logging.INFO)
+    set_log_level(log_level=logging.DEBUG)
 
     pipeline = LinearPipeline(config)
     pipeline.set_source(FileSourceStage(config, filename=input_file, repeat=repeat))
@@ -99,7 +98,8 @@ def fil_pipeline(config: Config, input_file, repeat, output_file, model_name):
 
 def ae_pipeline(config: Config, input_glob, repeat, train_data_glob, output_file):
 
-    configure_logging(log_level=logging.INFO)
+    set_log_level(log_level=logging.DEBUG)
+
     pipeline = LinearPipeline(config)
     pipeline.set_source(CloudTrailSourceStage(config, input_glob=input_glob, max_files=200, repeat=repeat))
     pipeline.add_stage(
@@ -196,30 +196,3 @@ def test_phishing_nlp_e2e(benchmark, tmp_path):
     model_name = "phishing-bert-onnx"
 
     benchmark(nlp_pipeline, config, input_filepath, repeat, vocab_filepath, output_filepath, model_name)
-
-
-@pytest.mark.benchmark
-def test_cloudtrail_ae_e2e(benchmark, tmp_path):
-
-    config = Config()
-    config.mode = PipelineModes.AE
-    config.num_threads = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["num_threads"]
-    config.pipeline_batch_size = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["pipeline_batch_size"]
-    config.model_max_batch_size = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["model_max_batch_size"]
-    config.feature_length = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["feature_length"]
-    config.edge_buffer_size = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["edge_buffer_size"]
-    config.class_labels = ["reconstruct_loss", "zscore"]
-
-    config.ae = ConfigAutoEncoder()
-    config.ae.userid_column_name = "userIdentityaccountId"
-    config.ae.userid_filter = "Account-123456789"
-    ae_cols_filepath = os.path.join(TEST_DIRS.data_dir, 'columns_ae_cloudtrail.txt')
-    config.ae.feature_columns = load_labels_file(ae_cols_filepath)
-    CppConfig.set_should_use_cpp(False)
-
-    input_glob = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["input_glob_path"]
-    repeat = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["repeat"]
-    train_glob = E2E_TEST_CONFIGS["test_cloudtrail_ae_e2e"]["train_glob_path"]
-    output_filepath = os.path.join(tmp_path, "cloudtrail_ae_e2e_output.csv")
-
-    benchmark(ae_pipeline, config, input_glob, repeat, train_glob, output_filepath)

tests/benchmarks/test_bench_rag_standalone_pipeline.py

@@ -135,10 +135,12 @@ def test_rag_standalone_pipe(benchmark: collections.abc.Callable[[collections.ab
                              repeat_count: int,
                              import_mod: types.ModuleType,
                              llm_service_name: str):
+    if llm_service_name=="openai":
+        os.environ.update({"OPENAI_API_KEY": "test_api_key"})
     collection_name = f"test_bench_rag_standalone_pipe_{llm_service_name}"
     populate_milvus(milvus_server_uri=milvus_server_uri,
                     collection_name=collection_name,
-                    resource_kwargs=import_mod.build_milvus_config(embedding_size=EMBEDDING_SIZE),
+                    resource_kwargs=import_mod.build_default_milvus_config(embedding_size=EMBEDDING_SIZE),
                     df=dataset["service/milvus_rss_data.json"],
                     overwrite=True)
 

tests/benchmarks/test_bench_vdb_upload_pipeline.py

@@ -79,7 +79,7 @@ def _run_pipeline(config: Config,
     pipe.add_stage(
         WriteToVectorDBStage(config,
                              resource_name=collection_name,
-                             resource_kwargs=utils_mod.build_milvus_config(embedding_size=EMBEDDING_SIZE),
+                             resource_kwargs=utils_mod.build_default_milvus_config(embedding_size=EMBEDDING_SIZE),
                              recreate=True,
                              service="milvus",
                              uri=milvus_server_uri))
@@ -92,7 +92,7 @@ def _run_pipeline(config: Config,
 @pytest.mark.benchmark
 @pytest.mark.import_mod([
     os.path.join(TEST_DIRS.examples_dir, 'llm/common/utils.py'),
-    os.path.join(TEST_DIRS.examples_dir, 'llm/common/web_scraper_stage.py'),
+    os.path.join(TEST_DIRS.examples_dir, 'llm/vdb_upload/module/web_scraper_stage.py'),
 ])
 @mock.patch('feedparser.http.get')
 @mock.patch('requests.Session')