clp-bench: Refactor and Add New Results

README.md

@@ -1,23 +1,54 @@
 # clp-bench
-clp-bench is a tool for benchmarking [CLP] as well as other log management tools. The tool itself is
-a Python package, and we also provide a [web interface][ui] for viewing results.
 
-The methodology for the benchmarks is described [here](docs/methodology.md).
+**clp-bench** is a tool for benchmarking [glt] and other log management systems. It functions as a 
+Python package and includes a [web interface][ui] for displaying benchmark results.
 
 ## Requirements
 
-* Docker
-* Python v3.10 or higher
+- Docker
+- Python v3.10 or higher
 
-# Set up
+# Setup
 
 ```shell
 python3 -m venv venv
 . venv/bin/activate
 pip install -e .
 ```
 
-You can use `clp-bench --help` to see usage instructions.
+To view usage instructions, run `clp-bench --help`.
 
-[CLP]: https://github.com/y-scope/clp
-[ui]: ui
+# Results
+You can view the current benchmark results [here][webui]. The benchmark currently evaluates
+ingestion and query performance for the following tools:
+
+| Tool                           | Version     |
+|--------------------------------|-------------|
+| [ClickHouse][clickhouse]       | 23.3.1.2823 |
+| [glt][glt]                     | 0.2.0       |
+| [clp-s][clp-s]                 | 0.2.0       |
+| [Elasticsearch][elasticsearch] | 8.6.2       |
+| `grep`                         | 3.7         |
+| [Loki][loki]                   | 3.0.0       |
+| [MongoDB][mongodb]             | 6.0.19      |
+| [Splunk][splunk]               | 9.3.2       |
+
+Don't see a tool here? Feel free to file a [GitHub issue][new-issue] for one or follow this
+[guide][adding-a-tool] for how to add one. 
+
+For a detailed description of the benchmarking 
+methodology, see [here][methodology].
+
+[assets]: assets
+[clickhouse]: https://clickhouse.com/
+[glt]: https://github.com/y-scope/clp
+[clp-s]: https://docs.yscope.com/clp/main/user-guide/core-clp-s.html
+[webui]: https://benchmarks.yscope.com/clp/
+[elasticsearch]: https://www.elastic.co/downloads/elasticsearch
+[loki]: https://grafana.com/oss/loki/
+[mongodb]: https://www.mongodb.com/
+[methodology]: docs/methodology.md
+[new-issue]: https://github.com/y-scope/clp-bench/issues/new
+[adding-a-tool]: docs/adding-a-tool.md
+[splunk]: https://www.splunk.com/
+[ui]: ui

assets/dynamically-structured/clickhouse/Dockerfile

@@ -0,0 +1 @@
+FROM clickhouse/clickhouse-server:latest

assets/dynamically-structured/clickhouse/clear-cache.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+clickhouse-client --query "SYSTEM DROP UNCOMPRESSED CACHE"
+clickhouse-client --query "SYSTEM DROP MARK CACHE"
+sync
+echo 1 >/proc/sys/vm/drop_caches

assets/dynamically-structured/clickhouse/config.yaml

@@ -0,0 +1,29 @@
+system_metric:
+  enable: true
+  memory:
+    ingest_polling_interval: 5
+    run_query_benchmark_polling_interval: 5
+
+container_id: clickhouse-clp-bench
+assets_path: /home/assets
+datasets_path: /home/datasets/mongod.log
+hot_run_warm_up_times: 3
+related_processes:
+  - clickhouse-server
+  - clickhouse-client
+  - clickhouse-watchd
+queries:
+  - '"JSON_EXISTS(raw, ''$.attr.tickets'')"'
+  - '"JSONExtractInt(raw, ''id'') = 22419"'
+  - >
+    "JSON_VALUE(raw, '$.attr.message.msg') like 'log_release%' AND JSON_VALUE(raw, '$.attr.
+    message.session_name') = 'connection'"
+  - >
+    "JSON_VALUE(raw, '$.ctx') = 'initandlisten' AND (JSON_VALUE(raw, '$.attr.message.msg') 
+    like 'log_remove%' OR JSON_VALUE(raw, '$.msg') != 'WiredTiger message')"
+  - >
+    "JSON_VALUE(raw, '$.c') = 'WTWRTLOG' and JSONExtractInt(JSONExtractString(
+    JSONExtractString(raw, 'attr'), 'message'), 'ts_sec')  > 1679490000"
+  - >
+    "JSON_VALUE(raw, '$.ctx') = 'FlowControlRefresher' AND JSONExtractInt(JSONExtractString(
+    raw, 'attr'), 'numTrimmed') = 0 "

assets/dynamically-structured/clickhouse/container-name

@@ -0,0 +1 @@
+clickhouse-clp-bench

assets/dynamically-structured/clickhouse/docker-build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+container_name=$(cat "$script_dir/container-name")
+
+docker build \
+    --tag "$container_name" \
+    "$script_dir"

assets/dynamically-structured/clickhouse/docker-run.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "Error: Datasets path argument is missing."
+    echo "Usage: bash ./docker-run.sh <absolute_datasets_path>"
+    exit 1
+fi
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+container_name=$(cat "$script_dir/container-name")
+workdir=/home
+
+docker run \
+    --privileged \
+    -it \
+    --rm \
+    --workdir "$workdir" \
+    --network host \
+    --name "$container_name" \
+    --mount "type=bind,src=$script_dir,dst=/home/assets" \
+    --mount "type=bind,src=$1,dst=/home/datasets" \
+    "$container_name" \
+    bash -c "cd ${workdir} && /bin/bash -l"

assets/dynamically-structured/clickhouse/ingest.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -z "$1" ]; then
+    echo "Error: Datasets path argument is missing."
+    echo "Usage: bash ./ingest.sh <absolute_datasets_path_in_container>"
+    exit 1
+fi
+
+collection_name=clickhouse_clp_bench
+
+clickhouse-client \
+    --max_threads 1 \
+    --query "INSERT INTO ${collection_name} FROM INFILE '$1' FORMAT JSONAsString" \
+    >/dev/null 2>&1

assets/dynamically-structured/clickhouse/launch.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Start the ClickHouse server in daemon mode
+clickhouse-server --daemon
+
+# Wait until ClickHouse server is running
+while [ "$(clickhouse-client --query "SELECT 1" 2>/dev/null)" != "1" ]; do
+    sleep 1
+done

assets/dynamically-structured/clickhouse/measure-compressed-size.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+collection_name=clickhouse_clp_bench
+
+clickhouse-client \
+    --query "SELECT SUM(bytes) FROM system.parts WHERE active AND table = '${collection_name}'"

assets/dynamically-structured/clickhouse/measure-decompressed-size.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -z "$1" ]; then
+    echo "Error: Datasets path argument is missing."
+    echo "Usage: bash ./measure-decompressed-size.sh <absolute_datasets_path_in_container>"
+    exit 1
+fi
+
+du "$1" -bc | awk "END {print \$1}"

assets/dynamically-structured/clickhouse/methodology.md

@@ -0,0 +1,25 @@
+# ClickHouse methodology
+
+## Basics
+
+Version: [23.3.1.2823][download]
+
+## Setup
+
+We start the ClickHouse server in daemon mode.
+
+## Specifics
+
+To store JSON records, we use a 
+[single string field][jsonasstring] in ClickHouse, eliminating the need for preprocessing.
+
+For query benchmarking, we operate in [single-thread mode][max_threads] by setting
+`max_threads = 1`. Additionally, we configure the
+[minimum data volume required for direct I/O access][direct_io] to 1 byte
+(`min_bytes_to_use_direct_io = 1`) on the storage disk.
+
+
+[download]: https://hub.docker.com/layers/clickhouse/clickhouse-server/23.3.1.2823/images/sha256-b88fd8c71b64d3158751337557ff089ff7b0d1ebf81d9c4c7aa1f0b37a31ee64?context=explore
+[direct_io]: https://clickhouse.com/docs/en/operations/settings/settings#min_bytes_to_use_direct_io
+[jsonasstring]: https://clickhouse.com/docs/en/interfaces/formats#jsonasstring
+[max_threads]: https://clickhouse.com/docs/en/operations/settings/settings#max_threads

assets/dynamically-structured/clickhouse/reset.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+collection_name=clickhouse_clp_bench
+
+clickhouse-client \
+    --max_threads 1 \
+    --query "DROP TABLE IF EXISTS ${collection_name}" >/dev/null 2>&1
+clickhouse-client \
+    --max_threads 1 \
+    --query "CREATE TABLE ${collection_name}(raw String CODEC(ZSTD(3))) ENGINE = MergeTree ORDER \
+    BY tuple()" >/dev/null 2>&1

assets/dynamically-structured/clickhouse/results.json

@@ -0,0 +1,36 @@
+{
+    "target": "clickhouse",
+    "targetDisplayedName": "ClickHouse",
+    "displayedOrder": 3,
+    "isEnable": true,
+    "type": 2,
+    "ingestTime": 636409,
+    "compressedSize": 1050348093,
+    "avgIngestMem": 39008949248,
+    "metrics": [
+        {
+            "metric": 1,
+            "avgQueryMem": 707801088,
+            "queryTimes": [
+                189293, 
+                117725, 
+                199560, 
+                226136, 
+                221677, 
+                219761
+            ]
+        },
+        {
+            "metric": 2,
+            "avgQueryMem": 603047936,
+            "queryTimes": [
+                189381, 
+                115290, 
+                198320, 
+                225983, 
+                227425, 
+                224492
+            ]
+        }
+    ]
+}

assets/dynamically-structured/clickhouse/search.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -z "$1" ]; then
+    echo "Error: Query argument is missing."
+    echo "Usage: bash ./search.sh <query>"
+    exit 1
+fi
+
+collection_name=clickhouse_clp_bench
+
+clickhouse-client \
+    --max_threads 1 \
+    --query "SELECT * from ${collection_name} where $1 SETTINGS max_threads = 1, \
+    min_bytes_to_use_direct_io = 1" 2>/dev/null | wc -l

assets/dynamically-structured/clickhouse/terminate.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+/etc/init.d/clickhouse-server stop >/dev/null 2>&1

assets/dynamically-structured/clp-s/Dockerfile

@@ -0,0 +1,11 @@
+FROM ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    htop \
+    jq \
+    python3-venv \
+    rsync \
+    sqlite3 \
+    tmux \
+    vim

assets/dynamically-structured/clp-s/clear-cache.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+sync
+echo 1 >/proc/sys/vm/drop_caches

assets/dynamically-structured/clp-s/config.yaml

@@ -0,0 +1,19 @@
+system_metric:
+  enable: true
+  memory:
+    ingest_polling_interval: 5
+    run_query_benchmark_polling_interval: 5
+
+container_id: clp-clp-bench
+assets_path: /home/assets
+datasets_path: /home/datasets/mongod.log
+hot_run_warm_up_times: 3
+related_processes:
+  - /home/assets/clp-s
+queries:
+  - '"attr.tickets:*"'
+  - '"id: 22419"'
+  - '"attr.message.msg: log_release* AND attr.message.session_name: connection"'
+  - '''ctx: initandlisten AND (NOT msg: "WiredTigermessage" OR attr.message.msg: log_remove*)'''
+  - '"c: WTWRTLOG AND attr.message.ts_sec > 1679490000"'
+  - '"ctx: FlowControlRefresher AND attr.numTrimmed: 0"'

assets/dynamically-structured/clp-s/container-name

@@ -0,0 +1 @@
+clp-clp-bench

assets/dynamically-structured/clp-s/docker-build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+container_name=$(cat "$script_dir/container-name")
+
+docker build \
+    --tag "$container_name" \
+    "$script_dir"

assets/dynamically-structured/clp-s/docker-run.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "Error: Datasets path argument is missing."
+    echo "Usage: bash ./docker-run.sh <absolute_datasets_path>"
+    exit 1
+fi
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+container_name=$(cat "$script_dir/container-name")
+workdir=/home
+
+docker run \
+    --privileged \
+    -it \
+    --rm \
+    --workdir "$workdir" \
+    --network host \
+    --name "$container_name" \
+    --mount "type=bind,src=$script_dir,dst=/home/assets" \
+    --mount "type=bind,src=$1,dst=/home/datasets" \
+    "$container_name" \
+    bash -c "cd ${workdir} && /bin/bash -l"

assets/dynamically-structured/clp-s/ingest.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -z "$1" ]; then
+    echo "Error: Datasets path argument is missing."
+    echo "Usage: bash ./ingest.sh <absolute_datasets_path_in_container>"
+    exit 1
+fi
+
+clp_s_binary=/home/assets/clp-s
+data_path=/home/archives
+
+"${clp_s_binary}" c --timestamp-key "t.\$date" --target-encoded-size 268435456 "$data_path" "$1"

assets/dynamically-structured/clp-s/launch.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+data_path=/home/archives
+
+mkdir -p "$data_path"

assets/dynamically-structured/clp-s/measure-compressed-size.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+data_path=/home/archives
+du "$data_path" -bc | awk "END {print \$1}"

assets/dynamically-structured/clp-s/measure-decompressed-size.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+if [ -z "$1" ]; then
+    echo "Error: Datasets path argument is missing."
+    echo "Usage: bash ./measure-decompressed-size.sh <absolute_datasets_path_in_container>"
+    exit 1
+fi
+
+du "$1" -bc | awk "END {print \$1}"