Netperf benches

ambient-tests/.gitignore

@@ -0,0 +1,8 @@
+data
+graphs
+results
+secrets
+.venv
+*.pcap
+dumps
+__pycache__

ambient-tests/Makefile

@@ -0,0 +1,15 @@
+.PHONY: docker-build run-netperf
+
+run-netperf:
+	./scripts/netperf/run.sh
+	./scripts/netperf/gen-csv.sh
+	python ./scripts/netperf/graphs.py
+
+run-fortio:
+	./scripts/fortio/run.sh
+	./scripts/fortio/gen-csv.sh
+	python ./scripts/fortio/graphs.py
+
+docker-build:
+	make build push -C netperf
+

ambient-tests/README.md

@@ -0,0 +1,53 @@
+# Performance Tests for Ztunnel
+
+Performance tests for Istio Ambient, Istio Ambient + Waypoint, Istio Sidecar.
+Uses [netperf](https://github.com/HewlettPackard/netperf) and [fortio](https://github.com/fortio/fortio).
+
+## Setup
+
+To run, first set up a cluster with two user nodes: one with the `role=server` label and one with the `role=client` label.
+You can recreate this in a kind cluster using `yaml/cluster.yaml`.
+If you use a kind cluster, will either have to connect it to a local registry, or upload the images using `make push-local` in the `docker` directory.
+You will also have to modify the `imagePullPolicy` fields of the pods in `yaml/deploy.yaml` to `Never`.
+
+Note that if you are using AKS to set up your cluster, make sure that you use Azure CNI as your network plugin and **DO NOT** use a network policy.
+Also, make sure to attach to container registry to your cluster/that your container registry is accessible from your cluster.
+This ensures that client and server pods get deployed in different servers.
+
+Next, go into `docker/Makefile` and change the value of `CR` to your container registry and the image names in `yaml/deploy.yaml` accordingly.
+
+You will also need a Python 3 with `matplotlib`, `pandas`, and `python-dotenv`.
+Also, make sure that `python -V` is some version of Python 3.
+An easy way to get this on Ubuntu is running
+
+```bash
+sudo apt install python-is-python3
+sudo apt install python3-pip
+pip install matplotlib pandas python-dotenv
+```
+
+Install Istio Ambient.
+I don't automate this process because there are many installation methods and you might be testing a custom build.
+To install the latest release of Istio Ambient [install `istioctl`](https://istio.io/latest/docs/setup/getting-started/#download) and run
+
+```bash
+istioclt install --set profile=ambient
+```
+
+## Building
+
+To build the necessary containers, inside `docker/`, run
+
+```bash
+make build
+make push-cr
+# or `push-local` if running in a kind cluster.
+```
+
+## Running Benchmarks
+
+Now, update `scripts/config.sh` as desired or keep the default values, and run `./scripts/setup.sh` from the root of the repository to deploy the pods.
+Once the deployments are complete, run the tests with `make run-netperf run-fortio`.
+This will create graphs in the `graphs/` directory and put intermediate files in `results/` by default.
+
+For more configuration options, see `scripts/run.sh`.

ambient-tests/docker/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

ambient-tests/docker/Dockerfile

@@ -0,0 +1,37 @@
+# This is the only version that is available
+# hadolint ignore=DL3007
+FROM networkstatic/netserver:latest
+RUN apt-get update -y && apt-get upgrade -y       \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        netcat    \
+        net-tools \
+        tcpdump   \
+        netperf   \
+        less      \
+        ncat      \
+        termshark \
+        python3   \
+        curl \
+        && rm -rf /var/lib/apt/lists/*
+
+RUN curl -OL https://github.com/fortio/fortio/releases/download/v1.17.0/fortio_1.17.0_amd64.deb \
+    && dpkg -i fortio_1.17.0_amd64.deb
+
+# tcp echo port
+EXPOSE 6789
+# data port for netperf
+EXPOSE 35000
+# config port netperf 
+EXPOSE 12865
+# python http server
+EXPOSE 8000
+# fortio server
+EXPOSE 8080
+EXPOSE 8078
+EXPOSE 8079
+
+WORKDIR "/ambient-performance"
+
+COPY . .
+ENTRYPOINT ["./server-and-sleep.sh"]

ambient-tests/docker/Makefile

@@ -0,0 +1,21 @@
+CR=stjinxuan.azurecr.io
+NAME=ambient-performance
+CR_NAME=$(CR)/$(NAME)
+
+.PHONY: build push run push-cr push-local
+
+default: build
+
+build:
+	docker build . -t $(CR_NAME) -t $(NAME)
+
+push-cr:
+	docker push $(CR_NAME)
+
+push-local:
+	kind load docker-image $(CR_NAME)
+
+push: push-cr
+
+run:
+	docker run --rm --name $(NAME) $(NAME) -P

ambient-tests/docker/README.md

@@ -0,0 +1,4 @@
+# Netperf
+
+Lightweight Docker image running netperf and an echo server.
+Also includes various network debugging utilities.

ambient-tests/docker/server-and-sleep.sh

@@ -0,0 +1,23 @@
+#! /bin/bash
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+netserver "$@"
+ncat -e /bin/cat -k -l 6789 &
+python ./tcp_ping/server.py &
+fortio server &
+python3 -m http.server &
+sleep 365d

ambient-tests/requirements.txt

@@ -0,0 +1,3 @@
+matplotlib
+pandas
+python-dotenv

ambient-tests/scripts/config.sh

@@ -0,0 +1,72 @@
+# shellcheck shell=bash
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# server-client config
+export YAML_PATH=./yaml/deploy.yaml
+
+# name of the deployments
+export BENCHMARK_SERVER=server
+export BENCHMARK_CLIENT=client
+
+# where to store primary and intermediate results
+export NETPERF_RESULTS=results/netperf
+export FORTIO_RESULTS=results/fortio
+
+# name of the namespaces for each mesh setup
+export NS_NO_MESH=no-mesh
+export NS_SIDECAR=sidecar
+export NS_AMBIENT=ambient
+export NS_WAYPOINT=waypoint  # ambient w/ waypoint proxy
+# name of service account for server. Used for waypoint configuration.
+# This is also hardcoded in deployment configuration
+export SA_SERVER=server-sa
+
+# Separator for tests runs. Doesn't really matter. Just set to something weird
+export TEST_RUN_SEPARATOR="~~~~~~~~~~~~~~~~"
+# How many runs of each test
+export N_RUNS=2
+
+# Extra arguments for TCP_RR and TCP_CRR tests.
+# These are necessary because by default *RR tests send only one byte.
+# However, Envoy proxies won't create a connection until more bytes are sent.
+# This also means that reverse tests _DO NOT WORK_.
+export NETPERF_RR_ARGS="-r 100"
+
+# -P Have the data connection listen on port 35000 on the server.
+# -k Output all fields in key=value from.
+#    We will pick and choose later.
+export NETPERF_TEST_ARGS="-P ,35000 -k all"
+
+# -P toggles the tests banner.
+#    This is very confusing because it has nothing to do with ports.
+# -j to measure latency
+export NETPERF_GLOBAL_ARGS="-P 0 -j"
+
+# Args for serial tests
+# -qps          As many queries per second as possible
+# -c            Number of sending threads
+# -payload-file POST request with Makefile as data.
+#
+# This configuration seems to trigger the Nagle's + delayed ack interaction.
+export FORTIO_SERIAL_HTTP_ARGS="-qps -1 -c 1 -payload-size 300"
+
+# Arguments for prallel http tests
+# 3 sending threads should be enough(?)
+export FORTIO_PARALLEL_HTTP_ARGS="-qps -1 -c 3"
+
+# where to output the graphs
+export NETPERF_GRAPHS="graphs/netperf"
+export FORTIO_GRAPHS="graphs/fortio"

ambient-tests/scripts/fortio/gen-csv.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+# shellcheck disable=SC1091
+source scripts/config.sh
+
+# TODO: names?
+python scripts/fortio/results_to_csv.py < "$FORTIO_RESULTS/serial"   > "$FORTIO_RESULTS/serial.csv"
+python scripts/fortio/results_to_csv.py < "$FORTIO_RESULTS/parallel" > "$FORTIO_RESULTS/parallel.csv"
+

ambient-tests/scripts/fortio/graphs.py

@@ -0,0 +1,39 @@
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from scripts.lib import latency_graph
+import os
+import sys
+
+import dotenv
+import matplotlib.pyplot as plt
+import pandas as pd
+
+sys.path.append(".")  # kinda hacky, but easy
+
+
+# load data
+# TCP_STREAM figure
+dotenv.load_dotenv("./scripts/config.sh")
+FORTIO_RESULTS = os.environ["FORTIO_RESULTS"]
+FORTIO_GRAPHS = os.environ["FORTIO_GRAPHS"]
+
+
+if __name__ == "__main__":
+    os.makedirs(FORTIO_GRAPHS, exist_ok=True)
+    serial = latency_graph(f"{FORTIO_RESULTS}/serial.csv", "HTTP Echo (serial)")
+    serial.savefig(f"{FORTIO_GRAPHS}/serial.png")
+
+    serial = latency_graph(f"{FORTIO_RESULTS}/parallel.csv", "HTTP Echo (parallel)")
+    serial.savefig(f"{FORTIO_GRAPHS}/parallel.png")

ambient-tests/scripts/fortio/results_to_csv.py

@@ -0,0 +1,70 @@
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import csv
+import dotenv
+import os
+import json
+import sys
+
+dotenv.load_dotenv("./scripts/config.sh")
+
+TEST_RUN_SEPARATOR = os.environ["TEST_RUN_SEPARATOR"]
+
+# Format is
+# <source mesh>:<dest mesh>
+# <JSON output>
+# TEST_RUN_SEPARATOR
+
+raw = sys.stdin.read()
+raw = raw.replace("\r", "").strip()
+runs = raw.split(TEST_RUN_SEPARATOR)
+if not runs[-1]:
+    runs = runs[:-1]
+
+writer = csv.DictWriter(
+    sys.stdout,
+    fieldnames=[
+        "NAMESPACES",
+        "THROUGHPUT",
+        "THROUGHPUT_UNITS",
+        "P50_LATENCY",
+        "P90_LATENCY",
+        "P99_LATENCY",
+        "SOCKET_COUNT",
+    ],
+    extrasaction="ignore",
+)
+
+writer.writeheader()
+
+for run in runs:
+    row = dict()
+    run = run.strip()
+    run = run.split("\n")
+
+    # Hardcode field mappings
+    row["NAMESPACES"] = run[0]
+    run = "".join(run[1:])
+    run = json.loads(run)
+    row["THROUGHPUT"] = run["ActualQPS"]
+    row["THROUGHPUT_UNITS"] = "Trans/s"
+    for p in run["DurationHistogram"]["Percentiles"]:
+        # cursed fstring
+        row[f"P{p['Percentile']}_LATENCY"] = (
+            p["Value"] * 10**6
+        )  # seconds to microseconds
+    row["SOCKET_COUNT"] = run["SocketCount"]
+
+    writer.writerow(row)