Add benchmarking scripts

examples/benchmark.rs

@@ -1,23 +1,26 @@
+use std::env;
+use std::str::FromStr;
+
 use phastft::fft_dif;
 use utilities::gen_random_signal;
 
-fn bm_fft(num_qubits: usize) {
-    for i in 4..num_qubits {
-        println!("run PhastFT with {i} qubits");
-        let n = 1 << i;
-
-        let mut reals = vec![0.0; n];
-        let mut imags = vec![0.0; n];
-
-        gen_random_signal(&mut reals, &mut imags);
+fn benchmark_fft(n: usize) {
+    let big_n = 1 << n;
+    let mut reals = vec![0.0; big_n];
+    let mut imags = vec![0.0; big_n];
+    gen_random_signal(&mut reals, &mut imags);
 
-        let now = std::time::Instant::now();
-        fft_dif(&mut reals, &mut imags);
-        let elapsed = now.elapsed().as_micros();
-        println!("time elapsed: {elapsed} us\n----------------------------");
-    }
+    let now = std::time::Instant::now();
+    fft_dif(&mut reals, &mut imags);
+    let elapsed = now.elapsed().as_micros();
+    println!("{elapsed}");
 }
 
 fn main() {
-    bm_fft(31);
+    let args: Vec<String> = env::args().collect();
+    assert_eq!(args.len(), 2, "Usage {} <n>", args[0]);
+
+    let n = usize::from_str(&args[1]).unwrap();
+
+    benchmark_fft(n);
 }

examples/rustfft.rs

@@ -1,34 +1,40 @@
+use std::env;
+use std::str::FromStr;
+
 use utilities::{
     gen_random_signal,
     rustfft::{num_complex::Complex64, FftPlanner},
 };
 
-fn main() {
-    let big_n = 31;
+fn benchmark_rustfft(n: usize) {
+    let big_n = 1 << n;
 
-    for i in 4..big_n {
-        println!("run RustFFT with {i} qubits");
-        let n = 1 << i;
+    let mut reals = vec![0.0; big_n];
+    let mut imags = vec![0.0; big_n];
 
-        let mut reals = vec![0.0; n];
-        let mut imags = vec![0.0; n];
+    gen_random_signal(&mut reals, &mut imags);
+    let mut signal = vec![Complex64::default(); big_n];
+    reals
+        .drain(..)
+        .zip(imags.drain(..))
+        .zip(signal.iter_mut())
+        .for_each(|((re, im), z)| {
+            z.re = re;
+            z.im = im;
+        });
 
-        gen_random_signal(&mut reals, &mut imags);
-        let mut signal = vec![Complex64::default(); n];
-        reals
-            .drain(..)
-            .zip(imags.drain(..))
-            .zip(signal.iter_mut())
-            .for_each(|((re, im), z)| {
-                z.re = re;
-                z.im = im;
-            });
+    let now = std::time::Instant::now();
+    let mut planner = FftPlanner::new();
+    let fft = planner.plan_fft_forward(signal.len());
+    fft.process(&mut signal);
+    let elapsed = now.elapsed().as_micros();
+    println!("{elapsed}");
+}
+
+fn main() {
+    let args: Vec<String> = env::args().collect();
+    assert_eq!(args.len(), 2, "Usage {} <n>", args[0]);
 
-        let now = std::time::Instant::now();
-        let mut planner = FftPlanner::new();
-        let fft = planner.plan_fft_forward(signal.len());
-        fft.process(&mut signal);
-        let elapsed = now.elapsed().as_micros();
-        println!("time elapsed: {elapsed} us\n----------------------------");
-    }
+    let n = usize::from_str(&args[1]).unwrap();
+    benchmark_rustfft(n);
 }

scripts/benchmark.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+if [[ "$#" -ne 3 ]]
+then
+  echo "Usage: $0 <n-lower-bound> <n-upper-bound> <iters>"
+  exit 1
+fi
+
+start=$1
+end=$2
+iters=$3
+
+OUTPUT_DIR=benchmark-data.$(date +"%Y.%m.%d.%H-%M-%S")
+mkdir -p "$OUTPUT_DIR"/fftw3 && mkdir "$OUTPUT_DIR"/rustfft && mkdir "$OUTPUT_DIR"/phastft
+
+benchmark_phastft() {
+    cargo clean && cargo build --release --examples
+
+    for n in $(seq "$start" "$end"); do
+        echo "running PhastFT benchmark for N = 2^${n}" && \
+        for _ in $(seq 1 "$iters"); do
+            ../target/release/examples/benchmark "${n}" >> "${OUTPUT_DIR}"/phastft/size_"${n}"
+        done
+    done
+}
+
+benchmark_rustfft() {
+    cargo clean && cargo build --release --examples
+
+    for n in $(seq "$start" "$end"); do
+        echo "running RustFFT benchmark for N = 2^${n}" && \
+        for _ in $(seq 1 "$iters"); do
+            ../target/release/examples/rustfft "${n}" >> "${OUTPUT_DIR}"/rustfft/size_"${n}"
+        done
+    done
+}
+
+benchmark_rustfft
+benchmark_phastft

scripts/benchmark_plots.py

@@ -0,0 +1,69 @@
+"""
+Plot benchmark results for FFTW3, RustFFT, and PhastFT
+"""
+from collections import defaultdict
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+plt.style.use("fivethirtyeight")
+
+def read_file(filepath: str) -> list[int]:
+    y = []
+
+    with open(filepath) as f:
+        for line in f:
+            line = line.strip()
+            y.append(int(line))
+
+    return y
+
+
+def get_figure_of_interest(vals: list[int]) -> float:
+    return np.mean(vals)
+
+
+def build_and_clean_data(root_benchmark_dir: str, *names) -> defaultdict[str, list]:
+    libs = ("rustfft", "phastft")
+    n_range = range(4, 27)
+
+    data = defaultdict(list)
+
+    for lib in libs:
+        for n in n_range:
+            y = read_file(f"{root_benchmark_dir}/{lib}/size_{n}")
+            y_k = get_figure_of_interest(y)
+            data[lib].append(y_k)
+
+    return data
+
+def plot_lines(data: defaultdict[str, list]) -> None:
+    index = list(range(4, 27))
+    plt.figure()
+
+    print(len(data["phastft"]))
+
+    df = pd.DataFrame(
+        {
+            "PhastFT": data["phastft"],
+            "RustFFT": data["rustfft"],
+        },
+        index=index,
+    )
+
+    df.plot(kind='bar', linewidth=3, rot=0)
+    plt.xticks(fontsize=8)
+    plt.xlabel("size")
+    plt.ylabel("time (us)")
+    plt.yscale("log")
+    plt.show()
+    # plt.tight_layout(pad=0.0)
+    # plt.savefig("benchmarks_bar_plot.png", dpi=600)
+
+
+if __name__ == "__main__":
+    # y = read_file("benchmark-data.2024.02.02.11-02-07/phastft/size_16")
+    data = build_and_clean_data("benchmark-data.2024.02.02.11-27-33")
+    print(data)
+    # plot_lines(data)