Merge branch 'master' of github.com:cucapra/fuse-benchmarks-sdaccel

_common/experiment-common/extract.mk

@@ -0,0 +1,13 @@
+.PHONY: graphs clean
+
+graphs: summary.csv graphs.py
+	./graphs.py
+
+summary.csv: results.json key.json
+	../../_scripts/summarize.py -k key.json results.json -c routed_util
+
+results.json: jobs.txt
+	../../_scripts/extract.py ./ -c dse_template routed_util runtime
+
+clean:
+	rm -rf summary.csv results.json *.pdf *.png raw

benchmarking-helpers/benchmarking/plotting/plots.py

@@ -0,0 +1,83 @@
+import pandas as pd
+import seaborn as sns
+from matplotlib import pyplot as plt
+
+def make_absolute_plots(
+        df,
+        x_key,
+        y_keys,
+        group_by,
+        x_label,
+        fig_prefix,
+        group_labels = None,
+        group_markers = None,
+        group_runtime_supress = None):
+    """Make and save graphs plotting each value in `y_keys` against `x_key`.
+    Plotting each set of points separately by first running group_by on each
+    row.
+    """
+
+    sns.set()
+
+    figs = []
+
+    groups = df.groupby(group_by, axis=0)
+
+    # Configuration for subplot
+    pal = sns.color_palette('colorblind', len(groups))
+    sns.set_context('paper')
+
+    for key in y_keys:
+        fig = plt.figure()
+
+        for idx, group in groups:
+            if key == 'runtime_avg' and group_runtime_supress and group_runtime_supress[idx]:
+                continue
+            plt.scatter(x=x_key, y=key, data=group, color=pal[idx],
+                        label=group_labels[idx] if group_labels else None,
+                        marker=group_markers[idx] if group_markers else None,
+                        s=100)
+            plt.vlines(x=x_key, ymax=key, ymin=df[key].min(), data=group, colors=pal[idx])
+
+        if key == 'runtime_avg':
+            y_label = 'Runtime (ms)'
+        else:
+            y_label = key.replace('_', ' ').capitalize()
+
+        plt.ylabel(y_label, fontsize=18)
+        plt.xlabel(x_label, fontsize=18)
+        plt.xticks(fontsize=16)
+        plt.yticks(fontsize=16)
+
+        if group_labels:
+            plt.legend(prop={'size': 16})
+
+        fig.tight_layout()
+        fig.savefig('{}-{}.pdf'.format(fig_prefix, key.replace('_', '-')), dpi=100)
+
+        figs.append(fig)
+
+    return figs
+
+def make_sec2_plot(df, x_key, x_label, fig_prefix, factor=16):
+    y_keys = [
+        'runtime_avg',
+        'lut_used',
+    ]
+
+    def group(idx):
+        row = df.iloc[idx]
+        if row.status == 'error':
+            return 2
+        return int(factor % df.iloc[idx][x_key] == 0)
+
+    make_absolute_plots(
+        df,
+        x_key,
+        y_keys,
+        group_by = group,
+        group_labels = ['Unpredictable points', 'Predictable points', 'Incorrect hardware'],
+        group_markers = ['o', 'v', 'x'],
+        group_runtime_supress = [ False, False, True ],
+        x_label = x_label,
+        fig_prefix = fig_prefix)

benchmarking-helpers/benchmarking/summary/summary_json.py

@@ -0,0 +1,26 @@
+import os
+import sys
+import logging
+import json
+
+from benchmarking import common
+from . import collect_configs
+
+def job_json(batch_dir, job_id, collects):
+    """Generate json summary for a single job.
+    """
+
+    return {}
+
+
+def batch_json(batch_dir, static_collects, dyn_collects):
+    """Generate a combined JSON summary for the batch.
+    """
+
+    job_file = os.path.join(batch_dir, common.JOBS_FILE)
+    if not os.path.isfile(job_file):
+        raise Exception("{} is not a file".format(job_file))
+    with open(job_file, "r") as jobs:
+        job_ids = jobs.read().strip().split('\n')
+
+    failed_jobs = set()

experiments/Makefile

@@ -0,0 +1,4 @@
+EXPS := gemm-no-part-unroll gemm-const-unroll-part gemm-const-partition-unroll gemm-const-len-partition-and-unroll gemm-blocked-dse spatial-sweep
+
+paper-experiments:
+	for exp in $(EXPS); do $(MAKE) -C $$exp -f extract.mk; done

experiments/dahlia-dse/machsuite-md-grid/sda.mk

@@ -0,0 +1,107 @@
+# Include sdaccel utilities
+COMMON_REPO =  .
+ABS_COMMON_REPO = $(shell readlink -n $(COMMON_REPO))
+
+include ./utils.mk
+
+# Include sdaccel libraries
+include libs/opencl/opencl.mk
+include libs/xcl2/xcl2.mk
+
+# OCL compilation options.
+DEVICE    := xilinx_vcu1525_dynamic
+TARGET_FREQ  := 250 # clock is set using frequency, using 4ns for testing
+
+# Check if we have a valid MODE set
+ifeq (,$(filter $(MODE),sw_emu hw_emu hw estimate))
+$(error "Invalid MODE: $(MODE). Must be one of: hw, hw_emu, sw_emu, estimate")
+endif
+
+# The ordinary (software) C++ compiler.
+CXX   := $(XILINX_SDX)/bin/xcpp
+CXXFLAGS  := $(xcl2_CXXFLAGS)
+LDFLAGS   := $(xcl2_LDFLAGS)
+CXXFLAGS  += $(opencl_CXXFLAGS)
+LDFLAGS   += $(opencl_LDFLAGS)
+
+HOST_SRCS += $(xcl2_SRCS)
+
+# Add helper library
+HOST_SRCS += ${COMMON_REPO}/libs/helpers/helpers.cpp
+CXXFLAGS += -I${COMMON_REPO}/libs/helpers
+
+# Compiler flags
+CXXFLAGS  += -Wall -O0 -g -std=c++14
+
+# Host compiler global settings
+CXXFLAGS  += -fmessage-length=0
+LDFLAGS   += -lrt -lstdc++
+
+# The OCL compiler.
+XOCC  := $(XILINX_SDX)/bin/xocc
+# If in estimation mode, add the required flags
+ifeq ($(MODE),estimate)
+XOCCFLAGS += --report_level estimate
+override MODE := sw_emu
+endif
+
+XOCCFLAGS += -t $(MODE) --platform $(DEVICE) --save-temps --kernel_frequency=$(TARGET_FREQ) -DSDACCEL --xp "param:compiler.enableAutoPipelining=false"
+## How to add directives, can you add directives? other options?
+
+
+# Binaries
+BINARY_CONTAINERS     += $(XCLBIN)/$(KERNEL).$(MODE).$(DSA).xclbin
+BINARY_CONTAINER_OBJS += $(XCLBIN)/$(KERNEL).$(MODE).$(DSA).xo
+
+# emconfig stuff
+XCLBIN       := ./xclbin
+DSA          := $(call device2sandsa, $(DEVICE))
+EMCONFIG_DIR = $(XCLBIN)/$(DSA)
+CP           = cp -rf
+EXECUTABLE   := exe
+
+check: all
+ifeq ($(MODE),$(filter $(MODE),sw_emu hw_emu))
+	$(CP) $(EMCONFIG_DIR)/emconfig.json .
+endif
+
+.PHONY: all
+all: $(EXECUTABLE) $(BINARY_CONTAINERS) emconfig
+
+# emconfig
+.PHONY: emconfig
+emconfig:$(EMCONFIG_DIR)/emconfig.json
+$(EMCONFIG_DIR)/emconfig.json:
+	emconfigutil --platform $(DEVICE) --od $(EMCONFIG_DIR)
+
+# Building kernel
+$(XCLBIN)/$(KERNEL).$(MODE).$(DSA).xo: $(HW_SRCS)
+	mkdir -p $(XCLBIN)
+	$(XOCC) $(XOCCFLAGS) -c -k $(KERNEL) -I'$(<D)' -o'$@' '$<'
+$(XCLBIN)/$(KERNEL).$(MODE).$(DSA).xclbin: $(BINARY_CONTAINER_OBJS)
+	mkdir -p $(XCLBIN)
+	$(XOCC) $(XOCCFLAGS) -l $(LDCLFLAGS) --nk $(KERNEL):1 -o'$@' $(+)
+
+# Building Host
+$(EXECUTABLE): $(HOST_SRCS) $(HOST_HDRS)
+	mkdir -p $(XCLBIN)
+	$(CXX) $(CXXFLAGS) $(HOST_SRCS) $(HOST_HDRS) -o '$@' $(LDFLAGS)
+
+# Check for host
+checkhost: all
+	sudo sh -c 'source /opt/xilinx/xrt/setup.sh ; ./$(EXECUTABLE)'
+
+# Cleaning stuff
+.PHONY: clean cleanall
+clean:
+	-$(RMDIR) $(EXECUTABLE) $(XCLBIN)/{*sw_emu*,*hw_emu*}
+	-$(RMDIR) sdaccel_* TempConfig system_estimate.xtxt *.rpt
+	-$(RMDIR) ./*.ll _xocc_* .Xil emconfig.json dltmp* xmltmp* *.log *.jou *.wcfg *.wdb
+
+cleanall: clean
+	-$(RMDIR) $(XCLBIN)
+	-$(RMDIR) ./_x
+
+
+# Print value of a Makefile variable.
+print-%:  ; @echo $* = $($*)

experiments/gemm-const-partition-unroll/graphs.py

@@ -10,4 +10,5 @@
         df=df,
         x_key=x_key,
         x_label = 'Unrolling factor (partitioning = 8)',
+        factor = 8,
         fig_prefix = 'const-partition-unroll')

experiments/gemm-no-part-unroll/graphs.py

@@ -6,8 +6,11 @@
 if __name__ == '__main__':
     x_key = 'unroll'
     df = pd.read_csv('./summary.csv').sort_values(by=x_key).reset_index()
-    plots.make_sec2_plot(
+    plots.make_absolute_plots(
         df,
         x_key = x_key,
+        y_keys = ['runtime_avg', 'lut_used'],
+        group_by = lambda x: 0,
         x_label = 'Unrolling factor (no partitioning)',
-        fig_prefix = 'no-partition-unrolling')
+        fig_prefix = 'no-partition-unrolling',
+        )

experiments/spatial-sweep/extract.mk

@@ -1,7 +1,4 @@
 .PHONY: phony
 
-absolute.png: summary.csv
-	jupyter nbconvert --execute analysis.ipynb
-
-summary.csv: phony
-	./summary-rpts.py
+phony:
+	./graphs.py

experiments/spatial-sweep/graphs.py

@@ -7,9 +7,11 @@ def make_plots():
     x_key = 'unroll'
     y_keys = [
         'lut_used',
+        'reg_used',
+        'dsp_used',
+        'bram_tile_used'
     ]
     df = pd.read_csv('./summary.csv').sort_values(by=x_key).reset_index()
-
     plots.make_absolute_plots(
         df,
         x_key,