Support Event-based Scheduler in Holoscan Flow Benchmarking (#446)

* Support Event-based Scheduler in Holoscan Flow Benchmarking

Signed-off-by: sohams <sohams@nvidia.com>

* lint issue fixed

Signed-off-by: sohams <sohams@nvidia.com>

---------

Signed-off-by: sohams <sohams@nvidia.com>

Author: sohamm17

benchmarks/holoscan_flow_benchmarking/benchmark.hpp

@@ -77,6 +77,15 @@ class BenchmarkedApplication : public holoscan::Application {
       scheduler->add_arg(holoscan::Arg("stop_on_deadlock", true));
       scheduler->add_arg(holoscan::Arg("check_recession_period_ms", (double)0));
       scheduler->add_arg(holoscan::Arg("max_duration_ms", (int64_t)100000));
+    } else if (scheduler_str && std::string(scheduler_str) == "eventbased") {
+      holoscan::Fragment::scheduler(
+          holoscan::Fragment::make_scheduler<holoscan::EventBasedScheduler>(
+              "event-based-scheduler"));
+      auto scheduler = holoscan::Fragment::scheduler();
+
+      const char* num_threads_str = std::getenv("HOLOSCAN_EVENTBASED_WORKER_THREADS");
+      if (num_threads_str)
+        scheduler->add_arg(holoscan::Arg("worker_thread_number", std::stoi(num_threads_str)));
     } else {
       holoscan::Fragment::scheduler(
           holoscan::Fragment::make_scheduler<holoscan::GreedyScheduler>("greedy-scheduler"));

benchmarks/holoscan_flow_benchmarking/benchmark.py

@@ -48,8 +48,8 @@ def monitor_gpu(gpu_uuids, filename):
             sum(device.gpu_utilization() for device in devices) / len(devices)
         )
         time.sleep(2)
-        # write average gpu utilization to a file and a new line
 
+    # write average gpu utilization to a file and a new line
     with open(filename, "w") as f:
         # discard first 2 and last 2 values
         average_text = ",".join(map(str, average_gpu_utilizations[2:-2]))
@@ -87,7 +87,7 @@ def main():
     requiredArgument.add_argument(
         "--sched",
         nargs="+",
-        choices=["greedy", "multithread"],
+        choices=["greedy", "multithread", "eventbased"],
         required=True,
         help="scheduler(s) to use",
     )
@@ -171,7 +171,7 @@ def main():
         "--num_worker_threads",
         type=int,
         default=1,
-        help="number of worker threads for multithread scheduler (default: 1)",
+        help="number of worker threads for multithread or eventbased scheduler (default: 1)",
         required=False,
     )
 
@@ -181,8 +181,14 @@ def main():
 
     args = parser.parse_args()
 
-    if "multithread" not in args.sched and args.num_worker_threads != 1:
-        print("Warning: num_worker_threads is ignored as multithread scheduler is not used")
+    if (
+        "multithread" not in args.sched
+        and "eventbased" not in args.sched
+        and args.num_worker_threads != 1
+    ):
+        print(
+            "Warning: num_worker_threads is ignored as multithread or eventbased scheduler is not used"
+        )
 
     log_directory = None
     if args.log_directory is None:
@@ -200,10 +206,6 @@ def main():
             )
             os.mkdir(os.path.abspath(log_directory))
 
-    # if args.not_holohub or args.binary_path is not None:
-    #     print ("Currently non-HoloHub applications are not supported")
-    #     sys.exit(1)
-
     env = os.environ.copy()
     if args.gpu != "all":
         env["CUDA_VISIBLE_DEVICES"] = args.gpu
@@ -222,12 +224,16 @@ def main():
         if scheduler == "multithread":
             env["HOLOSCAN_SCHEDULER"] = scheduler
             env["HOLOSCAN_MULTITHREAD_WORKER_THREADS"] = str(args.num_worker_threads)
+        elif scheduler == "eventbased":
+            env["HOLOSCAN_SCHEDULER"] = scheduler
+            env["HOLOSCAN_EVENTBASED_WORKER_THREADS"] = str(args.num_worker_threads)
         elif scheduler != "greedy":
             print("Unsupported scheduler ", scheduler)
             sys.exit(1)
         # No need to set the scheduler for greedy scheduler
 
         for i in range(1, args.runs + 1):
+            print(f"Run {i} started for {scheduler} scheduler.")
             instance_threads = []
             if args.monitor_gpu:
                 gpu_utilization_logfile_name = (

benchmarks/holoscan_flow_benchmarking/benchmarked_application.py

@@ -2,7 +2,7 @@
 
 from holoscan.conditions import CountCondition
 from holoscan.core import Application
-from holoscan.schedulers import GreedyScheduler, MultiThreadScheduler
+from holoscan.schedulers import EventBasedScheduler, GreedyScheduler, MultiThreadScheduler
 
 
 class BenchmarkedApplication(Application):
@@ -43,6 +43,13 @@ def run(self):
                 check_recession_period_ms=0,
                 max_duration_ms=100000,
             )
+        elif scheduler_str and scheduler_str == "multithread":
+            num_threads = os.environ.get("HOLOSCAN_EVENTBASED_WORKER_THREADS", None)
+            scheduler = EventBasedScheduler(
+                self,
+                name="event-based-scheduler",
+                worker_thread_number=int(num_threads) if num_threads is not None else 1,
+            )
         else:
             scheduler = GreedyScheduler(self, name="greedy-scheduler")
         self.scheduler(scheduler)

benchmarks/holoscan_flow_benchmarking/flow_benchmarking_tutorial.md

@@ -319,23 +319,28 @@ your metrics options> -g endoscopy_outputs_<GPU Name>/logger* "GPU1" -g endoscop
 It is also possible to compare different schedulers using Holoscan flow
 benchmarking. The following commands can be used to evaluate performance
 of the endoscopy tool tracking sample application under the [Greedy
-scheduler](https://docs.nvidia.com/holoscan/sdk-user-guide/components/schedulers.html#greedy-scheduler)
-and the [Multithread scheduler](https://docs.nvidia.com/holoscan/sdk-user-guide/components/schedulers.html#multithreadscheduler):
+scheduler](https://docs.nvidia.com/holoscan/sdk-user-guide/components/schedulers.html#greedy-scheduler),
+the [Multithread
+scheduler](https://docs.nvidia.com/holoscan/sdk-user-guide/components/schedulers.html#multithreadscheduler)
+and the [Event-based Scheduler](https://docs.nvidia.com/holoscan/sdk-user-guide/components/schedulers.html#event-based-scheduler):
 
 ```
 $ python benchmarks/holoscan_flow_benchmarking/benchmark.py -a
 endoscopy_tool_tracking -r 10 -i 1 -m 1000 --sched greedy -d endoscopy_greedy_outputs
 
 $ python benchmarks/holoscan_flow_benchmarking/benchmark.py -a
-endoscopy_tool_tracking -r 10 -i 1 -m 1000 --sched multithread -w 5 -d ndoscopy_multithread_outputs
+endoscopy_tool_tracking -r 10 -i 1 -m 1000 --sched multithread -w 5 -d endoscopy_multithread_outputs
+
+$ python benchmarks/holoscan_flow_benchmarking/benchmark.py -a
+endoscopy_tool_tracking -r 10 -i 1 -m 1000 --sched eventbased -w 5 -d endoscopy_eventbased_outputs
 ```
 
 Then, the results can be analyzed and compared using the following
 command:
 
 ```
 $ python benchmarks/holoscan_flow_benchmarking/analyze.py <insert
-your metrics options> -g endoscopy_greedy_outputs/logger* "Endoscopy (Greedy)" -g endoscopy_multithread_outputs/logger* "Endoscopy (Multithread)"
+your metrics options> -g endoscopy_greedy_outputs/logger* "Endoscopy (Greedy)" -g endoscopy_multithread_outputs/logger* "Endoscopy (Multithread)" -g endoscopy_eventbased_outputs/logger* "Endoscopy (Event-based)"
 ```
 
 ## Tail and Flatness of the CDF Curve