- Took slowmo into account for filtering

- start time sometimes randomly not zero: used ffmpeg to determine it instead of cv2
- solved wrong coord padding whe person disappears from frame

Author: davidpagnon

Sports2D/Demo/Config_demo.toml

@@ -102,7 +102,7 @@ show_graphs = true            # Show plots of raw and processed results
 filter_type = 'butterworth'   # butterworth, gaussian, LOESS, median
    [post-processing.butterworth]
    order = 4 
-   cut_off_frequency = 3 # Hz
+   cut_off_frequency = 6 # Hz # Will be divided by slowmo_factor to be equivalent to non slowed-down video
    [post-processing.gaussian]
    sigma_kernel = 1 #px
    [post-processing.loess]

Sports2D/Sports2D.py

@@ -194,7 +194,7 @@
                                         'filter': True,
                                         'show_graphs': True,
                                         'filter_type': 'butterworth',
-                                        'butterworth': {'order': 4, 'cut_off_frequency': 3},
+                                        'butterworth': {'order': 4, 'cut_off_frequency': 6},
                                         'gaussian': {'sigma_kernel': 1},
                                         'loess': {'nb_values_used': 5},
                                         'median': {'kernel_size': 3}

Sports2D/Utilities/common.py

@@ -164,6 +164,9 @@ def make_homogeneous(list_of_arrays):
     '''
 
     def get_max_shape(list_of_arrays):
+        '''
+        Recursively determine the maximum shape of a list of arrays.
+        '''
         if isinstance(list_of_arrays[0], list):
             # Maximum length at the current level plus the max shape at the next level
             return [max(len(arr) for arr in list_of_arrays)] + get_max_shape(
@@ -176,12 +179,16 @@ def pad_with_nans(list_of_arrays, target_shape):
         '''
         Recursively pad list_of_arrays with nans to match the target shape.
         '''
-        if isinstance(list_of_arrays, np.ndarray):
-            if list_of_arrays.size == 0:  # Handle empty arrays
-            # Return an array of nans with the correct target shape
-                return np.full(target_shape, np.nan, dtype=float)
-            # Pad the current array to the target shape
-            pad_width = [(0, max_dim - curr_dim) for curr_dim, max_dim in zip(list_of_arrays.shape, target_shape)]
+        if isinstance(list_of_arrays, np.ndarray):        
+            # Pad the current array to the target shape        
+            pad_width = []        
+            for dim_index in range(0, len(target_shape)):
+                if dim_index == len(list_of_arrays.shape) or dim_index > len(list_of_arrays.shape):
+                    list_of_arrays = np.expand_dims(list_of_arrays, 0)
+            for dim_index in range(0, len(target_shape)):
+                max_dim = target_shape[dim_index]
+                curr_dim = list_of_arrays.shape[dim_index]
+                pad_width.append((0, max_dim - curr_dim))
             return np.pad(list_of_arrays.astype(float), pad_width, constant_values=np.nan)
         # Recursively pad each array in the list
         return [pad_with_nans(array, target_shape[1:]) for array in list_of_arrays]
@@ -194,6 +201,22 @@ def pad_with_nans(list_of_arrays, target_shape):
     return np.array(list_of_arrays)
 
 
+def get_start_time_ffmpeg(video_path):
+    '''
+    Get the start time of a video using FFmpeg.
+    '''
+
+    cmd = ["ffmpeg", "-i", video_path]
+    result = subprocess.run(cmd, stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, text=True)
+    for line in result.stderr.splitlines():
+        if "start:" in line:
+            parts = line.split("start:")
+            if len(parts) > 1:
+                start_time = parts[1].split(",")[0].strip()
+                return float(start_time)
+    return 0.0  # Default to 0 if not found
+
+
 def resample_video(vid_output_path, fps, desired_framerate):
     '''
     Resample video to the desired fps using ffmpeg.

Sports2D/process.py

@@ -1416,6 +1416,7 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
     gaussian_filter_kernel = config_dict.get('post-processing').get('gaussian').get('sigma_kernel')
     loess_filter_kernel = config_dict.get('post-processing').get('loess').get('nb_values_used')
     median_filter_kernel = config_dict.get('post-processing').get('median').get('kernel_size')
+    butterworth_filter_cutoff /= slowmo_factor
     filter_options = [do_filter, filter_type,
                            butterworth_filter_order, butterworth_filter_cutoff, frame_rate,
                            gaussian_filter_kernel, loess_filter_kernel, median_filter_kernel]
@@ -1452,7 +1453,8 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
         logging.warning('Webcam input: the framerate may vary. If results are filtered, Sports2D will use the average framerate as input.')
     else:
         cap, out_vid, cam_width, cam_height, fps = setup_video(video_file_path, save_vid, vid_output_path)
-        frame_range = [int(time_range[0] * frame_rate), int(time_range[1] * frame_rate)] if time_range else [0, int(cap.get(cv2.CAP_PROP_FRAME_COUNT))]
+        start_time = get_start_time_ffmpeg(video_file_path)
+        frame_range = [int((time_range[0]-start_time) * frame_rate), int((time_range[1]-start_time) * frame_rate)] if time_range else [0, int(cap.get(cv2.CAP_PROP_FRAME_COUNT))]
         frame_iterator = tqdm(range(*frame_range)) # use a progress bar
     if show_realtime_results:
         cv2.namedWindow(f'{video_file} Sports2D', cv2.WINDOW_NORMAL + cv2.WINDOW_KEEPRATIO)
@@ -1498,6 +1500,7 @@ def process_fun(config_dict, video_file, time_range, frame_rate, result_dir):
     frame_processing_times = []
     frame_count = 0
     while cap.isOpened():
+        # Skip to the starting frame
         if frame_count < frame_range[0]:
             cap.read()
             frame_count += 1