expose frame time at each step of processing

detect_objects.py

@@ -206,6 +206,7 @@ def main():
             'fps': mp.Value('d', float(config['fps'])),
             'skipped_fps': mp.Value('d', 0.0),
             'detection_fps': mp.Value('d', 0.0),
+            'detection_frame': mp.Value('d', 0.0),
             'read_start': mp.Value('d', 0.0),
             'ffmpeg_process': ffmpeg_process,
             'ffmpeg_cmd': ffmpeg_cmd,
@@ -217,7 +218,7 @@ def main():
         camera_process = mp.Process(target=track_camera, args=(name, config, GLOBAL_OBJECT_CONFIG, frame_queue, frame_shape,
             tflite_process.detection_queue, tracked_objects_queue, camera_processes[name]['fps'], 
             camera_processes[name]['skipped_fps'], camera_processes[name]['detection_fps'], 
-            camera_processes[name]['read_start']))
+            camera_processes[name]['read_start'], camera_processes[name]['detection_frame']))
         camera_process.daemon = True
         camera_processes[name]['process'] = camera_process
 
@@ -272,7 +273,12 @@ def main():
                 'detection_fps': round(camera_stats['detection_fps'].value, 2),
                 'read_start': camera_stats['read_start'].value,
                 'pid': camera_stats['process'].pid,
-                'ffmpeg_pid': camera_stats['ffmpeg_process'].pid
+                'ffmpeg_pid': camera_stats['ffmpeg_process'].pid,
+                'frame_info': {
+                    'read': camera_stats['capture_thread'].current_frame,
+                    'detect': camera_stats['detection_frame'].value,
+                    'process': object_processor.camera_data[name]['current_frame_time']
+                }
             }
         
         stats['coral'] = {

frigate/object_processing.py

@@ -34,6 +34,7 @@ class TrackedObjectProcessor(threading.Thread):
             'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')),
             'tracked_objects': {},
             'current_frame': np.zeros((720,1280,3), np.uint8),
+            'current_frame_time': 0.0,
             'object_id': None
         })
         self.plasma_client = PlasmaManager()
@@ -55,6 +56,7 @@ class TrackedObjectProcessor(threading.Thread):
             best_objects = self.camera_data[camera]['best_objects']
             current_object_status = self.camera_data[camera]['object_status']
             self.camera_data[camera]['tracked_objects'] = tracked_objects
+            self.camera_data[camera]['current_frame_time'] = frame_time
 
             ###
             # Draw tracked objects on the frame

frigate/video.py

@@ -125,6 +125,7 @@ class CameraCapture(threading.Thread):
         self.fps = fps
         self.plasma_client = PlasmaManager()
         self.ffmpeg_process = ffmpeg_process
+        self.current_frame = 0
 
     def run(self):
         frame_num = 0
@@ -134,7 +135,7 @@ class CameraCapture(threading.Thread):
                 break
 
             frame_bytes = self.ffmpeg_process.stdout.read(self.frame_size)
-            frame_time = datetime.datetime.now().timestamp()
+            self.current_frame = datetime.datetime.now().timestamp()
 
             if len(frame_bytes) == 0:
                 print(f"{self.name}: ffmpeg didnt return a frame. something is wrong.")
@@ -145,17 +146,17 @@ class CameraCapture(threading.Thread):
                 continue
 
             # put the frame in the plasma store
-            self.plasma_client.put(f"{self.name}{frame_time}",
+            self.plasma_client.put(f"{self.name}{self.current_frame}",
                     np
                         .frombuffer(frame_bytes, np.uint8)
                         .reshape(self.frame_shape)
                 )
             # add to the queue
-            self.frame_queue.put(frame_time)
+            self.frame_queue.put(self.current_frame)
 
             self.fps.update()
 
-def track_camera(name, config, global_objects_config, frame_queue, frame_shape, detection_queue, detected_objects_queue, fps, skipped_fps, detection_fps, read_start):
+def track_camera(name, config, global_objects_config, frame_queue, frame_shape, detection_queue, detected_objects_queue, fps, skipped_fps, detection_fps, read_start, detection_frame):
     print(f"Starting process for {name}: {os.getpid()}")
     listen()
 
@@ -204,6 +205,7 @@ def track_camera(name, config, global_objects_config, frame_queue, frame_shape,
         duration = datetime.datetime.now().timestamp()-read_start.value
         read_start.value = 0.0
         avg_wait = (avg_wait*99+duration)/100
+        detection_frame.value = frame_time
 
         fps_tracker.update()
         fps.value = fps_tracker.eps()