create tracked object class and save thumbnails

frigate/app.py

@@ -88,7 +88,7 @@ class FrigateApp():
                 self.detectors[name] = EdgeTPUProcess(name, self.detection_queue, out_events=self.detection_out_events, tf_device=detector.device)
 
     def start_detected_frames_processor(self):
-        self.detected_frames_processor = TrackedObjectProcessor(self.config.cameras, self.mqtt_client, self.config.mqtt.topic_prefix, 
+        self.detected_frames_processor = TrackedObjectProcessor(self.config, self.mqtt_client, self.config.mqtt.topic_prefix, 
             self.detected_frames_queue, self.event_queue, self.stop_event)
         self.detected_frames_processor.start()
 

frigate/config.py

@@ -91,16 +91,14 @@ OBJECTS_SCHEMA = vol.Schema(vol.All(filters_for_all_tracked_objects,
     }
 ))
 
-DEFAULT_CAMERA_MQTT = {
-    'crop_to_region': True
-}
 DEFAULT_CAMERA_SAVE_CLIPS = {
     'enabled': False
 }
 DEFAULT_CAMERA_SNAPSHOTS = {
     'show_timestamp': True,
     'draw_zones': False,
-    'draw_bounding_boxes': True
+    'draw_bounding_boxes': True,
+    'crop_to_region': True
 }
 
 CAMERA_FFMPEG_SCHEMA = vol.Schema(
@@ -122,10 +120,6 @@ CAMERAS_SCHEMA = vol.Schema(
             'fps': int,
             'mask': str,
             vol.Optional('best_image_timeout', default=60): int,
-            vol.Optional('mqtt', default=DEFAULT_CAMERA_MQTT): {
-                vol.Optional('crop_to_region', default=True): bool,
-                'snapshot_height': int
-            },
             vol.Optional('zones', default={}):  {
                 str: {
                     vol.Required('coordinates'): vol.Any(str, [str]),
@@ -140,7 +134,9 @@ CAMERAS_SCHEMA = vol.Schema(
              vol.Optional('snapshots', default=DEFAULT_CAMERA_SNAPSHOTS): {
                 vol.Optional('show_timestamp', default=True): bool,
                 vol.Optional('draw_zones', default=False): bool,
-                vol.Optional('draw_bounding_boxes', default=True): bool
+                vol.Optional('draw_bounding_boxes', default=True): bool,
+                vol.Optional('crop_to_region', default=True): bool,
+                'height': int
              },
              'objects': OBJECTS_SCHEMA
         }
@@ -296,6 +292,8 @@ class CameraSnapshotsConfig():
         self._show_timestamp = config['show_timestamp']
         self._draw_zones = config['draw_zones']
         self._draw_bounding_boxes = config['draw_bounding_boxes']
+        self._crop_to_region = config['crop_to_region']
+        self._height = config.get('height')
     
     @property
     def show_timestamp(self):
@@ -309,6 +307,14 @@ class CameraSnapshotsConfig():
     def draw_bounding_boxes(self):
         return self._draw_bounding_boxes
 
+    @property
+    def crop_to_region(self):
+        return self._crop_to_region
+
+    @property
+    def height(self):
+        return self._height
+
 class CameraSaveClipsConfig():
     def __init__(self, config):
         self._enabled = config['enabled']
@@ -327,19 +333,6 @@ class CameraSaveClipsConfig():
     def objects(self):
         return self._objects
 
-class CameraMqttConfig():
-    def __init__(self, config):
-        self._crop_to_region = config['crop_to_region']
-        self._snapshot_height = config.get('snapshot_height')
-    
-    @property
-    def crop_to_region(self):
-        return self._crop_to_region
-    
-    @property
-    def snapshot_height(self):
-        return self._snapshot_height
-
 class ZoneConfig():
     def __init__(self, name, config):
         self._coordinates = config['coordinates']
@@ -391,7 +384,6 @@ class CameraConfig():
         self._fps = config.get('fps')
         self._mask = self._create_mask(config.get('mask'))
         self._best_image_timeout = config['best_image_timeout']
-        self._mqtt = CameraMqttConfig(config['mqtt'])
         self._zones = { name: ZoneConfig(name, z) for name, z in config['zones'].items() }
         self._save_clips = CameraSaveClipsConfig(config['save_clips'])
         self._snapshots = CameraSnapshotsConfig(config['snapshots'])

frigate/object_processing.py

@@ -4,6 +4,7 @@ import hashlib
 import itertools
 import json
 import logging
+import os
 import queue
 import threading
 import time
@@ -15,7 +16,7 @@ import cv2
 import matplotlib.pyplot as plt
 import numpy as np
 
-from frigate.config import CameraConfig
+from frigate.config import FrigateConfig, CameraConfig
 from frigate.edgetpu import load_labels
 from frigate.util import SharedMemoryFrameManager, draw_box_with_label
 
@@ -30,28 +31,6 @@ COLOR_MAP = {}
 for key, val in LABELS.items():
     COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3])
 
-def zone_filtered(obj, object_config):
-    object_name = obj['label']
-
-    if object_name in object_config:
-        obj_settings = object_config[object_name]
-
-        # if the min area is larger than the
-        # detected object, don't add it to detected objects
-        if obj_settings.min_area > obj['area']:
-            return True
-        
-        # if the detected object is larger than the
-        # max area, don't add it to detected objects
-        if obj_settings.max_area < obj['area']:
-            return True
-
-        # if the score is lower than the threshold, skip
-        if obj_settings.threshold > obj['computed_score']:
-            return True
-        
-    return False
-
 def on_edge(box, frame_shape):
     if (
         box[0] == 0 or
@@ -80,19 +59,182 @@ def is_better_thumbnail(current_thumb, new_obj, frame_shape) -> bool:
     
     return False
 
+class TrackedObject():
+    def __init__(self, camera, camera_config: CameraConfig, thumbnail_frames, obj_data):
+        self.obj_data = obj_data
+        self.camera = camera
+        self.camera_config = camera_config
+        self.thumbnail_frames = thumbnail_frames
+        self.current_zones = []
+        self.entered_zones = set()
+        self._false_positive = True
+        self.top_score = self.computed_score = 0.0
+        self.thumbnail_data = {
+            'frame_time': obj_data['frame_time'],
+            'box': obj_data['box'],
+            'area': obj_data['area'],
+            'region': obj_data['region'],
+            'score': obj_data['score']
+        }
+        self.frame = None
+        self._snapshot_jpg_time = 0
+        self._snapshot_jpg = None
+
+        # start the score history
+        self.score_history = [self.obj_data['score']]
+
+    def false_positive(self):
+        # once a true positive, always a true positive
+        if not self._false_positive:
+            return False
+
+        threshold = self.camera_config.objects.filters[self.obj_data['label']].threshold
+        if self.computed_score < threshold:
+            return True
+        return False
+
+    def compute_score(self):
+        scores = self.score_history[:]
+        # pad with zeros if you dont have at least 3 scores
+        if len(scores) < 3:
+            scores += [0.0]*(3 - len(scores))
+        return median(scores)
+    
+    def update(self, current_frame_time, obj_data):
+        self.obj_data.update(obj_data)
+        # if the object is not in the current frame, add a 0.0 to the score history
+        if self.obj_data['frame_time'] != current_frame_time:
+            self.score_history.append(0.0)
+        else:
+            self.score_history.append(self.obj_data['score'])
+        # only keep the last 10 scores
+        if len(self.score_history) > 10:
+            self.score_history = self.score_history[-10:]
+
+        # calculate if this is a false positive
+        self.computed_score = self.compute_score()
+        if self.computed_score > self.top_score:
+            self.top_score = self.computed_score
+        self._false_positive = self.false_positive()
+
+        # determine if this frame is a better thumbnail
+        if is_better_thumbnail(self.thumbnail_data, self.obj_data, self.camera_config.frame_shape):
+            self.thumbnail_data = {
+                'frame_time': self.obj_data['frame_time'],
+                'box': self.obj_data['box'],
+                'area': self.obj_data['area'],
+                'region': self.obj_data['region'],
+                'score': self.obj_data['score']
+            }
+        
+        # check zones
+        current_zones = []
+        bottom_center = (self.obj_data['centroid'][0], self.obj_data['box'][3])
+        # check each zone
+        for name, zone in self.camera_config.zones.items():
+            contour = zone.contour
+            # check if the object is in the zone
+            if (cv2.pointPolygonTest(contour, bottom_center, False) >= 0):
+                # if the object passed the filters once, dont apply again
+                if name in self.current_zones or not zone_filtered(self, zone.filters):
+                    current_zones.append(name)
+                    self.entered_zones.add(name)
+                
+        self.current_zones = current_zones
+    
+    def to_dict(self):
+        return {
+            'id': self.obj_data['id'],
+            'camera': self.camera,
+            'frame_time': self.obj_data['frame_time'],
+            'label': self.obj_data['label'],
+            'top_score': self.top_score,
+            'false_positive': self._false_positive,
+            'start_time': self.obj_data['start_time'],
+            'end_time': self.obj_data.get('end_time', None),
+            'score': self.obj_data['score'],
+            'box': self.obj_data['box'],
+            'area': self.obj_data['area'],
+            'region': self.obj_data['region'],
+            'current_zones': self.current_zones.copy(),
+            'entered_zones': list(self.entered_zones).copy()
+        }
+    
+    @property
+    def snapshot_jpg(self):
+        if self._snapshot_jpg_time == self.thumbnail_data['frame_time']:
+            return self._snapshot_jpg
+
+        # TODO: crop first to avoid converting the entire frame?
+        snapshot_config = self.camera_config.snapshots
+        best_frame = cv2.cvtColor(self.thumbnail_frames[self.thumbnail_data['frame_time']], cv2.COLOR_YUV2BGR_I420)
+
+        if snapshot_config.draw_bounding_boxes:
+            thickness = 2
+            color = COLOR_MAP[self.obj_data['label']]
+            box = self.thumbnail_data['box']
+            draw_box_with_label(best_frame, box[0], box[1], box[2], box[3], self.obj_data['label'], 
+                f"{int(self.thumbnail_data['score']*100)}% {int(self.thumbnail_data['area'])}", thickness=thickness, color=color)
+            
+        if snapshot_config.crop_to_region:
+            region = self.thumbnail_data['region']
+            best_frame = best_frame[region[1]:region[3], region[0]:region[2]]
+
+        if snapshot_config.height: 
+            height = snapshot_config.height
+            width = int(height*best_frame.shape[1]/best_frame.shape[0])
+            best_frame = cv2.resize(best_frame, dsize=(width, height), interpolation=cv2.INTER_AREA)
+        
+        if snapshot_config.show_timestamp:
+            time_to_show = datetime.datetime.fromtimestamp(self.thumbnail_data['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
+            size = cv2.getTextSize(time_to_show, cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, thickness=2)
+            text_width = size[0][0]
+            desired_size = max(200, 0.33*best_frame.shape[1])
+            font_scale = desired_size/text_width
+            cv2.putText(best_frame, time_to_show, (5, best_frame.shape[0]-7), cv2.FONT_HERSHEY_SIMPLEX, 
+                fontScale=font_scale, color=(255, 255, 255), thickness=2)
+
+        ret, jpg = cv2.imencode('.jpg', best_frame)
+        if ret:
+            self._snapshot_jpg = jpg.tobytes()
+        
+        return self._snapshot_jpg
+
+def zone_filtered(obj: TrackedObject, object_config):
+    object_name = obj.obj_data['label']
+
+    if object_name in object_config:
+        obj_settings = object_config[object_name]
+
+        # if the min area is larger than the
+        # detected object, don't add it to detected objects
+        if obj_settings.min_area > obj.obj_data['area']:
+            return True
+        
+        # if the detected object is larger than the
+        # max area, don't add it to detected objects
+        if obj_settings.max_area < obj.obj_data['area']:
+            return True
+
+        # if the score is lower than the threshold, skip
+        if obj_settings.threshold > obj.computed_score:
+            return True
+        
+    return False
+
 # Maintains the state of a camera
 class CameraState():
     def __init__(self, name, config, frame_manager):
         self.name = name
         self.config = config
+        self.camera_config = config.cameras[name]
         self.frame_manager = frame_manager
-
         self.best_objects = {}
         self.object_status = defaultdict(lambda: 'OFF')
-        self.tracked_objects = {}
+        self.tracked_objects: Dict[str, TrackedObject] = {}
         self.thumbnail_frames = {}
         self.zone_objects = defaultdict(lambda: [])
-        self._current_frame = np.zeros(self.config.frame_shape_yuv, np.uint8)
+        self._current_frame = np.zeros(self.camera_config.frame_shape_yuv, np.uint8)
         self.current_frame_lock = threading.Lock()
         self.current_frame_time = 0.0
         self.previous_frame_id = None
@@ -102,7 +244,7 @@ class CameraState():
         with self.current_frame_lock:
             frame_copy = np.copy(self._current_frame)
             frame_time = self.current_frame_time
-            tracked_objects = copy.deepcopy(self.tracked_objects)
+            tracked_objects = {k: v.to_dict() for k,v in self.tracked_objects.items()}
         
         frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_YUV2BGR_I420)
         # draw on the frame
@@ -123,42 +265,25 @@ class CameraState():
                 region = obj['region']
                 cv2.rectangle(frame_copy, (region[0], region[1]), (region[2], region[3]), (0,255,0), 1)
             
-            if self.config.snapshots.show_timestamp:
+            if self.camera_config.snapshots.show_timestamp:
                 time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
                 cv2.putText(frame_copy, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
 
-            if self.config.snapshots.draw_zones:
-                for name, zone in self.config.zones.items():
-                    thickness = 8 if any([name in obj['zones'] for obj in tracked_objects.values()]) else 2
+            if self.camera_config.snapshots.draw_zones:
+                for name, zone in self.camera_config.zones.items():
+                    thickness = 8 if any([name in obj['current_zones'] for obj in tracked_objects.values()]) else 2
                     cv2.drawContours(frame_copy, [zone.contour], -1, zone.color, thickness)
         
         return frame_copy
 
-    def false_positive(self, obj):
-        # once a true positive, always a true positive
-        if not obj.get('false_positive', True):
-            return False
-
-        threshold = self.config.objects.filters[obj['label']].threshold
-        if obj['computed_score'] < threshold:
-            return True
-        return False
-
-    def compute_score(self, obj):
-        scores = obj['score_history'][:]
-        # pad with zeros if you dont have at least 3 scores
-        if len(scores) < 3:
-            scores += [0.0]*(3 - len(scores))
-        return median(scores)
-
     def on(self, event_type: str, callback: Callable[[Dict], None]):
         self.callbacks[event_type].append(callback)
 
     def update(self, frame_time, tracked_objects):
         self.current_frame_time = frame_time
-        # get the new frame and delete the old frame
+        # get the new frame
         frame_id = f"{self.name}{frame_time}"
-        current_frame = self.frame_manager.get(frame_id, self.config.frame_shape_yuv)
+        current_frame = self.frame_manager.get(frame_id, self.camera_config.frame_shape_yuv)
 
         current_ids = tracked_objects.keys()
         previous_ids = self.tracked_objects.keys()
@@ -167,59 +292,20 @@ class CameraState():
         updated_ids = list(set(current_ids).intersection(previous_ids))
 
         for id in new_ids:
-            new_obj = self.tracked_objects[id] = tracked_objects[id]
-            new_obj['zones'] = []
-            new_obj['entered_zones'] = set()
-            new_obj['thumbnail'] = {
-                'frame': new_obj['frame_time'],
-                'box': new_obj['box'],
-                'area': new_obj['area'],
-                'region': new_obj['region'],
-                'score': new_obj['score']
-            }
-
-            # start the score history
-            new_obj['score_history'] = [self.tracked_objects[id]['score']]
-
-            # calculate if this is a false positive
-            new_obj['computed_score'] = self.compute_score(self.tracked_objects[id])
-            new_obj['top_score'] = self.tracked_objects[id]['computed_score']
-            new_obj['false_positive'] = self.false_positive(self.tracked_objects[id])
+            new_obj = self.tracked_objects[id] = TrackedObject(self.name, self.camera_config, self.thumbnail_frames, tracked_objects[id])
 
             # call event handlers
             for c in self.callbacks['start']:
                 c(self.name, new_obj)
         
         for id in updated_ids:
-            self.tracked_objects[id].update(tracked_objects[id])
-
             updated_obj = self.tracked_objects[id]
+            updated_obj.update(frame_time, tracked_objects[id])
 
-            # if the object is not in the current frame, add a 0.0 to the score history
-            if updated_obj['frame_time'] != self.current_frame_time:
-                updated_obj['score_history'].append(0.0)
-            else:
-                updated_obj['score_history'].append(updated_obj['score'])
-            # only keep the last 10 scores
-            if len(updated_obj['score_history']) > 10:
-                updated_obj['score_history'] = updated_obj['score_history'][-10:]
-
-            # calculate if this is a false positive
-            computed_score = self.compute_score(updated_obj)
-            updated_obj['computed_score'] = computed_score
-            if computed_score > updated_obj['top_score']:
-                updated_obj['top_score'] = computed_score
-            updated_obj['false_positive'] = self.false_positive(updated_obj)
-
-            # determine if this frame is a better thumbnail
-            if is_better_thumbnail(updated_obj['thumbnail'], updated_obj, self.config.frame_shape):
-                updated_obj['thumbnail'] = {
-                    'frame': updated_obj['frame_time'],
-                    'box': updated_obj['box'],
-                    'area': updated_obj['area'],
-                    'region': updated_obj['region'],
-                    'score': updated_obj['score']
-                }
+            if (not updated_obj._false_positive 
+                and updated_obj.thumbnail_data['frame_time'] == frame_time 
+                and frame_time not in self.thumbnail_frames):
+                self.thumbnail_frames[frame_time] = np.copy(current_frame)
 
             # call event handlers
             for c in self.callbacks['update']:
@@ -227,53 +313,35 @@ class CameraState():
         
         for id in removed_ids:
             # publish events to mqtt
-            self.tracked_objects[id]['end_time'] = frame_time
+            removed_obj = self.tracked_objects[id]
+            removed_obj.obj_data['end_time'] = frame_time
             for c in self.callbacks['end']:
-                c(self.name, self.tracked_objects[id])
+                c(self.name, removed_obj)
             del self.tracked_objects[id]
 
-        # check to see if the objects are in any zones
-        for obj in self.tracked_objects.values():
-            current_zones = []
-            bottom_center = (obj['centroid'][0], obj['box'][3])
-            # check each zone
-            for name, zone in self.config.zones.items():
-                contour = zone.contour
-                # check if the object is in the zone
-                if (cv2.pointPolygonTest(contour, bottom_center, False) >= 0):
-                    # if the object passed the filters once, dont apply again
-                    if name in obj.get('zones', []) or not zone_filtered(obj, zone.filters):
-                        current_zones.append(name)
-                        obj['entered_zones'].add(name)
-
-                    
-            obj['zones'] = current_zones
-
-        # update frame storage for thumbnails based on thumbnails for all tracked objects
-        current_thumb_frames = set([obj['thumbnail']['frame'] for obj in self.tracked_objects.values()])
-        if self.current_frame_time in current_thumb_frames:
-            self.thumbnail_frames[self.current_frame_time] = np.copy(current_frame)
-        thumb_frames_to_delete = [t for t in self.thumbnail_frames.keys() if not t in current_thumb_frames]
-        for t in thumb_frames_to_delete: del self.thumbnail_frames[t]
-
+        # TODO: can i switch to looking this up and only changing when an event ends?
+        #       maybe make an api endpoint that pulls the thumbnail from the file system?
         # maintain best objects
         for obj in self.tracked_objects.values():
-            object_type = obj['label']
-            # if the object wasn't seen on the current frame, skip it
-            if obj['frame_time'] != self.current_frame_time or obj['false_positive']:
+            object_type = obj.obj_data['label']
+            # if the object's thumbnail is not from the current frame
+            if obj.thumbnail_data['frame_time'] != self.current_frame_time or obj.false_positive:
                 continue
-            obj_copy = copy.deepcopy(obj)
             if object_type in self.best_objects:
                 current_best = self.best_objects[object_type]
                 now = datetime.datetime.now().timestamp()
                 # if the object is a higher score than the current best score 
                 # or the current object is older than desired, use the new object
-                if obj_copy['score'] > current_best['score'] or (now - current_best['frame_time']) > self.config.best_image_timeout:
+                if is_better_thumbnail(current_best['thumbnail'], obj.thumbnail, self.camera_config.frame_shape) or (now - current_best['frame_time']) > self.config.best_image_timeout:
+                    obj_copy = copy.deepcopy(obj.obj_data)
+                    obj_copy['thumbnail'] = copy.deepcopy(obj.thumbnail_data)
                     obj_copy['frame'] = np.copy(current_frame)
                     self.best_objects[object_type] = obj_copy
                     for c in self.callbacks['snapshot']:
                         c(self.name, self.best_objects[object_type])
             else:
+                obj_copy = copy.deepcopy(obj)
+                obj_copy['thumbnail'] = copy.deepcopy(obj.thumbnail_data)
                 obj_copy['frame'] = np.copy(current_frame)
                 self.best_objects[object_type] = obj_copy
                 for c in self.callbacks['snapshot']:
@@ -282,8 +350,8 @@ class CameraState():
         # update overall camera state for each object type
         obj_counter = Counter()
         for obj in self.tracked_objects.values():
-            if not obj['false_positive']:
-                obj_counter[obj['label']] += 1
+            if not obj.false_positive:
+                obj_counter[obj.obj_data['label']] += 1
                 
         # report on detected objects
         for obj_name, count in obj_counter.items():
@@ -302,6 +370,12 @@ class CameraState():
             for c in self.callbacks['snapshot']:
                 c(self.name, self.best_objects[obj_name])
         
+        # cleanup thumbnail frame cache
+        current_thumb_frames = set([obj.thumbnail_data['frame_time'] for obj in self.tracked_objects.values() if not obj._false_positive])
+        thumb_frames_to_delete = [t for t in self.thumbnail_frames.keys() if not t in current_thumb_frames]
+        for t in thumb_frames_to_delete:
+            del self.thumbnail_frames[t]
+        
         with self.current_frame_lock:
             self._current_frame = current_frame
             if not self.previous_frame_id is None:
@@ -309,10 +383,10 @@ class CameraState():
             self.previous_frame_id = frame_id
 
 class TrackedObjectProcessor(threading.Thread):
-    def __init__(self, camera_config: Dict[str, CameraConfig], client, topic_prefix, tracked_objects_queue, event_queue, stop_event):
+    def __init__(self, config: FrigateConfig, client, topic_prefix, tracked_objects_queue, event_queue, stop_event):
         threading.Thread.__init__(self)
         self.name = "detected_frames_processor"
-        self.camera_config = camera_config
+        self.config = config
         self.client = client
         self.topic_prefix = topic_prefix
         self.tracked_objects_queue = tracked_objects_queue
@@ -321,76 +395,29 @@ class TrackedObjectProcessor(threading.Thread):
         self.camera_states: Dict[str, CameraState] = {}
         self.frame_manager = SharedMemoryFrameManager()
 
-        def start(camera, obj):
-            # publish events to mqtt
-            event_data = {
-              'id': obj['id'],
-              'label': obj['label'],
-              'camera': camera,
-              'start_time': obj['start_time'],
-              'top_score': obj['top_score'],
-              'false_positive': obj['false_positive'],
-              'zones': list(obj['entered_zones'])
-            }
-            self.client.publish(f"{self.topic_prefix}/{camera}/events/start", json.dumps(event_data), retain=False)
-            self.event_queue.put(('start', camera, obj))
+        def start(camera, obj: TrackedObject):
+            self.client.publish(f"{self.topic_prefix}/{camera}/events/start", json.dumps(obj.to_dict()), retain=False)
+            self.event_queue.put(('start', camera, obj.to_dict()))
 
-        def update(camera, obj):
+        def update(camera, obj: TrackedObject):
             pass
 
-        def end(camera, obj):
-            event_data = {
-              'id': obj['id'],
-              'label': obj['label'],
-              'camera': camera,
-              'start_time': obj['start_time'],
-              'end_time': obj['end_time'],
-              'top_score': obj['top_score'],
-              'false_positive': obj['false_positive'],
-              'zones': list(obj['entered_zones'])
-            }
-            self.client.publish(f"{self.topic_prefix}/{camera}/events/end", json.dumps(event_data), retain=False)
-            self.event_queue.put(('end', camera, obj))
+        def end(camera, obj: TrackedObject):
+            self.client.publish(f"{self.topic_prefix}/{camera}/events/end", json.dumps(obj.to_dict()), retain=False)
+            if self.config.cameras[camera].save_clips.enabled:
+                thumbnail_file_name = f"{camera}-{obj.obj_data['id']}.jpg"
+                with open(os.path.join(self.config.save_clips.clips_dir, thumbnail_file_name), 'wb') as f:
+                    f.write(obj.snapshot_jpg)
+            self.event_queue.put(('end', camera, obj.to_dict()))
         
-        def snapshot(camera, obj):
-            if not 'frame' in obj:
-                return
-            
-            best_frame = cv2.cvtColor(obj['frame'], cv2.COLOR_YUV2BGR_I420)
-            if self.camera_config[camera].snapshots.draw_bounding_boxes:
-                thickness = 2
-                color = COLOR_MAP[obj['label']]
-                box = obj['box']
-                draw_box_with_label(best_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
-                
-            mqtt_config = self.camera_config[camera].mqtt
-            if mqtt_config.crop_to_region:
-                region = obj['region']
-                best_frame = best_frame[region[1]:region[3], region[0]:region[2]]
-            if mqtt_config.snapshot_height: 
-                height = mqtt_config.snapshot_height
-                width = int(height*best_frame.shape[1]/best_frame.shape[0])
-                best_frame = cv2.resize(best_frame, dsize=(width, height), interpolation=cv2.INTER_AREA)
-            
-            if self.camera_config[camera].snapshots.show_timestamp:
-                time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
-                size = cv2.getTextSize(time_to_show, cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, thickness=2)
-                text_width = size[0][0]
-                text_height = size[0][1]
-                desired_size = max(200, 0.33*best_frame.shape[1])
-                font_scale = desired_size/text_width
-                cv2.putText(best_frame, time_to_show, (5, best_frame.shape[0]-7), cv2.FONT_HERSHEY_SIMPLEX, fontScale=font_scale, color=(255, 255, 255), thickness=2)
-
-            ret, jpg = cv2.imencode('.jpg', best_frame)
-            if ret:
-                jpg_bytes = jpg.tobytes()
-                self.client.publish(f"{self.topic_prefix}/{camera}/{obj['label']}/snapshot", jpg_bytes, retain=True)
+        def snapshot(camera, obj: TrackedObject):
+            self.client.publish(f"{self.topic_prefix}/{camera}/{obj['label']}/snapshot", obj.snapshot_jpg, retain=True)
         
         def object_status(camera, object_name, status):
             self.client.publish(f"{self.topic_prefix}/{camera}/{object_name}", status, retain=False)
 
-        for camera in self.camera_config.keys():
-            camera_state = CameraState(camera, self.camera_config[camera], self.frame_manager)
+        for camera in self.config.cameras.keys():
+            camera_state = CameraState(camera, self.config, self.frame_manager)
             camera_state.on('start', start)
             camera_state.on('update', update)
             camera_state.on('end', end)
@@ -398,14 +425,6 @@ class TrackedObjectProcessor(threading.Thread):
             camera_state.on('object_status', object_status)
             self.camera_states[camera] = camera_state
 
-        self.camera_data = defaultdict(lambda: {
-            'best_objects': {},
-            'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')),
-            'tracked_objects': {},
-            'current_frame': np.zeros((720,1280,3), np.uint8),
-            'current_frame_time': 0.0,
-            'object_id': None
-        })
         # {
         #   'zone_name': {
         #       'person': ['camera_1', 'camera_2']
@@ -439,9 +458,9 @@ class TrackedObjectProcessor(threading.Thread):
             camera_state.update(frame_time, current_tracked_objects)
 
             # update zone status for each label
-            for zone in camera_state.config.zones.keys():
+            for zone in self.config.cameras[camera].zones.keys():
                 # get labels for current camera and all labels in current zone
-                labels_for_camera = set([obj['label'] for obj in camera_state.tracked_objects.values() if zone in obj['zones'] and not obj['false_positive']])
+                labels_for_camera = set([obj.obj_data['label'] for obj in camera_state.tracked_objects.values() if zone in obj.current_zones and not obj._false_positive])
                 labels_to_check = labels_for_camera | set(self.zone_data[zone].keys())
                 # for each label in zone
                 for label in labels_to_check: