initial implementation of zones

config/config.example.yml

@@ -68,6 +68,41 @@ objects:
       max_area: 100000
       threshold: 0.5
 
+zones:
+  #################
+  # Name of the zone
+  ################
+  front_steps:
+    cameras:
+      front_door:
+        ####################
+        # For each camera, a list of x,y coordinates to define the polygon of the zone.
+        # Can also be a comma separated string of all x,y coordinates combined.
+        # The same zone can exist across multiple cameras if they have overlapping FOVs.
+        # An object is determined to be in the zone based on whether or not the bottom center
+        # of it's bounding box is within the polygon. The polygon must have at least 3 points.
+        # Coordinates can be generated at https://www.image-map.net/
+        ####################
+        coordinates:
+          - 545,1077
+          - 747,939
+          - 788,805
+        ################
+        # Zone level object filters. These are applied in addition to the global and camera filters
+        # and should be more restrictive than the global and camera filters. The global and camera
+        # filters are applied upstream.
+        ################
+        filters:
+          person:
+            min_area: 5000
+            max_area: 100000
+            threshold: 0.5
+  driveway:
+    cameras:
+      front_door:
+        coordinates: 545,1077,747,939,788,805
+  yard:
+
 cameras:
   back:
     ffmpeg:
@@ -137,6 +172,7 @@ cameras:
     ################
     snapshots:
       show_timestamp: True
+      draw_zones: False
 
     ################
     # Camera level object config. This config is merged with the global config above.

detect_objects.py

@@ -171,7 +171,8 @@ def main():
     ##
     for name, config in CONFIG['cameras'].items():
         config['snapshots'] = {
-            'show_timestamp': config.get('snapshots', {}).get('show_timestamp', True)
+            'show_timestamp': config.get('snapshots', {}).get('show_timestamp', True),
+            'draw_zones': config.get('snapshots', {}).get('draw_zones', False)
         }
 
     # Queue for cameras to push tracked objects to
@@ -264,8 +265,8 @@ def main():
 
     event_processor = EventProcessor(CONFIG['cameras'], camera_processes, '/cache', '/clips', event_queue)
     event_processor.start()
-
-    object_processor = TrackedObjectProcessor(CONFIG['cameras'], client, MQTT_TOPIC_PREFIX, tracked_objects_queue, event_queue)
+    
+    object_processor = TrackedObjectProcessor(CONFIG['cameras'], CONFIG.get('zones', {}), client, MQTT_TOPIC_PREFIX, tracked_objects_queue, event_queue)
     object_processor.start()
     
     camera_watchdog = CameraWatchdog(camera_processes, CONFIG['cameras'], tflite_process, tracked_objects_queue, plasma_process)

frigate/object_processing.py

@@ -27,10 +27,34 @@ def filter_false_positives(event):
         return True
     return False
 
+def zone_filtered(obj, object_config):
+    object_name = obj['label']
+    object_filters = object_config.get('filters', {})
+
+    if object_name in object_filters:
+        obj_settings = object_filters[object_name]
+
+        # if the min area is larger than the
+        # detected object, don't add it to detected objects
+        if obj_settings.get('min_area',-1) > obj['area']:
+            return True
+        
+        # if the detected object is larger than the
+        # max area, don't add it to detected objects
+        if obj_settings.get('max_area', 24000000) < obj['area']:
+            return True
+
+        # if the score is lower than the threshold, skip
+        if obj_settings.get('threshold', 0) > obj['score']:
+            return True
+        
+    return False
+
 class TrackedObjectProcessor(threading.Thread):
-    def __init__(self, config, client, topic_prefix, tracked_objects_queue, event_queue):
+    def __init__(self, camera_config, zone_config, client, topic_prefix, tracked_objects_queue, event_queue):
         threading.Thread.__init__(self)
-        self.config = config
+        self.camera_config = camera_config
+        self.zone_config = zone_config
         self.client = client
         self.topic_prefix = topic_prefix
         self.tracked_objects_queue = tracked_objects_queue
@@ -43,6 +67,28 @@ class TrackedObjectProcessor(threading.Thread):
             'current_frame_time': 0.0,
             'object_id': None
         })
+        self.zone_data = defaultdict(lambda: {
+            'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')),
+            'contours': {}
+        })
+
+        # create zone contours
+        for name, config in zone_config.items():
+            for camera, camera_zone_config in config.items():
+                coordinates = camera_zone_config['coordinates']
+                if isinstance(coordinates, list):
+                    self.zone_data[name]['contours'][camera] =  np.array([[int(p.split(',')[0]), int(p.split(',')[1])] for p in coordinates])
+                elif isinstance(coordinates, str):
+                    points = coordinates.split(',')
+                    self.zone_data[name]['contours'][camera] =  np.array([[int(points[i]), int(points[i+1])] for i in range(0, len(points), 2)])
+                else:
+                    print(f"Unable to parse zone coordinates for {name} - {camera}")
+        
+        # set colors for zones
+        colors = plt.cm.get_cmap('tab10', len(self.zone_data.keys()))
+        for i, zone in enumerate(self.zone_data.values()):
+            zone['color'] = tuple(int(round(255 * c)) for c in colors(i)[:3])
+
         self.plasma_client = PlasmaManager()
         
     def get_best(self, camera, label):
@@ -58,7 +104,7 @@ class TrackedObjectProcessor(threading.Thread):
         while True:
             camera, frame_time, current_tracked_objects = self.tracked_objects_queue.get()
 
-            config = self.config[camera]
+            camera_config = self.camera_config[camera]
             best_objects = self.camera_data[camera]['best_objects']
             current_object_status = self.camera_data[camera]['object_status']
             tracked_objects = self.camera_data[camera]['tracked_objects']
@@ -89,6 +135,17 @@ class TrackedObjectProcessor(threading.Thread):
 
             self.camera_data[camera]['current_frame_time'] = frame_time
 
+            # build a dict of objects in each zone for current camera
+            current_objects_in_zones = defaultdict(lambda: [])
+            for obj in tracked_objects.values():
+                bottom_center = (obj['centroid'][0], obj['box'][3])
+                # check each zone
+                for name, zone in self.zone_data.items():
+                    # check each camera with a contour for the zone
+                    for camera, contour in zone['contours'].items():
+                        if cv2.pointPolygonTest(contour, bottom_center, False) >= 0 and not zone_filtered(obj, self.zone_config[name][camera].get('filters', {})):
+                            current_objects_in_zones[name].append(obj['label'])
+
             ###
             # Draw tracked objects on the frame
             ###
@@ -111,10 +168,16 @@ class TrackedObjectProcessor(threading.Thread):
                     region = obj['region']
                     cv2.rectangle(current_frame, (region[0], region[1]), (region[2], region[3]), (0,255,0), 1)
                 
-                if config['snapshots']['show_timestamp']:
+                if camera_config['snapshots']['show_timestamp']:
                     time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
                     cv2.putText(current_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
 
+                if camera_config['snapshots']['draw_zones']:
+                    for name, zone in self.zone_data.items():
+                        thickness = 2 if len(current_objects_in_zones[name]) == 0 else 8
+                        if camera in zone['contours']:
+                            cv2.drawContours(current_frame, [zone['contours'][camera]], -1, zone['color'], thickness)
+
                 ###
                 # Set the current frame
                 ###
@@ -152,7 +215,26 @@ class TrackedObjectProcessor(threading.Thread):
             ###
             # Report over MQTT
             ###
-            # count objects by type
+            
+            # get the zones that are relevant for this camera
+            relevant_zones = [zone for zone, config in self.zone_config.items() if camera in config]
+            # for each zone
+            for zone in relevant_zones:
+                # create the set of labels in the current frame and previously reported
+                labels_for_zone = set(current_objects_in_zones[zone] + list(self.zone_data[zone]['object_status'][camera].keys()))
+                # for each label
+                for label in labels_for_zone:
+                    # compute the current 'ON' vs 'OFF' status by checking if any camera sees the object in the zone
+                    previous_state = any([camera[label] == 'ON' for camera in self.zone_data[zone]['object_status'].values()])
+                    self.zone_data[zone]['object_status'][camera][label] = 'ON' if label in current_objects_in_zones[zone] else 'OFF'
+                    new_state = any([camera[label] == 'ON' for camera in self.zone_data[zone]['object_status'].values()])
+                    # if the value is changing, send over MQTT
+                    if previous_state == False and new_state == True:
+                        self.client.publish(f"{self.topic_prefix}/{zone}/{label}", 'ON', retain=False)
+                    elif previous_state == True and new_state == False:
+                        self.client.publish(f"{self.topic_prefix}/{zone}/{label}", 'OFF', retain=False)
+
+            # count  by type
             obj_counter = Counter()
             for obj in tracked_objects.values():
                 obj_counter[obj['label']] += 1