add retention settings for snapshots

frigate/config.py

@@ -43,7 +43,7 @@ MQTT_SCHEMA = vol.Schema(
     }
 )
 
-CLIPS_RETAIN_SCHEMA = vol.Schema(
+RETAIN_SCHEMA = vol.Schema(
     {
         vol.Required('default',default=10): int,
         'objects': {
@@ -56,7 +56,7 @@ CLIPS_SCHEMA = vol.Schema(
     {
         vol.Optional('max_seconds', default=300): int,
         'tmpfs_cache_size': str,
-        vol.Optional('retain', default={}): CLIPS_RETAIN_SCHEMA
+        vol.Optional('retain', default={}): RETAIN_SCHEMA
     }
 )
 
@@ -183,7 +183,7 @@ CAMERAS_SCHEMA = vol.Schema(vol.All(
                 vol.Optional('pre_capture', default=5): int,
                 vol.Optional('post_capture', default=5): int,
                 'objects': [str],
-                vol.Optional('retain', default={}): CLIPS_RETAIN_SCHEMA,
+                vol.Optional('retain', default={}): RETAIN_SCHEMA,
             },
             vol.Optional('record', default={}): {
                 'enabled': bool,
@@ -197,7 +197,8 @@ CAMERAS_SCHEMA = vol.Schema(vol.All(
                 vol.Optional('timestamp', default=False): bool,
                 vol.Optional('bounding_box', default=False): bool,
                 vol.Optional('crop', default=False): bool,
-                'height': int
+                'height': int,
+                vol.Optional('retain', default={}): RETAIN_SCHEMA,
             },
             vol.Optional('mqtt', default={}): {
                 vol.Optional('enabled', default=True): bool,
@@ -228,6 +229,9 @@ FRIGATE_CONFIG_SCHEMA = vol.Schema(
             vol.Optional('default', default='info'): vol.In(['info', 'debug', 'warning', 'error', 'critical']),
             vol.Optional('logs', default={}): {str: vol.In(['info', 'debug', 'warning', 'error', 'critical']) }
         },
+        vol.Optional('snapshots', default={}): {
+            vol.Optional('retain', default={}): RETAIN_SCHEMA
+        },
         vol.Optional('clips', default={}): CLIPS_SCHEMA,
         vol.Optional('record', default={}): {
             vol.Optional('enabled', default=False): bool,
@@ -406,7 +410,7 @@ class CameraFfmpegConfig():
     def output_args(self):
         return {k: v if isinstance(v, list) else v.split(' ') for k, v in self._output_args.items()}
 
-class ClipsRetainConfig():
+class RetainConfig():
     def __init__(self, global_config, config):
         self._default = config.get('default', global_config.get('default'))
         self._objects = config.get('objects', global_config.get('objects', {}))
@@ -429,7 +433,7 @@ class ClipsConfig():
     def __init__(self, config):
         self._max_seconds = config['max_seconds']
         self._tmpfs_cache_size = config.get('tmpfs_cache_size', '').strip()
-        self._retain = ClipsRetainConfig(config['retain'], config['retain'])
+        self._retain = RetainConfig(config['retain'], config['retain'])
     
     @property
     def max_seconds(self):
@@ -523,12 +527,13 @@ class ObjectConfig():
         }
 
 class CameraSnapshotsConfig():
-    def __init__(self, config):
+    def __init__(self, global_config, config):
         self._enabled = config['enabled']
         self._timestamp = config['timestamp']
         self._bounding_box = config['bounding_box']
         self._crop = config['crop']
         self._height = config.get('height')
+        self._retain = RetainConfig(global_config['snapshots']['retain'], config['retain'])
     
     @property
     def enabled(self):
@@ -550,13 +555,18 @@ class CameraSnapshotsConfig():
     def height(self):
         return self._height
     
+    @property
+    def retain(self):
+        return self._retain
+    
     def to_dict(self):
         return {
             'enabled': self.enabled,
             'timestamp': self.timestamp,
             'bounding_box': self.bounding_box,
             'crop': self.crop,
-            'height': self.height
+            'height': self.height,
+            'retain': self.retain.to_dict()
         }
 
 class CameraMqttConfig():
@@ -602,7 +612,7 @@ class CameraClipsConfig():
         self._pre_capture = config['pre_capture']
         self._post_capture = config['post_capture']
         self._objects = config.get('objects', global_config['objects']['track'])
-        self._retain = ClipsRetainConfig(global_config['clips']['retain'], config['retain'])
+        self._retain = RetainConfig(global_config['clips']['retain'], config['retain'])
     
     @property
     def enabled(self):
@@ -758,7 +768,7 @@ class CameraConfig():
         self._clips = CameraClipsConfig(global_config, config['clips'])
         self._record = RecordConfig(global_config['record'], config['record'])
         self._rtmp = CameraRtmpConfig(global_config, config['rtmp'])
-        self._snapshots = CameraSnapshotsConfig(config['snapshots'])
+        self._snapshots = CameraSnapshotsConfig(global_config, config['snapshots'])
         self._mqtt = CameraMqttConfig(config['mqtt'])
         self._objects = ObjectConfig(global_config['objects'], config.get('objects', {}))
         self._motion = MotionConfig(global_config['motion'], config['motion'], self._height)

frigate/events.py

@@ -200,30 +200,60 @@ class EventCleanup(threading.Thread):
         self.name = 'event_cleanup'
         self.config = config
         self.stop_event = stop_event
+        self.camera_keys = list(self.config.cameras.keys())
 
-    def run(self):
-        counter = 0
-        while(True):
-            if self.stop_event.is_set():
-                logger.info(f"Exiting event cleanup...")
-                break
-
-            # only expire events every 10 minutes, but check for stop events every 10 seconds
-            time.sleep(10)
-            counter = counter + 1
-            if counter < 60:
-                continue
-            counter = 0
-
-            camera_keys = list(self.config.cameras.keys())
-
-            # Expire events from unlisted cameras based on the global config
+    def expire(self, media):
+        ## Expire events from unlisted cameras based on the global config
+        if media == 'clips':
             retain_config = self.config.clips.retain
-            
+            file_extension = 'mp4'
+            update_params = {'has_clip': False}
+        else:
+            retain_config = self.config.snapshots.retain
+            file_extension = 'jpg'
+            update_params = {'has_snapshot': False}
+        
+        distinct_labels = (Event.select(Event.label)
+                    .where(Event.camera.not_in(self.camera_keys))
+                    .distinct())
+        
+        # loop over object types in db
+        for l in distinct_labels:
+            # get expiration time for this label
+            expire_days = retain_config.objects.get(l.label, retain_config.default)
+            expire_after = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp()
+            # grab all events after specific time
+            expired_events = (
+                Event.select()
+                    .where(Event.camera.not_in(self.camera_keys), 
+                        Event.start_time < expire_after, 
+                        Event.label == l.label)
+            )
+            # delete the media from disk
+            for event in expired_events:
+                media_name = f"{event.camera}-{event.id}"
+                media = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}")
+                media.unlink(missing_ok=True)
+            # update the clips attribute for the db entry
+            update_query = (
+                Event.update(update_params)
+                    .where(Event.camera.not_in(self.camera_keys), 
+                        Event.start_time < expire_after, 
+                        Event.label == l.label)
+            )
+            update_query.execute()
+
+        ## Expire events from cameras based on the camera config
+        for name, camera in self.config.cameras.items():
+            if media == 'clips':
+                retain_config = camera.clips.retain
+            else:
+                retain_config = camera.snapshots.retain
+            # get distinct objects in database for this camera
             distinct_labels = (Event.select(Event.label)
-                        .where(Event.camera.not_in(camera_keys))
-                        .distinct())
-            
+                    .where(Event.camera == name)
+                    .distinct())
+
             # loop over object types in db
             for l in distinct_labels:
                 # get expiration time for this label
@@ -232,54 +262,45 @@ class EventCleanup(threading.Thread):
                 # grab all events after specific time
                 expired_events = (
                     Event.select()
-                        .where(Event.camera.not_in(camera_keys), 
+                        .where(Event.camera == name, 
                             Event.start_time < expire_after, 
                             Event.label == l.label)
                 )
                 # delete the grabbed clips from disk
                 for event in expired_events:
-                    clip_name = f"{event.camera}-{event.id}"
-                    clip = Path(f"{os.path.join(CLIPS_DIR, clip_name)}.mp4")
-                    clip.unlink(missing_ok=True)
-                # delete the event for this type from the db
-                delete_query = (
-                    Event.delete()
-                        .where(Event.camera.not_in(camera_keys), 
+                    media_name = f"{event.camera}-{event.id}"
+                    media = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}")
+                    media.unlink(missing_ok=True)
+                # update the clips attribute for the db entry
+                update_query = (
+                    Event.update(update_params)
+                        .where( Event.camera == name, 
                             Event.start_time < expire_after, 
                             Event.label == l.label)
                 )
-                delete_query.execute()
+                update_query.execute()
+    
+    def run(self):
+        counter = 0
+        while(True):
+            if self.stop_event.is_set():
+                logger.info(f"Exiting event cleanup...")
+                break
 
-            # Expire events from cameras based on the camera config
-            for name, camera in self.config.cameras.items():
-                retain_config = camera.clips.retain
-                # get distinct objects in database for this camera
-                distinct_labels = (Event.select(Event.label)
-                        .where(Event.camera == name)
-                        .distinct())
+            # only expire events every 10 minutes, but check for stop events every 10 seconds
+            time.sleep(10)
+            counter = counter + 1
+            if counter < 60:
+                continue
+            counter = 0
 
-                # loop over object types in db
-                for l in distinct_labels:
-                    # get expiration time for this label
-                    expire_days = retain_config.objects.get(l.label, retain_config.default)
-                    expire_after = (datetime.datetime.now() - datetime.timedelta(days=expire_days)).timestamp()
-                    # grab all events after specific time
-                    expired_events = (
-                        Event.select()
-                            .where(Event.camera == name, 
-                                Event.start_time < expire_after, 
-                                Event.label == l.label)
-                    )
-                    # delete the grabbed clips from disk
-                    for event in expired_events:
-                        clip_name = f"{event.camera}-{event.id}"
-                        clip = Path(f"{os.path.join(CLIPS_DIR, clip_name)}.mp4")
-                        clip.unlink(missing_ok=True)
-                    # delete the event for this type from the db
-                    delete_query = (
-                        Event.delete()
-                            .where( Event.camera == name, 
-                                Event.start_time < expire_after, 
-                                Event.label == l.label)
-                    )
-                    delete_query.execute()
+            self.expire('clips')
+            self.expire('snapshots')
+
+            # drop events from db where has_clip and has_snapshot are false
+            delete_query = (
+                Event.delete()
+                    .where( Event.has_clip == False, 
+                        Event.has_snapshot == False)
+            )
+            delete_query.execute()

frigate/test/test_config.py

@@ -325,7 +325,7 @@ class TestConfig(TestCase):
                     'ffmpeg': {
                         'inputs': [
                             { 'path': 'rtsp://10.0.0.1:554/video', 'roles': ['detect', 'rtmp'] },
-                            { 'path': 'rtsp://10.0.0.1:554/clips', 'roles': ['clips'] }
+                            { 'path': 'rtsp://10.0.0.1:554/record', 'roles': ['record'] }
                         ]
                     },
                     'height': 1080,