Ensure a wal checkpoint is scheduled nightly (#50746)

homeassistant/components/recorder/__init__.py

@@ -54,6 +54,7 @@ from .util import (
     dburl_to_path,
     end_incomplete_runs,
     move_away_broken_database,
+    perodic_db_cleanups,
     session_scope,
     setup_connection_for_dialect,
     validate_or_move_away_sqlite_database,
@@ -278,6 +279,10 @@ class PurgeTask(NamedTuple):
     apply_filter: bool
 
 
+class PerodicCleanupTask:
+    """An object to insert into the recorder to trigger cleanup tasks when auto purge is disabled."""
+
+
 class StatisticsTask(NamedTuple):
     """An object to insert into the recorder queue to run a statistics task."""
 
@@ -484,9 +489,15 @@ class Recorder(threading.Thread):
         self.async_recorder_ready.set()
 
     @callback
-    def async_purge(self, now):
+    def async_nightly_tasks(self, now):
         """Trigger the purge."""
-        self.queue.put(PurgeTask(self.keep_days, repack=False, apply_filter=False))
+        if self.auto_purge:
+            # Purge will schedule the perodic cleanups
+            # after it completes to ensure it does not happen
+            # until after the database is vacuumed
+            self.queue.put(PurgeTask(self.keep_days, repack=False, apply_filter=False))
+        else:
+            self.queue.put(PerodicCleanupTask())
 
     @callback
     def async_hourly_statistics(self, now):
@@ -496,11 +507,10 @@ class Recorder(threading.Thread):
 
     def _async_setup_periodic_tasks(self):
         """Prepare periodic tasks."""
-        if self.auto_purge:
-            # Purge every night at 4:12am
-            async_track_time_change(
-                self.hass, self.async_purge, hour=4, minute=12, second=0
-            )
+        # Run nightly tasks at 4:12am
+        async_track_time_change(
+            self.hass, self.async_nightly_tasks, hour=4, minute=12, second=0
+        )
         # Compile hourly statistics every hour at *:12
         async_track_time_change(
             self.hass, self.async_hourly_statistics, minute=12, second=0
@@ -646,6 +656,10 @@ class Recorder(threading.Thread):
     def _run_purge(self, keep_days, repack, apply_filter):
         """Purge the database."""
         if purge.purge_old_data(self, keep_days, repack, apply_filter):
+            # We always need to do the db cleanups after a purge
+            # is finished to ensure the WAL checkpoint and other
+            # tasks happen after a vacuum.
+            perodic_db_cleanups(self)
             return
         # Schedule a new purge task if this one didn't finish
         self.queue.put(PurgeTask(keep_days, repack, apply_filter))
@@ -662,6 +676,9 @@ class Recorder(threading.Thread):
         if isinstance(event, PurgeTask):
             self._run_purge(event.keep_days, event.repack, event.apply_filter)
             return
+        if isinstance(event, PerodicCleanupTask):
+            perodic_db_cleanups(self)
+            return
         if isinstance(event, StatisticsTask):
             self._run_statistics(event.start)
             return

homeassistant/components/recorder/util.py

@@ -318,3 +318,15 @@ def retryable_database_job(description: str):
         return wrapper
 
     return decorator
+
+
+def perodic_db_cleanups(instance: Recorder):
+    """Run any database cleanups that need to happen perodiclly.
+
+    These cleanups will happen nightly or after any purge.
+    """
+
+    if instance.engine.dialect.name == "sqlite":
+        # Execute sqlite to create a wal checkpoint and free up disk space
+        _LOGGER.debug("WAL checkpoint")
+        instance.engine.execute("PRAGMA wal_checkpoint(TRUNCATE);")

tests/components/recorder/test_init.py

@@ -8,6 +8,7 @@ from sqlalchemy.exc import DatabaseError, OperationalError, SQLAlchemyError
 
 from homeassistant.components import recorder
 from homeassistant.components.recorder import (
+    CONF_AUTO_PURGE,
     CONF_DB_URL,
     CONFIG_SCHEMA,
     DOMAIN,
@@ -610,30 +611,73 @@ def test_auto_purge(hass_recorder):
 
     with patch(
         "homeassistant.components.recorder.purge.purge_old_data", return_value=True
-    ) as purge_old_data:
+    ) as purge_old_data, patch(
+        "homeassistant.components.recorder.perodic_db_cleanups"
+    ) as perodic_db_cleanups:
         # Advance one day, and the purge task should run
         test_time = test_time + timedelta(days=1)
         run_tasks_at_time(hass, test_time)
         assert len(purge_old_data.mock_calls) == 1
+        assert len(perodic_db_cleanups.mock_calls) == 1
 
         purge_old_data.reset_mock()
+        perodic_db_cleanups.reset_mock()
 
         # Advance one day, and the purge task should run again
         test_time = test_time + timedelta(days=1)
         run_tasks_at_time(hass, test_time)
         assert len(purge_old_data.mock_calls) == 1
+        assert len(perodic_db_cleanups.mock_calls) == 1
 
         purge_old_data.reset_mock()
+        perodic_db_cleanups.reset_mock()
 
         # Advance less than one full day.  The alarm should not yet fire.
         test_time = test_time + timedelta(hours=23)
         run_tasks_at_time(hass, test_time)
         assert len(purge_old_data.mock_calls) == 0
+        assert len(perodic_db_cleanups.mock_calls) == 0
 
         # Advance to the next day and fire the alarm again
         test_time = test_time + timedelta(hours=1)
         run_tasks_at_time(hass, test_time)
         assert len(purge_old_data.mock_calls) == 1
+        assert len(perodic_db_cleanups.mock_calls) == 1
+
+    dt_util.set_default_time_zone(original_tz)
+
+
+def test_auto_purge_disabled(hass_recorder):
+    """Test periodic db cleanup still run when auto purge is disabled."""
+    hass = hass_recorder({CONF_AUTO_PURGE: False})
+
+    original_tz = dt_util.DEFAULT_TIME_ZONE
+
+    tz = dt_util.get_time_zone("Europe/Copenhagen")
+    dt_util.set_default_time_zone(tz)
+
+    # Purging is scheduled to happen at 4:12am every day. We want
+    # to verify that when auto purge is disabled perodic db cleanups
+    # are still scheduled
+    #
+    # The clock is started at 4:15am then advanced forward below
+    now = dt_util.utcnow()
+    test_time = datetime(now.year + 2, 1, 1, 4, 15, 0, tzinfo=tz)
+    run_tasks_at_time(hass, test_time)
+
+    with patch(
+        "homeassistant.components.recorder.purge.purge_old_data", return_value=True
+    ) as purge_old_data, patch(
+        "homeassistant.components.recorder.perodic_db_cleanups"
+    ) as perodic_db_cleanups:
+        # Advance one day, and the purge task should run
+        test_time = test_time + timedelta(days=1)
+        run_tasks_at_time(hass, test_time)
+        assert len(purge_old_data.mock_calls) == 0
+        assert len(perodic_db_cleanups.mock_calls) == 1
+
+        purge_old_data.reset_mock()
+        perodic_db_cleanups.reset_mock()
 
     dt_util.set_default_time_zone(original_tz)
 

tests/components/recorder/test_util.py

@@ -269,3 +269,11 @@ def test_end_incomplete_runs(hass_recorder, caplog):
         assert run_info.end == now_without_tz
 
     assert "Ended unfinished session" in caplog.text
+
+
+def test_perodic_db_cleanups(hass_recorder):
+    """Test perodic db cleanups."""
+    hass = hass_recorder()
+    with patch.object(hass.data[DATA_INSTANCE].engine, "execute") as execute_mock:
+        util.perodic_db_cleanups(hass.data[DATA_INSTANCE])
+    assert execute_mock.call_args[0][0] == "PRAGMA wal_checkpoint(TRUNCATE);"