From 6d311a31ddc865cc45d95247ec4a76fcd2e89120 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Wed, 25 Mar 2020 15:08:20 -0500
Subject: [PATCH] =?UTF-8?q?Ensure=20recorder=20event=20loop=20recovers=20i?=
 =?UTF-8?q?f=20the=20database=20server=20dis=E2=80=A6=20(#33253)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If the database server disconnects there were exceptions
that were not trapped which would cause the recorder event
loop to collapse.  As we never want the loop to end
we trap exceptions broadly.

Fix a bug in the new commit interval setting which caused
it to always commit after 1s
---
 homeassistant/components/recorder/__init__.py | 47 +++++++++++++------
 1 file changed, 33 insertions(+), 14 deletions(-)

diff --git a/homeassistant/components/recorder/__init__.py b/homeassistant/components/recorder/__init__.py
index a662a457add..ffd37720053 100644
--- a/homeassistant/components/recorder/__init__.py
+++ b/homeassistant/components/recorder/__init__.py
@@ -342,7 +342,6 @@ class Recorder(threading.Thread):
         # has changed.  This reduces the disk io.
         while True:
             event = self.queue.get()
-
             if event is None:
                 self._close_run()
                 self._close_connection()
@@ -356,7 +355,7 @@ class Recorder(threading.Thread):
                 self.queue.task_done()
                 if self.commit_interval:
                     self._timechanges_seen += 1
-                    if self.commit_interval >= self._timechanges_seen:
+                    if self._timechanges_seen >= self.commit_interval:
                         self._timechanges_seen = 0
                         self._commit_event_session_or_retry()
                 continue
@@ -376,6 +375,9 @@ class Recorder(threading.Thread):
                 self.event_session.flush()
             except (TypeError, ValueError):
                 _LOGGER.warning("Event is not JSON serializable: %s", event)
+            except Exception as err:  # pylint: disable=broad-except
+                # Must catch the exception to prevent the loop from collapsing
+                _LOGGER.exception("Error adding event: %s", err)
 
             if dbevent and event.event_type == EVENT_STATE_CHANGED:
                 try:
@@ -387,6 +389,9 @@ class Recorder(threading.Thread):
                         "State is not JSON serializable: %s",
                         event.data.get("new_state"),
                     )
+                except Exception as err:  # pylint: disable=broad-except
+                    # Must catch the exception to prevent the loop from collapsing
+                    _LOGGER.exception("Error adding state change: %s", err)
 
             # If they do not have a commit interval
             # than we commit right away
@@ -404,17 +409,26 @@ class Recorder(threading.Thread):
             try:
                 self._commit_event_session()
                 return
-
-            except exc.OperationalError as err:
-                _LOGGER.error(
-                    "Error in database connectivity: %s. " "(retrying in %s seconds)",
-                    err,
-                    self.db_retry_wait,
-                )
+            except (exc.InternalError, exc.OperationalError) as err:
+                if err.connection_invalidated:
+                    _LOGGER.error(
+                        "Database connection invalidated: %s. "
+                        "(retrying in %s seconds)",
+                        err,
+                        self.db_retry_wait,
+                    )
+                else:
+                    _LOGGER.error(
+                        "Error in database connectivity: %s. "
+                        "(retrying in %s seconds)",
+                        err,
+                        self.db_retry_wait,
+                    )
                 tries += 1
 
-            except exc.SQLAlchemyError:
-                _LOGGER.exception("Error saving events")
+            except Exception as err:  # pylint: disable=broad-except
+                # Must catch the exception to prevent the loop from collapsing
+                _LOGGER.exception("Error saving events: %s", err)
                 return
 
         _LOGGER.error(
@@ -423,10 +437,15 @@ class Recorder(threading.Thread):
         )
         try:
             self.event_session.close()
-        except exc.SQLAlchemyError:
-            _LOGGER.exception("Failed to close event session.")
+        except Exception as err:  # pylint: disable=broad-except
+            # Must catch the exception to prevent the loop from collapsing
+            _LOGGER.exception("Error while closing event session: %s", err)
 
-        self.event_session = self.get_session()
+        try:
+            self.event_session = self.get_session()
+        except Exception as err:  # pylint: disable=broad-except
+            # Must catch the exception to prevent the loop from collapsing
+            _LOGGER.exception("Error while creating new event session: %s", err)
 
     def _commit_event_session(self):
         try: