Add logs to Cloud component support package (#138230)

* Add logs to Cloud component support package

* Add section for logs

* Replace list with deque

* Copy the deque to avoid mutation during iteration

homeassistant/components/cloud/__init__.py

@@ -6,6 +6,7 @@ import asyncio
 from collections.abc import Awaitable, Callable
 from datetime import datetime, timedelta
 from enum import Enum
+import logging
 from typing import cast
 
 from hass_nabucasa import Cloud
@@ -19,6 +20,7 @@ from homeassistant.const import (
     CONF_NAME,
     CONF_REGION,
     EVENT_HOMEASSISTANT_STOP,
+    FORMAT_DATETIME,
     Platform,
 )
 from homeassistant.core import Event, HassJob, HomeAssistant, ServiceCall, callback
@@ -33,7 +35,7 @@ from homeassistant.helpers.dispatcher import (
 from homeassistant.helpers.event import async_call_later
 from homeassistant.helpers.service import async_register_admin_service
 from homeassistant.helpers.typing import ConfigType
-from homeassistant.loader import bind_hass
+from homeassistant.loader import async_get_integration, bind_hass
 from homeassistant.util.signal_type import SignalType
 
 # Pre-import backup to avoid it being imported
@@ -62,11 +64,13 @@ from .const import (
     CONF_THINGTALK_SERVER,
     CONF_USER_POOL_ID,
     DATA_CLOUD,
+    DATA_CLOUD_LOG_HANDLER,
     DATA_PLATFORMS_SETUP,
     DOMAIN,
     MODE_DEV,
     MODE_PROD,
 )
+from .helpers import FixedSizeQueueLogHandler
 from .prefs import CloudPreferences
 from .repairs import async_manage_legacy_subscription_issue
 from .subscription import async_subscription_info
@@ -245,6 +249,8 @@ def async_remote_ui_url(hass: HomeAssistant) -> str:
 
 async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
     """Initialize the Home Assistant cloud."""
+    log_handler = hass.data[DATA_CLOUD_LOG_HANDLER] = await _setup_log_handler(hass)
+
     # Process configs
     if DOMAIN in config:
         kwargs = dict(config[DOMAIN])
@@ -267,6 +273,8 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
     async def _shutdown(event: Event) -> None:
         """Shutdown event."""
         await cloud.stop()
+        logging.root.removeHandler(log_handler)
+        del hass.data[DATA_CLOUD_LOG_HANDLER]
 
     hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, _shutdown)
 
@@ -405,3 +413,19 @@ def _setup_services(hass: HomeAssistant, prefs: CloudPreferences) -> None:
     async_register_admin_service(
         hass, DOMAIN, SERVICE_REMOTE_DISCONNECT, _service_handler
     )
+
+
+async def _setup_log_handler(hass: HomeAssistant) -> FixedSizeQueueLogHandler:
+    fmt = (
+        "%(asctime)s.%(msecs)03d %(levelname)s (%(threadName)s) [%(name)s] %(message)s"
+    )
+    handler = FixedSizeQueueLogHandler()
+    handler.setFormatter(logging.Formatter(fmt, datefmt=FORMAT_DATETIME))
+
+    integration = await async_get_integration(hass, DOMAIN)
+    loggers: set[str] = {"snitun", integration.pkg_path, *(integration.loggers or [])}
+
+    for logger_name in loggers:
+        logging.getLogger(logger_name).addHandler(handler)
+
+    return handler

homeassistant/components/cloud/const.py

@@ -12,12 +12,14 @@ if TYPE_CHECKING:
     from hass_nabucasa import Cloud
 
     from .client import CloudClient
+    from .helpers import FixedSizeQueueLogHandler
 
 DOMAIN = "cloud"
 DATA_CLOUD: HassKey[Cloud[CloudClient]] = HassKey(DOMAIN)
 DATA_PLATFORMS_SETUP: HassKey[dict[str, asyncio.Event]] = HassKey(
     "cloud_platforms_setup"
 )
+DATA_CLOUD_LOG_HANDLER: HassKey[FixedSizeQueueLogHandler] = HassKey("cloud_log_handler")
 EVENT_CLOUD_EVENT = "cloud_event"
 
 REQUEST_TIMEOUT = 10

homeassistant/components/cloud/helpers.py

@@ -0,0 +1,31 @@
+"""Helpers for the cloud component."""
+
+from collections import deque
+import logging
+
+from homeassistant.core import HomeAssistant
+
+
+class FixedSizeQueueLogHandler(logging.Handler):
+    """Log handler to store messages, with auto rotation."""
+
+    MAX_RECORDS = 500
+
+    def __init__(self) -> None:
+        """Initialize a new LogHandler."""
+        super().__init__()
+        self._records: deque[logging.LogRecord] = deque(maxlen=self.MAX_RECORDS)
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Store log message."""
+        self._records.append(record)
+
+    async def get_logs(self, hass: HomeAssistant) -> list[str]:
+        """Get stored logs."""
+
+        def _get_logs() -> list[str]:
+            # copy the queue since it can mutate while iterating
+            records = self._records.copy()
+            return [self.format(record) for record in records]
+
+        return await hass.async_add_executor_job(_get_logs)

homeassistant/components/cloud/http_api.py

@@ -43,6 +43,7 @@ from .assist_pipeline import async_create_cloud_pipeline
 from .client import CloudClient
 from .const import (
     DATA_CLOUD,
+    DATA_CLOUD_LOG_HANDLER,
     EVENT_CLOUD_EVENT,
     LOGIN_MFA_TIMEOUT,
     PREF_ALEXA_REPORT_STATE,
@@ -397,8 +398,11 @@ class DownloadSupportPackageView(HomeAssistantView):
     url = "/api/cloud/support_package"
     name = "api:cloud:support_package"
 
-    def _generate_markdown(
-        self, hass_info: dict[str, Any], domains_info: dict[str, dict[str, str]]
+    async def _generate_markdown(
+        self,
+        hass: HomeAssistant,
+        hass_info: dict[str, Any],
+        domains_info: dict[str, dict[str, str]],
     ) -> str:
         def get_domain_table_markdown(domain_info: dict[str, Any]) -> str:
             if len(domain_info) == 0:
@@ -424,6 +428,17 @@ class DownloadSupportPackageView(HomeAssistantView):
                 "</details>\n\n"
             )
 
+        log_handler = hass.data[DATA_CLOUD_LOG_HANDLER]
+        logs = "\n".join(await log_handler.get_logs(hass))
+        markdown += (
+            "## Full logs\n\n"
+            "<details><summary>Logs</summary>\n\n"
+            "```logs\n"
+            f"{logs}\n"
+            "```\n\n"
+            "</details>\n"
+        )
+
         return markdown
 
     async def get(self, request: web.Request) -> web.Response:
@@ -433,7 +448,7 @@ class DownloadSupportPackageView(HomeAssistantView):
         domain_health = await get_system_health_info(hass)
 
         hass_info = domain_health.pop("homeassistant", {})
-        markdown = self._generate_markdown(hass_info, domain_health)
+        markdown = await self._generate_markdown(hass, hass_info, domain_health)
 
         return web.Response(
             body=markdown,

tests/components/cloud/snapshots/test_http_api.ambr

@@ -44,6 +44,17 @@
   
   </details>
   
+  ## Full logs
+  
+  <details><summary>Logs</summary>
+  
+  ```logs
+  2025-02-10 12:00:00.000 INFO (MainThread) [hass_nabucasa.iot] Hass nabucasa log
+  2025-02-10 12:00:00.000 WARNING (MainThread) [snitun.utils.aiohttp_client] Snitun log
+  2025-02-10 12:00:00.000 ERROR (MainThread) [homeassistant.components.cloud.client] Cloud log
+  ```
+  
+  </details>
   
   '''
 # ---

tests/components/cloud/test_http_api.py

@@ -2,12 +2,15 @@
 
 from collections.abc import Callable, Coroutine
 from copy import deepcopy
+import datetime
 from http import HTTPStatus
 import json
+import logging
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, Mock, PropertyMock, patch
 
 import aiohttp
+from freezegun.api import FrozenDateTimeFactory
 from hass_nabucasa import thingtalk
 from hass_nabucasa.auth import (
     InvalidTotpCode,
@@ -1869,15 +1872,18 @@ async def test_logout_view_dispatch_event(
     assert async_dispatcher_send_mock.mock_calls[0][1][2] == {"type": "logout"}
 
 
+@patch("homeassistant.components.cloud.helpers.FixedSizeQueueLogHandler.MAX_RECORDS", 3)
 async def test_download_support_package(
     hass: HomeAssistant,
     cloud: MagicMock,
     set_cloud_prefs: Callable[[dict[str, Any]], Coroutine[Any, Any, None]],
     hass_client: ClientSessionGenerator,
     aioclient_mock: AiohttpClientMocker,
+    freezer: FrozenDateTimeFactory,
     snapshot: SnapshotAssertion,
 ) -> None:
     """Test downloading a support package file."""
+
     aioclient_mock.get("https://cloud.bla.com/status", text="")
     aioclient_mock.get(
         "https://cert-server/directory", exc=Exception("Unexpected exception")
@@ -1936,6 +1942,16 @@ async def test_download_support_package(
         }
     )
 
+    now = dt_util.utcnow()
+    freezer.move_to(datetime.datetime.fromisoformat("2025-02-10T12:00:00.0+00:00"))
+    logging.getLogger("hass_nabucasa.iot").info(
+        "This message will be dropped since this test patches MAX_RECORDS"
+    )
+    logging.getLogger("hass_nabucasa.iot").info("Hass nabucasa log")
+    logging.getLogger("snitun.utils.aiohttp_client").warning("Snitun log")
+    logging.getLogger("homeassistant.components.cloud.client").error("Cloud log")
+    freezer.move_to(now)  # Reset time otherwise hass_client auth fails
+
     cloud_client = await hass_client()
     with (
         patch.object(hass.config, "config_dir", new="config"),