Image entity media source (#104743)

* Image entity media source

* MJPEG streaming

* Update on change rather than fixed interval

* Only send boundary twice

* return when image has no data

* Write each frame twice

* Use friendly name when browsing

* Fix sending of double frame

* Initial image proxy test

* Improve proxy stream test

* Refactor

* Code review fixes

homeassistant/components/image/__init__.py

@@ -15,7 +15,7 @@ import httpx
 
 from homeassistant.components.http import KEY_AUTHENTICATED, HomeAssistantView
 from homeassistant.config_entries import ConfigEntry
-from homeassistant.const import EVENT_HOMEASSISTANT_STOP
+from homeassistant.const import CONTENT_TYPE_MULTIPART, EVENT_HOMEASSISTANT_STOP
 from homeassistant.core import Event, HomeAssistant, callback
 from homeassistant.exceptions import HomeAssistantError
 from homeassistant.helpers.config_validation import (  # noqa: F401
@@ -24,9 +24,13 @@ from homeassistant.helpers.config_validation import (  # noqa: F401
 )
 from homeassistant.helpers.entity import Entity, EntityDescription
 from homeassistant.helpers.entity_component import EntityComponent
-from homeassistant.helpers.event import async_track_time_interval
+from homeassistant.helpers.event import (
+    EventStateChangedData,
+    async_track_state_change_event,
+    async_track_time_interval,
+)
 from homeassistant.helpers.httpx_client import get_async_client
-from homeassistant.helpers.typing import UNDEFINED, ConfigType, UndefinedType
+from homeassistant.helpers.typing import UNDEFINED, ConfigType, EventType, UndefinedType
 
 from .const import DOMAIN, IMAGE_TIMEOUT  # noqa: F401
 
@@ -49,6 +53,10 @@ _RND: Final = SystemRandom()
 
 GET_IMAGE_TIMEOUT: Final = 10
 
+FRAME_BOUNDARY = "frame-boundary"
+FRAME_SEPARATOR = bytes(f"\r\n--{FRAME_BOUNDARY}\r\n", "utf-8")
+LAST_FRAME_MARKER = bytes(f"\r\n--{FRAME_BOUNDARY}--\r\n", "utf-8")
+
 
 class ImageEntityDescription(EntityDescription, frozen_or_thawed=True):
     """A class that describes image entities."""
@@ -92,6 +100,7 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
     )
 
     hass.http.register_view(ImageView(component))
+    hass.http.register_view(ImageStreamView(component))
 
     await component.async_setup(config)
 
@@ -295,3 +304,71 @@ class ImageView(HomeAssistantView):
             raise web.HTTPInternalServerError() from ex
 
         return web.Response(body=image.content, content_type=image.content_type)
+
+
+async def async_get_still_stream(
+    request: web.Request,
+    image_entity: ImageEntity,
+) -> web.StreamResponse:
+    """Generate an HTTP multipart stream from the Image."""
+    response = web.StreamResponse()
+    response.content_type = CONTENT_TYPE_MULTIPART.format(FRAME_BOUNDARY)
+    await response.prepare(request)
+
+    async def _write_frame() -> bool:
+        img_bytes = await image_entity.async_image()
+        if img_bytes is None:
+            await response.write(LAST_FRAME_MARKER)
+            return False
+        frame = bytearray(FRAME_SEPARATOR)
+        header = bytes(
+            f"Content-Type: {image_entity.content_type}\r\n"
+            f"Content-Length: {len(img_bytes)}\r\n\r\n",
+            "utf-8",
+        )
+        frame.extend(header)
+        frame.extend(img_bytes)
+        # Chrome shows the n-1 frame so send the frame twice
+        # https://issues.chromium.org/issues/41199053
+        # https://issues.chromium.org/issues/40791855
+        # While this results in additional bandwidth usage,
+        # given the low frequency of image updates, it is acceptable.
+        frame.extend(frame)
+        await response.write(frame)
+        # Drain to ensure that the latest frame is available to the client
+        await response.drain()
+        return True
+
+    event = asyncio.Event()
+
+    async def image_state_update(_event: EventType[EventStateChangedData]) -> None:
+        """Write image to stream."""
+        event.set()
+
+    hass: HomeAssistant = request.app["hass"]
+    remove = async_track_state_change_event(
+        hass,
+        image_entity.entity_id,
+        image_state_update,
+    )
+    try:
+        while True:
+            if not await _write_frame():
+                return response
+            await event.wait()
+            event.clear()
+    finally:
+        remove()
+
+
+class ImageStreamView(ImageView):
+    """Image View to serve an multipart stream."""
+
+    url = "/api/image_proxy_stream/{entity_id}"
+    name = "api:image:stream"
+
+    async def handle(
+        self, request: web.Request, image_entity: ImageEntity
+    ) -> web.StreamResponse:
+        """Serve image stream."""
+        return await async_get_still_stream(request, image_entity)

homeassistant/components/image/media_source.py

@@ -0,0 +1,84 @@
+"""Expose iamges as media sources."""
+from __future__ import annotations
+
+from typing import cast
+
+from homeassistant.components.media_player import BrowseError, MediaClass
+from homeassistant.components.media_source.error import Unresolvable
+from homeassistant.components.media_source.models import (
+    BrowseMediaSource,
+    MediaSource,
+    MediaSourceItem,
+    PlayMedia,
+)
+from homeassistant.const import ATTR_FRIENDLY_NAME
+from homeassistant.core import HomeAssistant, State
+from homeassistant.helpers.entity_component import EntityComponent
+
+from . import ImageEntity
+from .const import DOMAIN
+
+
+async def async_get_media_source(hass: HomeAssistant) -> ImageMediaSource:
+    """Set up image media source."""
+    return ImageMediaSource(hass)
+
+
+class ImageMediaSource(MediaSource):
+    """Provide images as media sources."""
+
+    name: str = "Image"
+
+    def __init__(self, hass: HomeAssistant) -> None:
+        """Initialize ImageMediaSource."""
+        super().__init__(DOMAIN)
+        self.hass = hass
+
+    async def async_resolve_media(self, item: MediaSourceItem) -> PlayMedia:
+        """Resolve media to a url."""
+        component: EntityComponent[ImageEntity] = self.hass.data[DOMAIN]
+        image = component.get_entity(item.identifier)
+
+        if not image:
+            raise Unresolvable(f"Could not resolve media item: {item.identifier}")
+
+        return PlayMedia(
+            f"/api/image_proxy_stream/{image.entity_id}", image.content_type
+        )
+
+    async def async_browse_media(
+        self,
+        item: MediaSourceItem,
+    ) -> BrowseMediaSource:
+        """Return media."""
+        if item.identifier:
+            raise BrowseError("Unknown item")
+
+        component: EntityComponent[ImageEntity] = self.hass.data[DOMAIN]
+        children = [
+            BrowseMediaSource(
+                domain=DOMAIN,
+                identifier=image.entity_id,
+                media_class=MediaClass.VIDEO,
+                media_content_type=image.content_type,
+                title=cast(State, self.hass.states.get(image.entity_id)).attributes.get(
+                    ATTR_FRIENDLY_NAME, image.name
+                ),
+                thumbnail=f"/api/image_proxy/{image.entity_id}",
+                can_play=True,
+                can_expand=False,
+            )
+            for image in component.entities
+        ]
+
+        return BrowseMediaSource(
+            domain=DOMAIN,
+            identifier=None,
+            media_class=MediaClass.APP,
+            media_content_type="",
+            title="Image",
+            can_play=False,
+            can_expand=True,
+            children_media_class=MediaClass.IMAGE,
+            children=children,
+        )

tests/components/image/test_init.py

@@ -1,4 +1,5 @@
 """The tests for the image component."""
+import datetime
 from http import HTTPStatus
 import ssl
 from unittest.mock import MagicMock, patch
@@ -287,3 +288,39 @@ async def test_fetch_image_url_wrong_content_type(
 
     resp = await client.get("/api/image_proxy/image.test")
     assert resp.status == HTTPStatus.INTERNAL_SERVER_ERROR
+
+
+async def test_image_stream(
+    hass: HomeAssistant, hass_client: ClientSessionGenerator
+) -> None:
+    """Test image stream."""
+
+    mock_integration(hass, MockModule(domain="test"))
+    mock_image = MockURLImageEntity(hass)
+    mock_platform(hass, "test.image", MockImagePlatform([mock_image]))
+    assert await async_setup_component(
+        hass, image.DOMAIN, {"image": {"platform": "test"}}
+    )
+    await hass.async_block_till_done()
+
+    client = await hass_client()
+
+    with patch.object(mock_image, "async_image", return_value=b""):
+        resp = await client.get("/api/image_proxy_stream/image.test")
+        assert not resp.closed
+        assert resp.status == HTTPStatus.OK
+
+        mock_image.image_last_updated = datetime.datetime.now()
+        mock_image.async_write_ha_state()
+        # Two blocks to ensure the frame is written
+        await hass.async_block_till_done()
+        await hass.async_block_till_done()
+
+    with patch.object(mock_image, "async_image", return_value=None):
+        mock_image.image_last_updated = datetime.datetime.now()
+        mock_image.async_write_ha_state()
+        # Two blocks to ensure the frame is written
+        await hass.async_block_till_done()
+        await hass.async_block_till_done()
+
+    assert resp.closed

tests/components/image/test_media_source.py

@@ -0,0 +1,45 @@
+"""Test image media source."""
+
+import pytest
+
+from homeassistant.components import media_source
+from homeassistant.core import HomeAssistant
+from homeassistant.setup import async_setup_component
+
+
+@pytest.fixture(autouse=True)
+async def setup_media_source(hass):
+    """Set up media source."""
+    assert await async_setup_component(hass, "media_source", {})
+
+
+async def test_browsing(hass: HomeAssistant, mock_image_platform) -> None:
+    """Test browsing image media source."""
+    item = await media_source.async_browse_media(hass, "media-source://image")
+    assert item is not None
+    assert item.title == "Image"
+    assert len(item.children) == 1
+    assert item.children[0].media_content_type == "image/jpeg"
+
+
+async def test_resolving(hass: HomeAssistant, mock_image_platform) -> None:
+    """Test resolving."""
+    item = await media_source.async_resolve_media(
+        hass, "media-source://image/image.test", None
+    )
+    assert item is not None
+    assert item.url == "/api/image_proxy_stream/image.test"
+    assert item.mime_type == "image/jpeg"
+
+
+async def test_resolving_non_existing_camera(
+    hass: HomeAssistant, mock_image_platform
+) -> None:
+    """Test resolving."""
+    with pytest.raises(
+        media_source.Unresolvable,
+        match="Could not resolve media item: image.non_existing",
+    ):
+        await media_source.async_resolve_media(
+            hass, "media-source://image/image.non_existing", None
+        )