Home-Assistant
/
core
mirror of https://github.com/home-assistant/core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update nest camera to pull still images from stream component (#66427) 

* Update nest to use stream thumbnail when it exists

* Update nest camera to always pull still image from stream

Update nest camera to always pull the still iamge from the stream component,
removing the use of the separate ffmpeg call, and removing use of the nest event
image. Image for events can now be pulled using the media source APIs, rather
than relying on the camera snapshot.

* Simplify a comment

* Remove more unused variables

* Simplify comments, image, and test code

* Remove assertions for placeholder images
pull/64892/head
Allen Porter 2022-02-14 21:45:09 -08:00 committed by GitHub
parent 94980399cf
commit 572fa7d055
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 55 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
homeassistant/components/nest/camera_sdm.py
View File

@ -8,20 +8,16 @@ import logging
from pathlib import Path
from google_nest_sdm.camera_traits import (
CameraEventImageTrait,
CameraImageTrait,
CameraLiveStreamTrait,
RtspStream,
StreamingProtocol,
)
from google_nest_sdm.device import Device
from google_nest_sdm.event_media import EventMedia
from google_nest_sdm.exceptions import ApiException
from haffmpeg.tools import IMAGE_JPEG
from homeassistant.components.camera import SUPPORT_STREAM, Camera
from homeassistant.components.camera.const import STREAM_TYPE_WEB_RTC
from homeassistant.components.ffmpeg import async_get_image
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import HomeAssistant
from homeassistant.exceptions import HomeAssistantError, PlatformNotReady
@ -207,30 +203,16 @@ class NestCamera(Camera):
self, width: int | None = None, height: int | None = None
) -> bytes | None:
"""Return bytes of camera image."""
if CameraEventImageTrait.NAME in self._device.traits:
# Returns the snapshot of the last event for ~30 seconds after the event
event_media: EventMedia | None = None
try:
event_media = (
await self._device.event_media_manager.get_active_event_media()
)
except ApiException as err:
_LOGGER.debug("Failure while getting image for event: %s", err)
if event_media:
return event_media.media.contents
# Fetch still image from the live stream
stream_url = await self.stream_source()
if not stream_url:
if self.frontend_stream_type != STREAM_TYPE_WEB_RTC:
return None
# Nest Web RTC cams only have image previews for events, and not
# for "now" by design to save batter, and need a placeholder.
if not self._placeholder_image:
self._placeholder_image = await self.hass.async_add_executor_job(
PLACEHOLDER.read_bytes
)
return self._placeholder_image
return await async_get_image(self.hass, stream_url, output_format=IMAGE_JPEG)
# Use the thumbnail from RTSP stream, or a placeholder if stream is
# not supported (e.g. WebRTC)
stream = await self.async_create_stream()
if stream:
return await stream.async_get_image(width, height)
if not self._placeholder_image:
self._placeholder_image = await self.hass.async_add_executor_job(
PLACEHOLDER.read_bytes
)
return self._placeholder_image
async def async_handle_web_rtc_offer(self, offer_sdp: str) -> str | None:
"""Return the source of the stream."""

254
tests/components/nest/test_camera_sdm.py
View File

@ -7,7 +7,7 @@ pubsub subscriber.
import datetime
from http import HTTPStatus
from unittest.mock import patch
from unittest.mock import AsyncMock, Mock, patch
import aiohttp
from google_nest_sdm.device import Device
@ -22,7 +22,6 @@ from homeassistant.components.camera import (
STREAM_TYPE_WEB_RTC,
)
from homeassistant.components.websocket_api.const import TYPE_RESULT
from homeassistant.exceptions import HomeAssistantError
from homeassistant.helpers import device_registry as dr, entity_registry as er
from homeassistant.setup import async_setup_component
from homeassistant.util.dt import utcnow
@ -54,9 +53,6 @@ DOMAIN = "nest"
MOTION_EVENT_ID = "FWWVQVUdGNUlTU2V4MGV2aTNXV..."
EVENT_SESSION_ID = "CjY5Y3VKaTZwR3o4Y19YbTVfMF..."
# Tests can assert that image bytes came from an event or was decoded
# from the live stream.
IMAGE_BYTES_FROM_EVENT = b"test url image bytes"
IMAGE_BYTES_FROM_STREAM = b"test stream image bytes"
TEST_IMAGE_URL = "https://domain/sdm_event_snapshot/dGTZwR3o4Y1..."
@ -116,6 +112,30 @@ def make_stream_url_response(
)
@pytest.fixture
async def mock_create_stream(hass) -> Mock:
"""Fixture to mock out the create stream call."""
assert await async_setup_component(hass, "stream", {})
with patch(
"homeassistant.components.camera.create_stream", autospec=True
) as mock_stream:
mock_stream.return_value.endpoint_url.return_value = (
"http://home.assistant/playlist.m3u8"
)
mock_stream.return_value.async_get_image = AsyncMock()
mock_stream.return_value.async_get_image.return_value = IMAGE_BYTES_FROM_STREAM
yield mock_stream
async def async_get_image(hass, width=None, height=None):
"""Get the camera image."""
image = await camera.async_get_image(
hass, "camera.my_camera", width=width, height=height
)
assert image.content_type == "image/jpeg"
return image.content
async def async_setup_camera(hass, traits={}, auth=None):
"""Set up the platform and prerequisites."""
devices = {}
@ -138,21 +158,6 @@ async def fire_alarm(hass, point_in_time):
await hass.async_block_till_done()
async def async_get_image(hass, width=None, height=None):
"""Get image from the camera, a wrapper around camera.async_get_image."""
# Note: this patches ImageFrame to simulate decoding an image from a live
# stream, however the test may not use it. Tests assert on the image
# contents to determine if the image came from the live stream or event.
with patch(
"homeassistant.components.ffmpeg.ImageFrame.get_image",
autopatch=True,
return_value=IMAGE_BYTES_FROM_STREAM,
):
return await camera.async_get_image(
hass, "camera.my_camera", width=width, height=height
)
async def test_no_devices(hass):
"""Test configuration that returns no devices."""
await async_setup_camera(hass)
@ -194,7 +199,7 @@ async def test_camera_device(hass):
assert device.identifiers == {("nest", DEVICE_ID)}
async def test_camera_stream(hass, auth):
async def test_camera_stream(hass, auth, mock_create_stream):
"""Test a basic camera and fetch its live stream."""
auth.responses = [make_stream_url_response()]
await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
@ -208,11 +213,10 @@ async def test_camera_stream(hass, auth):
stream_source = await camera.async_get_stream_source(hass, "camera.my_camera")
assert stream_source == "rtsp://some/url?auth=g.0.streamingToken"
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
assert await async_get_image(hass) == IMAGE_BYTES_FROM_STREAM
async def test_camera_ws_stream(hass, auth, hass_ws_client):
async def test_camera_ws_stream(hass, auth, hass_ws_client, mock_create_stream):
"""Test a basic camera that supports web rtc."""
auth.responses = [make_stream_url_response()]
await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
@ -223,23 +227,23 @@ async def test_camera_ws_stream(hass, auth, hass_ws_client):
assert cam.state == STATE_STREAMING
assert cam.attributes["frontend_stream_type"] == STREAM_TYPE_HLS
with patch("homeassistant.components.camera.create_stream") as mock_stream:
mock_stream().endpoint_url.return_value = "http://home.assistant/playlist.m3u8"
client = await hass_ws_client(hass)
await client.send_json(
{
"id": 2,
"type": "camera/stream",
"entity_id": "camera.my_camera",
}
)
msg = await client.receive_json()
client = await hass_ws_client(hass)
await client.send_json(
{
"id": 2,
"type": "camera/stream",
"entity_id": "camera.my_camera",
}
)
msg = await client.receive_json()
assert msg["id"] == 2
assert msg["type"] == TYPE_RESULT
assert msg["success"]
assert msg["result"]["url"] == "http://home.assistant/playlist.m3u8"
assert await async_get_image(hass) == IMAGE_BYTES_FROM_STREAM
async def test_camera_ws_stream_failure(hass, auth, hass_ws_client):
"""Test a basic camera that supports web rtc."""
@ -292,9 +296,8 @@ async def test_camera_stream_missing_trait(hass, auth):
stream_source = await camera.async_get_stream_source(hass, "camera.my_camera")
assert stream_source is None
# Unable to get an image from the live stream
with pytest.raises(HomeAssistantError):
await async_get_image(hass)
# Fallback to placeholder image
await async_get_image(hass)
async def test_refresh_expired_stream_token(hass, auth):
@ -422,16 +425,6 @@ async def test_camera_removed(hass, auth):
stream_source = await camera.async_get_stream_source(hass, "camera.my_camera")
assert stream_source == "rtsp://some/url?auth=g.0.streamingToken"
# Fetch an event image, exercising cleanup on remove
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
auth.responses = [
aiohttp.web.json_response(GENERATE_IMAGE_URL_RESPONSE),
aiohttp.web.Response(body=IMAGE_BYTES_FROM_EVENT),
]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_EVENT
for config_entry in hass.config_entries.async_entries(DOMAIN):
await hass.config_entries.async_remove(config_entry.entry_id)
await hass.async_block_till_done()
@ -517,160 +510,6 @@ async def test_refresh_expired_stream_failure(hass, auth):
assert create_stream.called
async def test_camera_image_from_last_event(hass, auth):
"""Test an image generated from an event."""
# The subscriber receives a message related to an image event. The camera
# holds on to the event message. When the test asks for a capera snapshot
# it exchanges the event id for an image url and fetches the image.
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
# Simulate a pubsub message received by the subscriber with a motion event.
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
auth.responses = [
# Fake response from API that returns url image
aiohttp.web.json_response(GENERATE_IMAGE_URL_RESPONSE),
# Fake response for the image content fetch
aiohttp.web.Response(body=IMAGE_BYTES_FROM_EVENT),
]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_EVENT
# Verify expected image fetch request was captured
assert auth.url == TEST_IMAGE_URL
assert auth.headers == IMAGE_AUTHORIZATION_HEADERS
# An additional fetch uses the cache and does not send another RPC
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_EVENT
# Verify expected image fetch request was captured
assert auth.url == TEST_IMAGE_URL
assert auth.headers == IMAGE_AUTHORIZATION_HEADERS
async def test_camera_image_from_event_not_supported(hass, auth):
"""Test fallback to stream image when event images are not supported."""
# Create a device that does not support the CameraEventImgae trait
traits = DEVICE_TRAITS.copy()
del traits["sdm.devices.traits.CameraEventImage"]
subscriber = await async_setup_camera(hass, traits, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
# Camera fetches a stream url since CameraEventImage is not supported
auth.responses = [make_stream_url_response()]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
async def test_generate_event_image_url_failure(hass, auth):
"""Test fallback to stream on failure to create an image url."""
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
auth.responses = [
# Fail to generate the image url
aiohttp.web.Response(status=HTTPStatus.INTERNAL_SERVER_ERROR),
# Camera fetches a stream url as a fallback
make_stream_url_response(),
]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
async def test_fetch_event_image_failure(hass, auth):
"""Test fallback to a stream on image download failure."""
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
auth.responses = [
# Fake response from API that returns url image
aiohttp.web.json_response(GENERATE_IMAGE_URL_RESPONSE),
# Fail to download the image
aiohttp.web.Response(status=HTTPStatus.INTERNAL_SERVER_ERROR),
# Camera fetches a stream url as a fallback
make_stream_url_response(),
]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
async def test_event_image_expired(hass, auth):
"""Test fallback for an event event image that has expired."""
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
# Simulate a pubsub message has already expired
event_timestamp = utcnow() - datetime.timedelta(seconds=40)
await subscriber.async_receive_event(make_motion_event(timestamp=event_timestamp))
await hass.async_block_till_done()
# Fallback to a stream url since the event message is expired.
auth.responses = [make_stream_url_response()]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
async def test_multiple_event_images(hass, auth):
"""Test fallback for an event event image that has been cleaned up on expiration."""
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
# Simplify test setup
subscriber.cache_policy.fetch = False
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
event_timestamp = utcnow()
await subscriber.async_receive_event(
make_motion_event(event_session_id="event-session-1", timestamp=event_timestamp)
)
await hass.async_block_till_done()
auth.responses = [
# Fake response from API that returns url image
aiohttp.web.json_response(GENERATE_IMAGE_URL_RESPONSE),
# Fake response for the image content fetch
aiohttp.web.Response(body=IMAGE_BYTES_FROM_EVENT),
# Image is refetched after being cleared by expiration alarm
aiohttp.web.json_response(GENERATE_IMAGE_URL_RESPONSE),
aiohttp.web.Response(body=b"updated image bytes"),
]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_EVENT
next_event_timestamp = event_timestamp + datetime.timedelta(seconds=25)
await subscriber.async_receive_event(
make_motion_event(
event_id="updated-event-id",
event_session_id="event-session-2",
timestamp=next_event_timestamp,
)
)
await hass.async_block_till_done()
image = await async_get_image(hass)
assert image.content == b"updated image bytes"
async def test_camera_web_rtc(hass, auth, hass_ws_client):
"""Test a basic camera that supports web rtc."""
expiration = utcnow() + datetime.timedelta(seconds=100)
@ -724,10 +563,8 @@ async def test_camera_web_rtc(hass, auth, hass_ws_client):
assert msg["result"]["answer"] == "v=0\r\ns=-\r\n"
# Nest WebRTC cameras return a placeholder
content = await async_get_image(hass)
assert content.content_type == "image/jpeg"
content = await async_get_image(hass, width=1024, height=768)
assert content.content_type == "image/jpeg"
await async_get_image(hass)
await async_get_image(hass, width=1024, height=768)
async def test_camera_web_rtc_unsupported(hass, auth, hass_ws_client):
@ -802,7 +639,7 @@ async def test_camera_web_rtc_offer_failure(hass, auth, hass_ws_client):
assert msg["error"]["message"].startswith("Nest API error")
async def test_camera_multiple_streams(hass, auth, hass_ws_client):
async def test_camera_multiple_streams(hass, auth, hass_ws_client, mock_create_stream):
"""Test a camera supporting multiple stream types."""
expiration = utcnow() + datetime.timedelta(seconds=100)
auth.responses = [
@ -846,8 +683,7 @@ async def test_camera_multiple_streams(hass, auth, hass_ws_client):
stream_source = await camera.async_get_stream_source(hass, "camera.my_camera")
assert stream_source == "rtsp://some/url?auth=g.0.streamingToken"
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
assert await async_get_image(hass) == IMAGE_BYTES_FROM_STREAM
# WebRTC stream
client = await hass_ws_client(hass)