Update ONVIF to Zeep 4.0/httpx (#42020)

* convert to httpx and zeep 4.0

* fix tests

* add onvif-zeep-async to manifest

* pin zeep to fool CI cache

* address review comments

homeassistant/components/onvif/__init__.py

@@ -1,9 +1,7 @@
 """The ONVIF integration."""
 import asyncio
 
-import requests
+from onvif.exceptions import ONVIFAuthError, ONVIFError, ONVIFTimeoutError
-from requests.auth import HTTPDigestAuth
-from urllib3.exceptions import ReadTimeoutError
 import voluptuous as vol
 
 from homeassistant.components.ffmpeg import CONF_EXTRA_ARGUMENTS
@@ -17,7 +15,6 @@ from homeassistant.const import (
     EVENT_HOMEASSISTANT_STOP,
     HTTP_BASIC_AUTHENTICATION,
     HTTP_DIGEST_AUTHENTICATION,
-    HTTP_UNAUTHORIZED,
 )
 from homeassistant.core import HomeAssistant
 from homeassistant.exceptions import ConfigEntryNotReady
@@ -78,6 +75,7 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry):
     device = ONVIFDevice(hass, entry)
 
     if not await device.async_setup():
+        await device.device.close()
         return False
 
     if not device.available:
@@ -90,7 +88,7 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry):
 
     platforms = ["camera"]
 
-    if device.capabilities.events and await device.events.async_start():
+    if device.capabilities.events:
         platforms += ["binary_sensor", "sensor"]
 
     for component in platforms:
@@ -123,37 +121,26 @@ async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry):
     )
 
 
-async def _get_snapshot_auth(hass, device, entry):
+async def _get_snapshot_auth(device):
-    if not (device.username and device.password):
+    """Determine auth type for snapshots."""
+    if not device.capabilities.snapshot or not (device.username and device.password):
         return HTTP_DIGEST_AUTHENTICATION
 
-    snapshot_uri = await device.async_get_snapshot_uri(device.profiles[0])
-    if not snapshot_uri:
-        return HTTP_DIGEST_AUTHENTICATION
-    auth = HTTPDigestAuth(device.username, device.password)
-
-    def _get():
-        # so we can handle keyword arguments
-        return requests.get(snapshot_uri, timeout=1, auth=auth)
-
     try:
-        response = await hass.async_add_executor_job(_get)
+        snapshot = await device.device.get_snapshot(device.profiles[0].token)
 
-        if response.status_code == HTTP_UNAUTHORIZED:
+        if snapshot:
-            return HTTP_BASIC_AUTHENTICATION
+            return HTTP_DIGEST_AUTHENTICATION
-
-        return HTTP_DIGEST_AUTHENTICATION
-    except requests.exceptions.Timeout:
         return HTTP_BASIC_AUTHENTICATION
-    except requests.exceptions.ConnectionError as error:
+    except (ONVIFAuthError, ONVIFTimeoutError):
-        if isinstance(error.args[0], ReadTimeoutError):
+        return HTTP_BASIC_AUTHENTICATION
-            return HTTP_BASIC_AUTHENTICATION
+    except ONVIFError:
         return HTTP_DIGEST_AUTHENTICATION
 
 
 async def async_populate_snapshot_auth(hass, device, entry):
     """Check if digest auth for snapshots is possible."""
-    auth = await _get_snapshot_auth(hass, device, entry)
+    auth = await _get_snapshot_auth(device)
     new_data = {**entry.data, CONF_SNAPSHOT_AUTH: auth}
     hass.config_entries.async_update_entry(entry, data=new_data)
 

homeassistant/components/onvif/camera.py

@@ -3,8 +3,7 @@ import asyncio
 
 from haffmpeg.camera import CameraMjpeg
 from haffmpeg.tools import IMAGE_JPEG, ImageFrame
-import requests
+from onvif.exceptions import ONVIFError
-from requests.auth import HTTPBasicAuth, HTTPDigestAuth
 import voluptuous as vol
 
 from homeassistant.components.camera import SUPPORT_STREAM, Camera
@@ -86,7 +85,6 @@ class ONVIFCameraEntity(ONVIFBaseEntity, Camera):
             == HTTP_BASIC_AUTHENTICATION
         )
         self._stream_uri = None
-        self._snapshot_uri = None
 
     @property
     def supported_features(self) -> int:
@@ -119,29 +117,16 @@ class ONVIFCameraEntity(ONVIFBaseEntity, Camera):
         image = None
 
         if self.device.capabilities.snapshot:
-            auth = None
+            try:
-            if self.device.username and self.device.password:
+                image = await self.device.device.get_snapshot(
-                if self._basic_auth:
+                    self.profile.token, self._basic_auth
-                    auth = HTTPBasicAuth(self.device.username, self.device.password)
+                )
-                else:
+            except ONVIFError as err:
-                    auth = HTTPDigestAuth(self.device.username, self.device.password)
+                LOGGER.error(
-
+                    "Fetch snapshot image failed from %s, falling back to FFmpeg; %s",
-            def fetch():
+                    self.device.name,
-                """Read image from a URL."""
+                    err,
-                try:
+                )
-                    response = requests.get(self._snapshot_uri, timeout=5, auth=auth)
-                    if response.status_code < 300:
-                        return response.content
-                except requests.exceptions.RequestException as error:
-                    LOGGER.error(
-                        "Fetch snapshot image failed from %s, falling back to FFmpeg; %s",
-                        self.device.name,
-                        error,
-                    )
-
-                return None
-
-            image = await self.hass.async_add_executor_job(fetch)
 
         if image is None:
             ffmpeg = ImageFrame(self.hass.data[DATA_FFMPEG].binary, loop=self.hass.loop)
@@ -187,9 +172,6 @@ class ONVIFCameraEntity(ONVIFBaseEntity, Camera):
             "rtsp://", f"rtsp://{self.device.username}:{self.device.password}@", 1
         )
 
-        if self.device.capabilities.snapshot:
-            self._snapshot_uri = await self.device.async_get_snapshot_uri(self.profile)
-
     async def async_perform_ptz(
         self,
         distance,

homeassistant/components/onvif/config_flow.py

@@ -199,9 +199,8 @@ class OnvifFlowHandler(config_entries.ConfigFlow, domain=DOMAIN):
             self.onvif_config[CONF_PASSWORD],
         )
 
-        await device.update_xaddrs()
-
         try:
+            await device.update_xaddrs()
             device_mgmt = device.create_devicemgmt_service()
 
             # Get the MAC address to use as the unique ID for the config flow
@@ -250,6 +249,8 @@ class OnvifFlowHandler(config_entries.ConfigFlow, domain=DOMAIN):
             if not h264:
                 return self.async_abort(reason="no_h264")
 
+            await device.close()
+
             title = f"{self.onvif_config[CONF_NAME]} - {self.device_id}"
             return self.async_create_entry(title=title, data=self.onvif_config)
 
@@ -259,11 +260,13 @@ class OnvifFlowHandler(config_entries.ConfigFlow, domain=DOMAIN):
                 self.onvif_config[CONF_NAME],
                 err,
             )
+            await device.close()
             return self.async_abort(reason="onvif_error")
 
         except Fault:
             errors["base"] = "cannot_connect"
 
+        await device.close()
         return self.async_show_form(step_id="auth", errors=errors)
 
     async def async_step_import(self, user_input):

homeassistant/components/onvif/device.py

@@ -4,7 +4,7 @@ import datetime as dt
 import os
 from typing import List
 
-from aiohttp.client_exceptions import ClientConnectionError, ServerDisconnectedError
+from httpx import RequestError
 import onvif
 from onvif import ONVIFCamera
 from onvif.exceptions import ONVIFError
@@ -93,25 +93,31 @@ class ONVIFDevice:
         try:
             await self.device.update_xaddrs()
             await self.async_check_date_and_time()
+
+            # Create event manager
+            self.events = EventManager(
+                self.hass, self.device, self.config_entry.unique_id
+            )
+
+            # Fetch basic device info and capabilities
             self.info = await self.async_get_device_info()
             self.capabilities = await self.async_get_capabilities()
             self.profiles = await self.async_get_profiles()
 
+            # No camera profiles to add
+            if not self.profiles:
+                return False
+
             if self.capabilities.ptz:
                 self.device.create_ptz_service()
 
-            if self.capabilities.events:
-                self.events = EventManager(
-                    self.hass, self.device, self.config_entry.unique_id
-                )
-
             # Determine max resolution from profiles
             self.max_resolution = max(
                 profile.video.resolution.width
                 for profile in self.profiles
                 if profile.video.encoding == "H264"
             )
-        except ClientConnectionError as err:
+        except RequestError as err:
             LOGGER.warning(
                 "Couldn't connect to camera '%s', but will retry later. Error: %s",
                 self.name,
@@ -195,7 +201,7 @@ class ONVIFDevice:
                         cam_date_utc,
                         system_date,
                     )
-        except ServerDisconnectedError as err:
+        except RequestError as err:
             LOGGER.warning(
                 "Couldn't get device '%s' date/time. Error: %s", self.name, err
             )
@@ -237,14 +243,12 @@ class ONVIFDevice:
             media_service = self.device.create_media_service()
             media_capabilities = await media_service.GetServiceCapabilities()
             snapshot = media_capabilities and media_capabilities.SnapshotUri
-        except (ONVIFError, Fault, ServerDisconnectedError):
+        except (ONVIFError, Fault, RequestError):
             pass
 
         pullpoint = False
         try:
-            event_service = self.device.create_events_service()
+            pullpoint = await self.events.async_start()
-            event_capabilities = await event_service.GetServiceCapabilities()
-            pullpoint = event_capabilities and event_capabilities.WSPullPointSupport
         except (ONVIFError, Fault):
             pass
 
@@ -262,6 +266,10 @@ class ONVIFDevice:
         media_service = self.device.create_media_service()
         result = await media_service.GetProfiles()
         profiles = []
+
+        if not isinstance(result, list):
+            return profiles
+
         for key, onvif_profile in enumerate(result):
             # Only add H264 profiles
             if (
@@ -298,7 +306,7 @@ class ONVIFDevice:
                     ptz_service = self.device.create_ptz_service()
                     presets = await ptz_service.GetPresets(profile.token)
                     profile.ptz.presets = [preset.token for preset in presets if preset]
-                except (Fault, ServerDisconnectedError):
+                except (Fault, RequestError):
                     # It's OK if Presets aren't supported
                     profile.ptz.presets = []
 
@@ -306,17 +314,6 @@ class ONVIFDevice:
 
         return profiles
 
-    async def async_get_snapshot_uri(self, profile: Profile) -> str:
-        """Get the snapshot URI for a specified profile."""
-        if not self.capabilities.snapshot:
-            return None
-
-        media_service = self.device.create_media_service()
-        req = media_service.create_type("GetSnapshotUri")
-        req.ProfileToken = profile.token
-        result = await media_service.GetSnapshotUri(req)
-        return result.Uri
-
     async def async_get_stream_uri(self, profile: Profile) -> str:
         """Get the stream URI for a specified profile."""
         media_service = self.device.create_media_service()

homeassistant/components/onvif/event.py

@@ -3,11 +3,7 @@ import asyncio
 import datetime as dt
 from typing import Callable, Dict, List, Optional, Set
 
-from aiohttp.client_exceptions import (
+from httpx import RemoteProtocolError, TransportError
-    ClientConnectorError,
-    ClientOSError,
-    ServerDisconnectedError,
-)
 from onvif import ONVIFCamera, ONVIFService
 from zeep.exceptions import Fault
 
@@ -21,11 +17,9 @@ from .parsers import PARSERS
 
 UNHANDLED_TOPICS = set()
 SUBSCRIPTION_ERRORS = (
-    ClientConnectorError,
-    ClientOSError,
     Fault,
-    ServerDisconnectedError,
     asyncio.TimeoutError,
+    TransportError,
 )
 
 
@@ -173,10 +167,14 @@ class EventManager:
                     dt_util.as_utc(response.TerminationTime) - dt_util.utcnow()
                 ).total_seconds() < 7200:
                     await self.async_renew()
-            except SUBSCRIPTION_ERRORS:
+            except RemoteProtocolError:
+                # Likley a shutdown event, nothing to see here
+                return
+            except SUBSCRIPTION_ERRORS as err:
                 LOGGER.warning(
-                    "Failed to fetch ONVIF PullPoint subscription messages for '%s'",
+                    "Failed to fetch ONVIF PullPoint subscription messages for '%s': %s",
                     self.unique_id,
+                    err,
                 )
                 # Treat errors as if the camera restarted. Assume that the pullpoint
                 # subscription is no longer valid.

homeassistant/components/onvif/manifest.json

@@ -3,9 +3,9 @@
   "name": "ONVIF",
   "documentation": "https://www.home-assistant.io/integrations/onvif",
   "requirements": [
-    "onvif-zeep-async==0.6.0",
+    "onvif-zeep-async==1.0.0",
     "WSDiscovery==2.0.0",
-    "zeep[async]==3.4.0"
+    "zeep[async]==4.0.0"
   ],
   "dependencies": ["ffmpeg"],
   "codeowners": ["@hunterjm"],

requirements_all.txt

@@ -1028,7 +1028,7 @@ omnilogic==0.4.2
 onkyo-eiscp==1.2.7
 
 # homeassistant.components.onvif
-onvif-zeep-async==0.6.0
+onvif-zeep-async==1.0.0
 
 # homeassistant.components.opengarage
 open-garage==0.1.4
@@ -2322,7 +2322,7 @@ yeelightsunflower==0.0.10
 youtube_dl==2020.09.20
 
 # homeassistant.components.onvif
-zeep[async]==3.4.0
+zeep[async]==4.0.0
 
 # homeassistant.components.zengge
 zengge==0.2

requirements_test_all.txt

@@ -499,7 +499,7 @@ oauth2client==4.0.0
 omnilogic==0.4.2
 
 # homeassistant.components.onvif
-onvif-zeep-async==0.6.0
+onvif-zeep-async==1.0.0
 
 # homeassistant.components.openerz
 openerz-api==0.1.0
@@ -1097,7 +1097,7 @@ xmltodict==0.12.0
 yeelight==0.5.4
 
 # homeassistant.components.onvif
-zeep[async]==3.4.0
+zeep[async]==4.0.0
 
 # homeassistant.components.zeroconf
 zeroconf==0.28.6

tests/components/onvif/test_config_flow.py

@@ -69,6 +69,7 @@ def setup_mock_onvif_camera(
     mock_onvif_camera.update_xaddrs = AsyncMock(return_value=True)
     mock_onvif_camera.create_devicemgmt_service = MagicMock(return_value=devicemgmt)
     mock_onvif_camera.create_media_service = MagicMock(return_value=media_service)
+    mock_onvif_camera.close = AsyncMock(return_value=None)
 
     def mock_constructor(
         host,