Support XML conversion for RESTful sensors (#31809)

* Support XML conversion for RESTful sensors

Many devices continue to use XML for RESTful
APIs.  Interfacing with these APIs requires custom
integrations or command line fork()/exec() overhead
which many of these devices can work with as if
they were JSON using xmltojson via this spec:
https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html

This change implements converting XML output to
JSON via xmltojson so it can work with the existing
rest sensor component.  As the attributes that
usually need to be scraped are deeper in the document
support for passing in a template to find the
JSON attributes that have been added.  JSON APIs that
do not have their attributes at the top level
can also benefit from this change.

* Auto convert xml, change out the template for jsonpath

* Address review items and potentially unexpected normalize behavior with jsonpath

* Revert "Address review items and potentially unexpected normalize behavior with jsonpath"

This reverts commit fe9e179092.

* json_dict[0] turned out to be needed

homeassistant/components/rest/manifest.json

@@ -2,7 +2,7 @@
   "domain": "rest",
   "name": "RESTful",
   "documentation": "https://www.home-assistant.io/integrations/rest",
-  "requirements": [],
+  "requirements": ["jsonpath==0.82", "xmltodict==0.12.0"],
   "dependencies": [],
   "codeowners": []
 }

homeassistant/components/rest/sensor.py

@@ -1,10 +1,13 @@
 """Support for RESTful API sensors."""
 import json
 import logging
+from xml.parsers.expat import ExpatError
 
+from jsonpath import jsonpath
 import requests
 from requests.auth import HTTPBasicAuth, HTTPDigestAuth
 import voluptuous as vol
+import xmltodict
 
 from homeassistant.components.sensor import DEVICE_CLASSES_SCHEMA, PLATFORM_SCHEMA
 from homeassistant.const import (
@@ -38,7 +41,9 @@ DEFAULT_VERIFY_SSL = True
 DEFAULT_FORCE_UPDATE = False
 DEFAULT_TIMEOUT = 10
 
+
 CONF_JSON_ATTRS = "json_attributes"
+CONF_JSON_ATTRS_PATH = "json_attributes_path"
 METHODS = ["POST", "GET"]
 
 PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
@@ -57,6 +62,7 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
         vol.Optional(CONF_UNIT_OF_MEASUREMENT): cv.string,
         vol.Optional(CONF_DEVICE_CLASS): DEVICE_CLASSES_SCHEMA,
         vol.Optional(CONF_USERNAME): cv.string,
+        vol.Optional(CONF_JSON_ATTRS_PATH): cv.string,
         vol.Optional(CONF_VALUE_TEMPLATE): cv.template,
         vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean,
         vol.Optional(CONF_FORCE_UPDATE, default=DEFAULT_FORCE_UPDATE): cv.boolean,
@@ -84,6 +90,7 @@ def setup_platform(hass, config, add_entities, discovery_info=None):
     device_class = config.get(CONF_DEVICE_CLASS)
     value_template = config.get(CONF_VALUE_TEMPLATE)
     json_attrs = config.get(CONF_JSON_ATTRS)
+    json_attrs_path = config.get(CONF_JSON_ATTRS_PATH)
     force_update = config.get(CONF_FORCE_UPDATE)
     timeout = config.get(CONF_TIMEOUT)
 
@@ -120,6 +127,7 @@ def setup_platform(hass, config, add_entities, discovery_info=None):
                 json_attrs,
                 force_update,
                 resource_template,
+                json_attrs_path,
             )
         ],
         True,
@@ -140,6 +148,7 @@ class RestSensor(Entity):
         json_attrs,
         force_update,
         resource_template,
+        json_attrs_path,
     ):
         """Initialize the REST sensor."""
         self._hass = hass
@@ -153,6 +162,7 @@ class RestSensor(Entity):
         self._attributes = None
         self._force_update = force_update
         self._resource_template = resource_template
+        self._json_attrs_path = json_attrs_path
 
     @property
     def name(self):
@@ -191,12 +201,27 @@ class RestSensor(Entity):
 
         self.rest.update()
         value = self.rest.data
+        content_type = self.rest.headers.get("content-type")
+
+        if content_type and content_type.startswith("text/xml"):
+            try:
+                value = json.dumps(xmltodict.parse(value))
+            except ExpatError:
+                _LOGGER.warning(
+                    "REST xml result could not be parsed and converted to JSON."
+                )
+                _LOGGER.debug("Erroneous XML: %s", value)
 
         if self._json_attrs:
             self._attributes = {}
             if value:
                 try:
                     json_dict = json.loads(value)
+                    if self._json_attrs_path is not None:
+                        json_dict = jsonpath(json_dict, self._json_attrs_path)
+                    # jsonpath will always store the result in json_dict[0]
+                    # so the next line happens to work exactly as needed to
+                    # find the result
                     if isinstance(json_dict, list):
                         json_dict = json_dict[0]
                     if isinstance(json_dict, dict):
@@ -240,6 +265,7 @@ class RestData:
         self._verify_ssl = verify_ssl
         self._timeout = timeout
         self.data = None
+        self.headers = None
 
     def set_url(self, url):
         """Set url."""
@@ -259,6 +285,8 @@ class RestData:
                 verify=self._verify_ssl,
             )
             self.data = response.text
+            self.headers = response.headers
         except requests.exceptions.RequestException as ex:
             _LOGGER.error("Error fetching data: %s failed with %s", self._resource, ex)
             self.data = None
+            self.headers = None

requirements_all.txt

@@ -742,6 +742,7 @@ iperf3==0.1.11
 # homeassistant.components.route53
 ipify==1.0.0
 
+# homeassistant.components.rest
 # homeassistant.components.verisure
 jsonpath==0.82
 
@@ -2097,6 +2098,7 @@ xfinity-gateway==0.0.4
 xknx==0.11.2
 
 # homeassistant.components.bluesound
+# homeassistant.components.rest
 # homeassistant.components.startca
 # homeassistant.components.ted5000
 # homeassistant.components.yr

requirements_test_all.txt

@@ -278,6 +278,7 @@ iaqualink==0.3.1
 # homeassistant.components.influxdb
 influxdb==5.2.3
 
+# homeassistant.components.rest
 # homeassistant.components.verisure
 jsonpath==0.82
 
@@ -711,6 +712,7 @@ withings-api==2.1.3
 wled==0.2.1
 
 # homeassistant.components.bluesound
+# homeassistant.components.rest
 # homeassistant.components.startca
 # homeassistant.components.ted5000
 # homeassistant.components.yr

tests/components/rest/test_sensor.py

@@ -6,6 +6,7 @@ import pytest
 from pytest import raises
 import requests
 from requests.exceptions import RequestException, Timeout
+from requests.structures import CaseInsensitiveDict
 import requests_mock
 
 import homeassistant.components.rest.sensor as rest
@@ -166,6 +167,33 @@ class TestRestSensorSetup(unittest.TestCase):
             )
         assert 2 == mock_req.call_count
 
+    @requests_mock.Mocker()
+    def test_setup_get_xml(self, mock_req):
+        """Test setup with valid configuration."""
+        mock_req.get("http://localhost", status_code=200)
+        with assert_setup_component(1, "sensor"):
+            assert setup_component(
+                self.hass,
+                "sensor",
+                {
+                    "sensor": {
+                        "platform": "rest",
+                        "resource": "http://localhost",
+                        "method": "GET",
+                        "value_template": "{{ value_json.key }}",
+                        "name": "foo",
+                        "unit_of_measurement": DATA_MEGABYTES,
+                        "verify_ssl": "true",
+                        "timeout": 30,
+                        "authentication": "basic",
+                        "username": "my username",
+                        "password": "my password",
+                        "headers": {"Accept": "text/xml"},
+                    }
+                },
+            )
+        assert 2 == mock_req.call_count
+
 
 class TestRestSensor(unittest.TestCase):
     """Tests for REST sensor platform."""
@@ -178,13 +206,15 @@ class TestRestSensor(unittest.TestCase):
         self.rest.update = Mock(
             "rest.RestData.update",
             side_effect=self.update_side_effect(
-                '{ "key": "' + self.initial_state + '" }'
+                '{ "key": "' + self.initial_state + '" }',
+                CaseInsensitiveDict({"Content-Type": "application/json"}),
             ),
         )
         self.name = "foo"
         self.unit_of_measurement = DATA_MEGABYTES
         self.device_class = None
         self.value_template = template("{{ value_json.key }}")
+        self.json_attrs_path = None
         self.value_template.hass = self.hass
         self.force_update = False
         self.resource_template = None
@@ -199,15 +229,17 @@ class TestRestSensor(unittest.TestCase):
             [],
             self.force_update,
             self.resource_template,
+            self.json_attrs_path,
         )
 
     def tearDown(self):
         """Stop everything that was started."""
         self.hass.stop()
 
-    def update_side_effect(self, data):
+    def update_side_effect(self, data, headers):
         """Side effect function for mocking RestData.update()."""
         self.rest.data = data
+        self.rest.headers = headers
 
     def test_name(self):
         """Test the name."""
@@ -229,7 +261,8 @@ class TestRestSensor(unittest.TestCase):
     def test_update_when_value_is_none(self):
         """Test state gets updated to unknown when sensor returns no data."""
         self.rest.update = Mock(
-            "rest.RestData.update", side_effect=self.update_side_effect(None)
+            "rest.RestData.update",
+            side_effect=self.update_side_effect(None, CaseInsensitiveDict()),
         )
         self.sensor.update()
         assert self.sensor.state is None
@@ -239,7 +272,10 @@ class TestRestSensor(unittest.TestCase):
         """Test state gets updated when sensor returns a new status."""
         self.rest.update = Mock(
             "rest.RestData.update",
-            side_effect=self.update_side_effect('{ "key": "updated_state" }'),
+            side_effect=self.update_side_effect(
+                '{ "key": "updated_state" }',
+                CaseInsensitiveDict({"Content-Type": "application/json"}),
+            ),
         )
         self.sensor.update()
         assert "updated_state" == self.sensor.state
@@ -248,7 +284,10 @@ class TestRestSensor(unittest.TestCase):
     def test_update_with_no_template(self):
         """Test update when there is no value template."""
         self.rest.update = Mock(
-            "rest.RestData.update", side_effect=self.update_side_effect("plain_state")
+            "rest.RestData.update",
+            side_effect=self.update_side_effect(
+                "plain_state", CaseInsensitiveDict({"Content-Type": "application/json"})
+            ),
         )
         self.sensor = rest.RestSensor(
             self.hass,
@@ -260,6 +299,7 @@ class TestRestSensor(unittest.TestCase):
             [],
             self.force_update,
             self.resource_template,
+            self.json_attrs_path,
         )
         self.sensor.update()
         assert "plain_state" == self.sensor.state
@@ -269,7 +309,10 @@ class TestRestSensor(unittest.TestCase):
         """Test attributes get extracted from a JSON result."""
         self.rest.update = Mock(
             "rest.RestData.update",
-            side_effect=self.update_side_effect('{ "key": "some_json_value" }'),
+            side_effect=self.update_side_effect(
+                '{ "key": "some_json_value" }',
+                CaseInsensitiveDict({"Content-Type": "application/json"}),
+            ),
         )
         self.sensor = rest.RestSensor(
             self.hass,
@@ -281,6 +324,7 @@ class TestRestSensor(unittest.TestCase):
             ["key"],
             self.force_update,
             self.resource_template,
+            self.json_attrs_path,
         )
         self.sensor.update()
         assert "some_json_value" == self.sensor.device_state_attributes["key"]
@@ -289,7 +333,10 @@ class TestRestSensor(unittest.TestCase):
         """Test attributes get extracted from a JSON list[0] result."""
         self.rest.update = Mock(
             "rest.RestData.update",
-            side_effect=self.update_side_effect('[{ "key": "another_value" }]'),
+            side_effect=self.update_side_effect(
+                '[{ "key": "another_value" }]',
+                CaseInsensitiveDict({"Content-Type": "application/json"}),
+            ),
         )
         self.sensor = rest.RestSensor(
             self.hass,
@@ -301,6 +348,7 @@ class TestRestSensor(unittest.TestCase):
             ["key"],
             self.force_update,
             self.resource_template,
+            self.json_attrs_path,
         )
         self.sensor.update()
         assert "another_value" == self.sensor.device_state_attributes["key"]
@@ -309,7 +357,10 @@ class TestRestSensor(unittest.TestCase):
     def test_update_with_json_attrs_no_data(self, mock_logger):
         """Test attributes when no JSON result fetched."""
         self.rest.update = Mock(
-            "rest.RestData.update", side_effect=self.update_side_effect(None)
+            "rest.RestData.update",
+            side_effect=self.update_side_effect(
+                None, CaseInsensitiveDict({"Content-Type": "application/json"})
+            ),
         )
         self.sensor = rest.RestSensor(
             self.hass,
@@ -321,6 +372,7 @@ class TestRestSensor(unittest.TestCase):
             ["key"],
             self.force_update,
             self.resource_template,
+            self.json_attrs_path,
         )
         self.sensor.update()
         assert {} == self.sensor.device_state_attributes
@@ -331,7 +383,10 @@ class TestRestSensor(unittest.TestCase):
         """Test attributes get extracted from a JSON result."""
         self.rest.update = Mock(
             "rest.RestData.update",
-            side_effect=self.update_side_effect('["list", "of", "things"]'),
+            side_effect=self.update_side_effect(
+                '["list", "of", "things"]',
+                CaseInsensitiveDict({"Content-Type": "application/json"}),
+            ),
         )
         self.sensor = rest.RestSensor(
             self.hass,
@@ -343,6 +398,7 @@ class TestRestSensor(unittest.TestCase):
             ["key"],
             self.force_update,
             self.resource_template,
+            self.json_attrs_path,
         )
         self.sensor.update()
         assert {} == self.sensor.device_state_attributes
@@ -353,7 +409,10 @@ class TestRestSensor(unittest.TestCase):
         """Test attributes get extracted from a JSON result."""
         self.rest.update = Mock(
             "rest.RestData.update",
-            side_effect=self.update_side_effect("This is text rather than JSON data."),
+            side_effect=self.update_side_effect(
+                "This is text rather than JSON data.",
+                CaseInsensitiveDict({"Content-Type": "text/plain"}),
+            ),
         )
         self.sensor = rest.RestSensor(
             self.hass,
@@ -365,6 +424,7 @@ class TestRestSensor(unittest.TestCase):
             ["key"],
             self.force_update,
             self.resource_template,
+            self.json_attrs_path,
         )
         self.sensor.update()
         assert {} == self.sensor.device_state_attributes
@@ -376,7 +436,8 @@ class TestRestSensor(unittest.TestCase):
         self.rest.update = Mock(
             "rest.RestData.update",
             side_effect=self.update_side_effect(
-                '{ "key": "json_state_updated_value" }'
+                '{ "key": "json_state_updated_value" }',
+                CaseInsensitiveDict({"Content-Type": "application/json"}),
             ),
         )
         self.sensor = rest.RestSensor(
@@ -389,6 +450,7 @@ class TestRestSensor(unittest.TestCase):
             ["key"],
             self.force_update,
             self.resource_template,
+            self.json_attrs_path,
         )
         self.sensor.update()
 
@@ -397,6 +459,136 @@ class TestRestSensor(unittest.TestCase):
             "json_state_updated_value" == self.sensor.device_state_attributes["key"]
         ), self.force_update
 
+    def test_update_with_json_attrs_with_json_attrs_path(self):
+        """Test attributes get extracted from a JSON result with a template for the attributes."""
+        json_attrs_path = "$.toplevel.second_level"
+        value_template = template("{{ value_json.toplevel.master_value }}")
+        value_template.hass = self.hass
+
+        self.rest.update = Mock(
+            "rest.RestData.update",
+            side_effect=self.update_side_effect(
+                '{ "toplevel": {"master_value": "master", "second_level": {"some_json_key": "some_json_value", "some_json_key2": "some_json_value2" } } }',
+                CaseInsensitiveDict({"Content-Type": "application/json"}),
+            ),
+        )
+        self.sensor = rest.RestSensor(
+            self.hass,
+            self.rest,
+            self.name,
+            self.unit_of_measurement,
+            self.device_class,
+            value_template,
+            ["some_json_key", "some_json_key2"],
+            self.force_update,
+            self.resource_template,
+            json_attrs_path,
+        )
+
+        self.sensor.update()
+        assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"]
+        assert (
+            "some_json_value2" == self.sensor.device_state_attributes["some_json_key2"]
+        )
+        assert "master" == self.sensor.state
+
+    def test_update_with_xml_convert_json_attrs_with_json_attrs_path(self):
+        """Test attributes get extracted from a JSON result that was converted from XML with a template for the attributes."""
+        json_attrs_path = "$.toplevel.second_level"
+        value_template = template("{{ value_json.toplevel.master_value }}")
+        value_template.hass = self.hass
+
+        self.rest.update = Mock(
+            "rest.RestData.update",
+            side_effect=self.update_side_effect(
+                "<toplevel><master_value>master</master_value><second_level><some_json_key>some_json_value</some_json_key><some_json_key2>some_json_value2</some_json_key2></second_level></toplevel>",
+                CaseInsensitiveDict({"Content-Type": "text/xml+svg"}),
+            ),
+        )
+        self.sensor = rest.RestSensor(
+            self.hass,
+            self.rest,
+            self.name,
+            self.unit_of_measurement,
+            self.device_class,
+            value_template,
+            ["some_json_key", "some_json_key2"],
+            self.force_update,
+            self.resource_template,
+            json_attrs_path,
+        )
+
+        self.sensor.update()
+        assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"]
+        assert (
+            "some_json_value2" == self.sensor.device_state_attributes["some_json_key2"]
+        )
+        assert "master" == self.sensor.state
+
+    def test_update_with_xml_convert_json_attrs_with_jsonattr_template(self):
+        """Test attributes get extracted from a JSON result that was converted from XML."""
+        json_attrs_path = "$.response"
+        value_template = template("{{ value_json.response.bss.wlan }}")
+        value_template.hass = self.hass
+
+        self.rest.update = Mock(
+            "rest.RestData.update",
+            side_effect=self.update_side_effect(
+                '<?xml version="1.0" encoding="utf-8"?><response><scan>0</scan><ver>12556</ver><count>48</count><ssid>alexander</ssid><bss><valid>0</valid><name>0</name><privacy>0</privacy><wlan>bogus</wlan><strength>0</strength></bss><led0>0</led0><led1>0</led1><led2>0</led2><led3>0</led3><led4>0</led4><led5>0</led5><led6>0</led6><led7>0</led7><btn0>up</btn0><btn1>up</btn1><btn2>up</btn2><btn3>up</btn3><pot0>0</pot0><usr0>0</usr0><temp0>0x0XF0x0XF</temp0><time0> 0</time0></response>',
+                CaseInsensitiveDict({"Content-Type": "text/xml"}),
+            ),
+        )
+        self.sensor = rest.RestSensor(
+            self.hass,
+            self.rest,
+            self.name,
+            self.unit_of_measurement,
+            self.device_class,
+            value_template,
+            ["led0", "led1", "temp0", "time0", "ver"],
+            self.force_update,
+            self.resource_template,
+            json_attrs_path,
+        )
+
+        self.sensor.update()
+        assert "0" == self.sensor.device_state_attributes["led0"]
+        assert "0" == self.sensor.device_state_attributes["led1"]
+        assert "0x0XF0x0XF" == self.sensor.device_state_attributes["temp0"]
+        assert "0" == self.sensor.device_state_attributes["time0"]
+        assert "12556" == self.sensor.device_state_attributes["ver"]
+        assert "bogus" == self.sensor.state
+
+    @patch("homeassistant.components.rest.sensor._LOGGER")
+    def test_update_with_xml_convert_bad_xml(self, mock_logger):
+        """Test attributes get extracted from a XML result with bad xml."""
+        value_template = template("{{ value_json.toplevel.master_value }}")
+        value_template.hass = self.hass
+
+        self.rest.update = Mock(
+            "rest.RestData.update",
+            side_effect=self.update_side_effect(
+                "this is not xml", CaseInsensitiveDict({"Content-Type": "text/xml"})
+            ),
+        )
+        self.sensor = rest.RestSensor(
+            self.hass,
+            self.rest,
+            self.name,
+            self.unit_of_measurement,
+            self.device_class,
+            value_template,
+            ["key"],
+            self.force_update,
+            self.resource_template,
+            self.json_attrs_path,
+        )
+
+        self.sensor.update()
+        assert {} == self.sensor.device_state_attributes
+        assert mock_logger.warning.called
+        assert mock_logger.debug.called
+
 
 class TestRestData(unittest.TestCase):
     """Tests for RestData."""