Add new price sensors with API token access to pvpc hourly pricing (#85769)

* ✨ Implement optional API token in config-flow + options

to make the data download from an authenticated path in ESIOS server

As this is an *alternative* access, and current public path works for the PVPC,
no user (current or new) is compelled to obtain a token,
and it can be enabled anytime in options, or doing the setup again

When enabling the token, it is verified (or "invalid_auth" error),
and a 'reauth' flow is implemented, which can change or disable the token if
it starts failing.

The 1st step of config/options flow adds a bool to enable this private access,
- if unchecked (default), entry is set for public access (like before)
- if checked, a 2nd step opens to input the token, with instructions
  of how to get one (with a direct link to create a 'request email').
  If the token is valid, the entry is set for authenticated access

The 'reauth' flow shows the boolean flag so the user could disable a bad token
by unchecking the boolean flag 'use_api_token'

* ♻️ Remove storage of flag 'use_api_token' in config entry

leaving it only to enable/disable the optional token in the config-flow

* ♻️ Adjust async_update_options

* ✨ Add new price sensors with API token access

New price sensors added:
- Injection price: price of excess energy from self-consumption
- OMIE price: electricity price in the 'open' market
- MAG price: Temporal tax cost for gas compensation

* ✅ Adapt tests to work with multiple sensors

* 🐛 Fix all integration sensors going unavailable

when any sensor lacks data for the current day (usually the 'OMIE price')

* Fix rebase

* Customize icons and display precision for new sensors

* Disable MAG Tax and OMIE price sensors by default

* Move logic to assign sensor unique ids to integration

* Move helper functions to helpers.py

* Fix sensor activation for API download

homeassistant/components/pvpc_hourly_pricing/__init__.py

@@ -10,10 +10,12 @@ from homeassistant.core import HomeAssistant
 from homeassistant.exceptions import ConfigEntryAuthFailed
 from homeassistant.helpers import config_validation as cv
 from homeassistant.helpers.aiohttp_client import async_get_clientsession
+import homeassistant.helpers.entity_registry as er
 from homeassistant.helpers.update_coordinator import DataUpdateCoordinator, UpdateFailed
 from homeassistant.util import dt as dt_util
 
 from .const import ATTR_POWER, ATTR_POWER_P3, ATTR_TARIFF, DOMAIN
+from .helpers import get_enabled_sensor_keys
 
 _LOGGER = logging.getLogger(__name__)
 PLATFORMS: list[Platform] = [Platform.SENSOR]
@@ -22,7 +24,12 @@ CONFIG_SCHEMA = cv.removed(DOMAIN, raise_if_present=False)
 
 async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
     """Set up pvpc hourly pricing from a config entry."""
-    coordinator = ElecPricesDataUpdateCoordinator(hass, entry)
+    entity_registry = er.async_get(hass)
+    sensor_keys = get_enabled_sensor_keys(
+        using_private_api=entry.data.get(CONF_API_TOKEN) is not None,
+        entries=er.async_entries_for_config_entry(entity_registry, entry.entry_id),
+    )
+    coordinator = ElecPricesDataUpdateCoordinator(hass, entry, sensor_keys)
     await coordinator.async_config_entry_first_refresh()
 
     hass.data.setdefault(DOMAIN, {})[entry.entry_id] = coordinator
@@ -55,7 +62,9 @@ async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
 class ElecPricesDataUpdateCoordinator(DataUpdateCoordinator[EsiosApiData]):
     """Class to manage fetching Electricity prices data from API."""
 
-    def __init__(self, hass: HomeAssistant, entry: ConfigEntry) -> None:
+    def __init__(
+        self, hass: HomeAssistant, entry: ConfigEntry, sensor_keys: set[str]
+    ) -> None:
         """Initialize."""
         self.api = PVPCData(
             session=async_get_clientsession(hass),
@@ -64,6 +73,7 @@ class ElecPricesDataUpdateCoordinator(DataUpdateCoordinator[EsiosApiData]):
             power=entry.data[ATTR_POWER],
             power_valley=entry.data[ATTR_POWER_P3],
             api_token=entry.data.get(CONF_API_TOKEN),
+            sensor_keys=tuple(sensor_keys),
         )
         super().__init__(
             hass, _LOGGER, name=DOMAIN, update_interval=timedelta(minutes=30)
@@ -84,7 +94,7 @@ class ElecPricesDataUpdateCoordinator(DataUpdateCoordinator[EsiosApiData]):
         if (
             not api_data
             or not api_data.sensors
-            or not all(api_data.availability.values())
+            or not any(api_data.availability.values())
         ):
             raise UpdateFailed
         return api_data

homeassistant/components/pvpc_hourly_pricing/const.py

@@ -1,5 +1,5 @@
 """Constant values for pvpc_hourly_pricing."""
-from aiopvpc import TARIFFS
+from aiopvpc.const import TARIFFS
 import voluptuous as vol
 
 DOMAIN = "pvpc_hourly_pricing"

homeassistant/components/pvpc_hourly_pricing/helpers.py

@@ -0,0 +1,49 @@
+"""Helper functions to relate sensors keys and unique ids."""
+from aiopvpc.const import (
+    ALL_SENSORS,
+    KEY_INJECTION,
+    KEY_MAG,
+    KEY_OMIE,
+    KEY_PVPC,
+    TARIFFS,
+)
+
+from homeassistant.helpers.entity_registry import RegistryEntry
+
+_ha_uniqueid_to_sensor_key = {
+    TARIFFS[0]: KEY_PVPC,
+    TARIFFS[1]: KEY_PVPC,
+    f"{TARIFFS[0]}_{KEY_INJECTION}": KEY_INJECTION,
+    f"{TARIFFS[1]}_{KEY_INJECTION}": KEY_INJECTION,
+    f"{TARIFFS[0]}_{KEY_MAG}": KEY_MAG,
+    f"{TARIFFS[1]}_{KEY_MAG}": KEY_MAG,
+    f"{TARIFFS[0]}_{KEY_OMIE}": KEY_OMIE,
+    f"{TARIFFS[1]}_{KEY_OMIE}": KEY_OMIE,
+}
+
+
+def get_enabled_sensor_keys(
+    using_private_api: bool, entries: list[RegistryEntry]
+) -> set[str]:
+    """Get enabled API indicators."""
+    if not using_private_api:
+        return {KEY_PVPC}
+    if len(entries) > 1:
+        # activate only enabled sensors
+        return {
+            _ha_uniqueid_to_sensor_key[sensor.unique_id]
+            for sensor in entries
+            if not sensor.disabled
+        }
+    # default sensors when enabling token access
+    return {KEY_PVPC, KEY_INJECTION}
+
+
+def make_sensor_unique_id(config_entry_id: str | None, sensor_key: str) -> str:
+    """Generate unique_id for each sensor kind and config entry."""
+    assert sensor_key in ALL_SENSORS
+    assert config_entry_id is not None
+    if sensor_key == KEY_PVPC:
+        # for old compatibility
+        return config_entry_id
+    return f"{config_entry_id}_{sensor_key}"

homeassistant/components/pvpc_hourly_pricing/sensor.py

@@ -6,6 +6,8 @@ from datetime import datetime
 import logging
 from typing import Any
 
+from aiopvpc.const import KEY_INJECTION, KEY_MAG, KEY_OMIE, KEY_PVPC
+
 from homeassistant.components.sensor import (
     SensorEntity,
     SensorEntityDescription,
@@ -22,19 +24,49 @@ from homeassistant.helpers.update_coordinator import CoordinatorEntity
 
 from . import ElecPricesDataUpdateCoordinator
 from .const import DOMAIN
+from .helpers import make_sensor_unique_id
 
 _LOGGER = logging.getLogger(__name__)
 PARALLEL_UPDATES = 1
 SENSOR_TYPES: tuple[SensorEntityDescription, ...] = (
     SensorEntityDescription(
-        key="PVPC",
+        key=KEY_PVPC,
         icon="mdi:currency-eur",
         native_unit_of_measurement=f"{CURRENCY_EURO}/{UnitOfEnergy.KILO_WATT_HOUR}",
         state_class=SensorStateClass.MEASUREMENT,
+        suggested_display_precision=5,
         name="PVPC",
     ),
+    SensorEntityDescription(
+        key=KEY_INJECTION,
+        icon="mdi:transmission-tower-export",
+        native_unit_of_measurement=f"{CURRENCY_EURO}/{UnitOfEnergy.KILO_WATT_HOUR}",
+        state_class=SensorStateClass.MEASUREMENT,
+        suggested_display_precision=5,
+        name="Injection Price",
+    ),
+    SensorEntityDescription(
+        key=KEY_MAG,
+        icon="mdi:bank-transfer",
+        native_unit_of_measurement=f"{CURRENCY_EURO}/{UnitOfEnergy.KILO_WATT_HOUR}",
+        state_class=SensorStateClass.MEASUREMENT,
+        suggested_display_precision=5,
+        name="MAG tax",
+        entity_registry_enabled_default=False,
+    ),
+    SensorEntityDescription(
+        key=KEY_OMIE,
+        icon="mdi:shopping",
+        native_unit_of_measurement=f"{CURRENCY_EURO}/{UnitOfEnergy.KILO_WATT_HOUR}",
+        state_class=SensorStateClass.MEASUREMENT,
+        suggested_display_precision=5,
+        name="OMIE Price",
+        entity_registry_enabled_default=False,
+    ),
 )
 _PRICE_SENSOR_ATTRIBUTES_MAP = {
+    "data_id": "data_id",
+    "name": "data_name",
     "tariff": "tariff",
     "period": "period",
     "available_power": "available_power",
@@ -119,7 +151,11 @@ async def async_setup_entry(
 ) -> None:
     """Set up the electricity price sensor from config_entry."""
     coordinator: ElecPricesDataUpdateCoordinator = hass.data[DOMAIN][entry.entry_id]
-    async_add_entities([ElecPriceSensor(coordinator, SENSOR_TYPES[0], entry.unique_id)])
+    sensors = [ElecPriceSensor(coordinator, SENSOR_TYPES[0], entry.unique_id)]
+    if coordinator.api.using_private_api:
+        for sensor_desc in SENSOR_TYPES[1:]:
+            sensors.append(ElecPriceSensor(coordinator, sensor_desc, entry.unique_id))
+    async_add_entities(sensors)
 
 
 class ElecPriceSensor(CoordinatorEntity[ElecPricesDataUpdateCoordinator], SensorEntity):
@@ -137,7 +173,7 @@ class ElecPriceSensor(CoordinatorEntity[ElecPricesDataUpdateCoordinator], Sensor
         super().__init__(coordinator)
         self.entity_description = description
         self._attr_attribution = coordinator.api.attribution
-        self._attr_unique_id = unique_id
+        self._attr_unique_id = make_sensor_unique_id(unique_id, description.key)
         self._attr_device_info = DeviceInfo(
             configuration_url="https://api.esios.ree.es",
             entry_type=DeviceEntryType.SERVICE,
@@ -146,9 +182,23 @@ class ElecPriceSensor(CoordinatorEntity[ElecPricesDataUpdateCoordinator], Sensor
             name="ESIOS",
         )
 
+    @property
+    def available(self) -> bool:
+        """Return if entity is available."""
+        return self.coordinator.data.availability.get(
+            self.entity_description.key, False
+        )
+
     async def async_added_to_hass(self) -> None:
         """Handle entity which will be added."""
         await super().async_added_to_hass()
+        # Enable API downloads for this sensor
+        self.coordinator.api.update_active_sensors(self.entity_description.key, True)
+        self.async_on_remove(
+            lambda: self.coordinator.api.update_active_sensors(
+                self.entity_description.key, False
+            )
+        )
 
         # Update 'state' value in hour changes
         self.async_on_remove(
@@ -157,10 +207,10 @@ class ElecPriceSensor(CoordinatorEntity[ElecPricesDataUpdateCoordinator], Sensor
             )
         )
         _LOGGER.debug(
-            "Setup of price sensor %s (%s) with tariff '%s'",
+            "Setup of ESIOS sensor %s (%s, unique_id: %s)",
-            self.name,
+            self.entity_description.key,
             self.entity_id,
-            self.coordinator.api.tariff,
+            self._attr_unique_id,
         )
 
     @callback

tests/components/pvpc_hourly_pricing/conftest.py

@@ -11,6 +11,7 @@ from tests.test_util.aiohttp import AiohttpClientMocker
 
 FIXTURE_JSON_PUBLIC_DATA_2023_01_06 = "PVPC_DATA_2023_01_06.json"
 FIXTURE_JSON_ESIOS_DATA_PVPC_2023_01_06 = "PRICES_ESIOS_1001_2023_01_06.json"
+_ESIOS_INDICATORS_FOR_EACH_SENSOR = ("1001", "1739", "1900", "10211")
 
 
 def check_valid_state(state, tariff: str, value=None, key_attr=None):
@@ -43,18 +44,19 @@ def pvpc_aioclient_mock(aioclient_mock: AiohttpClientMocker):
         "https://api.esios.ree.es/archives/70/download_json?locale=es&date={0}"
     )
     mask_url_esios = (
-        "https://api.esios.ree.es/indicators/1001"
+        "https://api.esios.ree.es/indicators/{0}"
-        "?start_date={0}T00:00&end_date={0}T23:59"
+        "?start_date={1}T00:00&end_date={1}T23:59"
     )
     example_day = "2023-01-06"
     aioclient_mock.get(
         mask_url_public.format(example_day),
         text=load_fixture(f"{DOMAIN}/{FIXTURE_JSON_PUBLIC_DATA_2023_01_06}"),
     )
-    aioclient_mock.get(
+    for esios_ind in _ESIOS_INDICATORS_FOR_EACH_SENSOR:
-        mask_url_esios.format(example_day),
+        aioclient_mock.get(
-        text=load_fixture(f"{DOMAIN}/{FIXTURE_JSON_ESIOS_DATA_PVPC_2023_01_06}"),
+            mask_url_esios.format(esios_ind, example_day),
-    )
+            text=load_fixture(f"{DOMAIN}/{FIXTURE_JSON_ESIOS_DATA_PVPC_2023_01_06}"),
+        )
 
     # simulate missing days
     aioclient_mock.get(
@@ -62,22 +64,24 @@ def pvpc_aioclient_mock(aioclient_mock: AiohttpClientMocker):
         status=HTTPStatus.OK,
         text='{"message":"No values for specified archive"}',
     )
-    aioclient_mock.get(
+    for esios_ind in _ESIOS_INDICATORS_FOR_EACH_SENSOR:
-        mask_url_esios.format("2023-01-07"),
+        aioclient_mock.get(
-        status=HTTPStatus.OK,
+            mask_url_esios.format(esios_ind, "2023-01-07"),
-        text=(
+            status=HTTPStatus.OK,
-            '{"indicator":{"name":"Término de facturación de energía activa del '
+            text=(
-            'PVPC 2.0TD","short_name":"PVPC T. 2.0TD","id":1001,"composited":false,'
+                '{"indicator":{"name":"Término de facturación de energía activa del '
-            '"step_type":"linear","disaggregated":true,"magnitud":'
+                'PVPC 2.0TD","short_name":"PVPC T. 2.0TD","id":1001,"composited":false,'
-            '[{"name":"Precio","id":23}],"tiempo":[{"name":"Hora","id":4}],"geos":[],'
+                '"step_type":"linear","disaggregated":true,"magnitud":'
-            '"values_updated_at":null,"values":[]}}'
+                '[{"name":"Precio","id":23}],"tiempo":[{"name":"Hora","id":4}],"geos":[],'
-        ),
+                '"values_updated_at":null,"values":[]}}'
-    )
+            ).replace("1001", esios_ind),
+        )
     # simulate bad authentication
-    aioclient_mock.get(
+    for esios_ind in _ESIOS_INDICATORS_FOR_EACH_SENSOR:
-        mask_url_esios.format("2023-01-08"),
+        aioclient_mock.get(
-        status=HTTPStatus.UNAUTHORIZED,
+            mask_url_esios.format(esios_ind, "2023-01-08"),
-        text="HTTP Token: Access denied.",
+            status=HTTPStatus.UNAUTHORIZED,
-    )
+            text="HTTP Token: Access denied.",
+        )
 
     return aioclient_mock

tests/components/pvpc_hourly_pricing/test_config_flow.py

@@ -64,6 +64,10 @@ async def test_config_flow(
     check_valid_state(state, tariff=TARIFFS[1])
     assert pvpc_aioclient_mock.call_count == 1
 
+    # no extra sensors created without enabled API token
+    state_inyection = hass.states.get("sensor.injection_price")
+    assert state_inyection is None
+
     # Check abort when configuring another with same tariff
     result = await hass.config_entries.flow.async_init(
         DOMAIN, context={"source": config_entries.SOURCE_USER}
@@ -117,18 +121,27 @@ async def test_config_flow(
     assert pvpc_aioclient_mock.call_count == 2
 
     result = await hass.config_entries.options.async_configure(
-        result["flow_id"], user_input={CONF_API_TOKEN: "good-token"}
+        result["flow_id"], user_input={CONF_API_TOKEN: "test-token"}
     )
     assert result["type"] == data_entry_flow.FlowResultType.CREATE_ENTRY
     assert pvpc_aioclient_mock.call_count == 2
     await hass.async_block_till_done()
     state = hass.states.get("sensor.esios_pvpc")
     check_valid_state(state, tariff=TARIFFS[1])
-    assert pvpc_aioclient_mock.call_count == 3
+    assert pvpc_aioclient_mock.call_count == 4
     assert state.attributes["period"] == "P3"
     assert state.attributes["next_period"] == "P2"
     assert state.attributes["available_power"] == 4600
 
+    state_inyection = hass.states.get("sensor.esios_injection_price")
+    state_mag = hass.states.get("sensor.esios_mag_tax")
+    state_omie = hass.states.get("sensor.esios_omie_price")
+    assert state_inyection
+    assert not state_mag
+    assert not state_omie
+    assert "period" not in state_inyection.attributes
+    assert "available_power" not in state_inyection.attributes
+
     # check update failed
     freezer.tick(timedelta(days=1))
     async_fire_time_changed(hass)
@@ -136,7 +149,7 @@ async def test_config_flow(
     state = hass.states.get("sensor.esios_pvpc")
     check_valid_state(state, tariff=TARIFFS[0], value="unavailable")
     assert "period" not in state.attributes
-    assert pvpc_aioclient_mock.call_count == 4
+    assert pvpc_aioclient_mock.call_count == 6
 
     # disable api token in options
     result = await hass.config_entries.options.async_init(config_entry.entry_id)
@@ -148,8 +161,18 @@ async def test_config_flow(
         user_input={ATTR_POWER: 3.0, ATTR_POWER_P3: 4.6, CONF_USE_API_TOKEN: False},
     )
     assert result["type"] == data_entry_flow.FlowResultType.CREATE_ENTRY
-    assert pvpc_aioclient_mock.call_count == 4
+    assert pvpc_aioclient_mock.call_count == 6
     await hass.async_block_till_done()
+    assert pvpc_aioclient_mock.call_count == 7
+
+    state = hass.states.get("sensor.esios_pvpc")
+    state_inyection = hass.states.get("sensor.esios_injection_price")
+    state_mag = hass.states.get("sensor.esios_mag_tax")
+    state_omie = hass.states.get("sensor.esios_omie_price")
+    check_valid_state(state, tariff=TARIFFS[1])
+    assert state_inyection.state == "unavailable"
+    assert not state_mag
+    assert not state_omie
 
 
 async def test_reauth(
@@ -181,7 +204,7 @@ async def test_reauth(
     assert pvpc_aioclient_mock.call_count == 0
 
     result = await hass.config_entries.flow.async_configure(
-        result["flow_id"], user_input={CONF_API_TOKEN: "bad-token"}
+        result["flow_id"], user_input={CONF_API_TOKEN: "test-token"}
     )
     assert result["type"] == data_entry_flow.FlowResultType.FORM
     assert result["step_id"] == "api_token"
@@ -190,17 +213,17 @@ async def test_reauth(
 
     freezer.move_to(_MOCK_TIME_VALID_RESPONSES)
     result = await hass.config_entries.flow.async_configure(
-        result["flow_id"], user_input={CONF_API_TOKEN: "good-token"}
+        result["flow_id"], user_input={CONF_API_TOKEN: "test-token"}
     )
     assert result["type"] == data_entry_flow.FlowResultType.CREATE_ENTRY
     config_entry = result["result"]
-    assert pvpc_aioclient_mock.call_count == 3
+    assert pvpc_aioclient_mock.call_count == 4
 
     # check reauth trigger with bad-auth responses
     freezer.move_to(_MOCK_TIME_BAD_AUTH_RESPONSES)
     async_fire_time_changed(hass, _MOCK_TIME_BAD_AUTH_RESPONSES)
     await hass.async_block_till_done()
-    assert pvpc_aioclient_mock.call_count == 4
+    assert pvpc_aioclient_mock.call_count == 6
 
     result = hass.config_entries.flow.async_progress_by_handler(DOMAIN)[0]
     assert result["context"]["entry_id"] == config_entry.entry_id
@@ -208,11 +231,11 @@ async def test_reauth(
     assert result["step_id"] == "reauth_confirm"
 
     result = await hass.config_entries.flow.async_configure(
-        result["flow_id"], user_input={CONF_API_TOKEN: "bad-token"}
+        result["flow_id"], user_input={CONF_API_TOKEN: "test-token"}
     )
     assert result["type"] == data_entry_flow.FlowResultType.FORM
     assert result["step_id"] == "reauth_confirm"
-    assert pvpc_aioclient_mock.call_count == 5
+    assert pvpc_aioclient_mock.call_count == 7
 
     result = hass.config_entries.flow.async_progress_by_handler(DOMAIN)[0]
     assert result["context"]["entry_id"] == config_entry.entry_id
@@ -222,11 +245,11 @@ async def test_reauth(
     freezer.move_to(_MOCK_TIME_VALID_RESPONSES)
     async_fire_time_changed(hass, _MOCK_TIME_VALID_RESPONSES)
     result = await hass.config_entries.flow.async_configure(
-        result["flow_id"], user_input={CONF_API_TOKEN: "good-token"}
+        result["flow_id"], user_input={CONF_API_TOKEN: "test-token"}
     )
     assert result["type"] == data_entry_flow.FlowResultType.ABORT
     assert result["reason"] == "reauth_successful"
-    assert pvpc_aioclient_mock.call_count == 6
+    assert pvpc_aioclient_mock.call_count == 8
 
     await hass.async_block_till_done()
-    assert pvpc_aioclient_mock.call_count == 7
+    assert pvpc_aioclient_mock.call_count == 10