diff --git a/homeassistant/components/cloud/google_config.py b/homeassistant/components/cloud/google_config.py
index 8f190103e87..f30be66cb42 100644
--- a/homeassistant/components/cloud/google_config.py
+++ b/homeassistant/components/cloud/google_config.py
@@ -195,6 +195,8 @@ class CloudGoogleConfig(AbstractConfig):
         ):
             await async_setup_component(self.hass, GOOGLE_DOMAIN, {})
 
+        sync_entities = False
+
         if self.should_report_state != self.is_reporting_state:
             if self.should_report_state:
                 self.async_enable_report_state()
@@ -203,7 +205,7 @@ class CloudGoogleConfig(AbstractConfig):
 
             # State reporting is reported as a property on entities.
             # So when we change it, we need to sync all entities.
-            await self.async_sync_entities_all()
+            sync_entities = True
 
         # If entity prefs are the same or we have filter in config.yaml,
         # don't sync.
@@ -215,12 +217,17 @@ class CloudGoogleConfig(AbstractConfig):
 
         if self.enabled and not self.is_local_sdk_active:
             self.async_enable_local_sdk()
+            sync_entities = True
         elif not self.enabled and self.is_local_sdk_active:
             self.async_disable_local_sdk()
+            sync_entities = True
 
         self._cur_entity_prefs = prefs.google_entity_configs
         self._cur_default_expose = prefs.google_default_expose
 
+        if sync_entities:
+            await self.async_sync_entities_all()
+
     @callback
     def _handle_entity_registry_updated(self, event: Event) -> None:
         """Handle when entity registry updated."""
diff --git a/homeassistant/components/google_assistant/helpers.py b/homeassistant/components/google_assistant/helpers.py
index b425367f5c3..15a8d832403 100644
--- a/homeassistant/components/google_assistant/helpers.py
+++ b/homeassistant/components/google_assistant/helpers.py
@@ -213,6 +213,9 @@ class AbstractConfig(ABC):
 
     async def async_sync_entities_all(self):
         """Sync all entities to Google for all registered agents."""
+        if not self._store.agent_user_ids:
+            return 204
+
         res = await gather(
             *(
                 self.async_sync_entities(agent_user_id)