diff --git a/homeassistant/components/bluetooth/manager.py b/homeassistant/components/bluetooth/manager.py
index c3a0e0998f1..d29023acef7 100644
--- a/homeassistant/components/bluetooth/manager.py
+++ b/homeassistant/components/bluetooth/manager.py
@@ -127,6 +127,7 @@ class BluetoothManager:
         self._non_connectable_scanners: list[BaseHaScanner] = []
         self._connectable_scanners: list[BaseHaScanner] = []
         self._adapters: dict[str, AdapterDetails] = {}
+        self._sources: set[str] = set()
 
     @property
     def supports_passive_scan(self) -> bool:
@@ -379,6 +380,7 @@ class BluetoothManager:
         if (
             (old_service_info := all_history.get(address))
             and source != old_service_info.source
+            and old_service_info.source in self._sources
             and self._prefer_previous_adv_from_different_source(
                 old_service_info, service_info
             )
@@ -398,6 +400,7 @@ class BluetoothManager:
                     # the old connectable advertisement
                     or (
                         source != old_connectable_service_info.source
+                        and old_connectable_service_info.source in self._sources
                         and self._prefer_previous_adv_from_different_source(
                             old_connectable_service_info, service_info
                         )
@@ -597,8 +600,10 @@ class BluetoothManager:
         def _unregister_scanner() -> None:
             self._advertisement_tracker.async_remove_source(scanner.source)
             scanners.remove(scanner)
+            self._sources.remove(scanner.source)
 
         scanners.append(scanner)
+        self._sources.add(scanner.source)
         return _unregister_scanner
 
     @hass_callback
diff --git a/tests/components/bluetooth/test_manager.py b/tests/components/bluetooth/test_manager.py
index c6a65046ef9..0375f68309f 100644
--- a/tests/components/bluetooth/test_manager.py
+++ b/tests/components/bluetooth/test_manager.py
@@ -5,11 +5,14 @@ from unittest.mock import AsyncMock, MagicMock, patch
 
 from bleak.backends.scanner import BLEDevice
 from bluetooth_adapters import AdvertisementHistory
+import pytest
 
 from homeassistant.components import bluetooth
+from homeassistant.components.bluetooth import models
 from homeassistant.components.bluetooth.manager import (
     FALLBACK_MAXIMUM_STALE_ADVERTISEMENT_SECONDS,
 )
+from homeassistant.core import HomeAssistant
 from homeassistant.setup import async_setup_component
 
 from . import (
@@ -20,8 +23,28 @@ from . import (
 )
 
 
+@pytest.fixture
+def register_hci0_scanner(hass: HomeAssistant) -> None:
+    """Register an hci0 scanner."""
+    cancel = bluetooth.async_register_scanner(
+        hass, models.BaseHaScanner(hass, "hci0"), True
+    )
+    yield
+    cancel()
+
+
+@pytest.fixture
+def register_hci1_scanner(hass: HomeAssistant) -> None:
+    """Register an hci1 scanner."""
+    cancel = bluetooth.async_register_scanner(
+        hass, models.BaseHaScanner(hass, "hci1"), True
+    )
+    yield
+    cancel()
+
+
 async def test_advertisements_do_not_switch_adapters_for_no_reason(
-    hass, enable_bluetooth
+    hass, enable_bluetooth, register_hci0_scanner, register_hci1_scanner
 ):
     """Test we only switch adapters when needed."""
 
@@ -68,7 +91,9 @@ async def test_advertisements_do_not_switch_adapters_for_no_reason(
     )
 
 
-async def test_switching_adapters_based_on_rssi(hass, enable_bluetooth):
+async def test_switching_adapters_based_on_rssi(
+    hass, enable_bluetooth, register_hci0_scanner, register_hci1_scanner
+):
     """Test switching adapters based on rssi."""
 
     address = "44:44:33:11:23:45"
@@ -122,7 +147,9 @@ async def test_switching_adapters_based_on_rssi(hass, enable_bluetooth):
     )
 
 
-async def test_switching_adapters_based_on_zero_rssi(hass, enable_bluetooth):
+async def test_switching_adapters_based_on_zero_rssi(
+    hass, enable_bluetooth, register_hci0_scanner, register_hci1_scanner
+):
     """Test switching adapters based on zero rssi."""
 
     address = "44:44:33:11:23:45"
@@ -176,7 +203,9 @@ async def test_switching_adapters_based_on_zero_rssi(hass, enable_bluetooth):
     )
 
 
-async def test_switching_adapters_based_on_stale(hass, enable_bluetooth):
+async def test_switching_adapters_based_on_stale(
+    hass, enable_bluetooth, register_hci0_scanner, register_hci1_scanner
+):
     """Test switching adapters based on the previous advertisement being stale."""
 
     address = "44:44:33:11:23:41"
@@ -256,7 +285,7 @@ async def test_restore_history_from_dbus(hass, one_adapter):
 
 
 async def test_switching_adapters_based_on_rssi_connectable_to_non_connectable(
-    hass, enable_bluetooth
+    hass, enable_bluetooth, register_hci0_scanner, register_hci1_scanner
 ):
     """Test switching adapters based on rssi from connectable to non connectable."""
 
@@ -339,7 +368,7 @@ async def test_switching_adapters_based_on_rssi_connectable_to_non_connectable(
 
 
 async def test_connectable_advertisement_can_be_retrieved_with_best_path_is_non_connectable(
-    hass, enable_bluetooth
+    hass, enable_bluetooth, register_hci0_scanner, register_hci1_scanner
 ):
     """Test we can still get a connectable BLEDevice when the best path is non-connectable.
 
@@ -384,3 +413,54 @@ async def test_connectable_advertisement_can_be_retrieved_with_best_path_is_non_
         bluetooth.async_ble_device_from_address(hass, address, True)
         is switchbot_device_poor_signal
     )
+
+
+async def test_switching_adapters_when_one_goes_away(
+    hass, enable_bluetooth, register_hci0_scanner
+):
+    """Test switching adapters when one goes away."""
+    cancel_hci2 = bluetooth.async_register_scanner(
+        hass, models.BaseHaScanner(hass, "hci2"), True
+    )
+
+    address = "44:44:33:11:23:45"
+
+    switchbot_device_good_signal = BLEDevice(address, "wohand_good_signal")
+    switchbot_adv_good_signal = generate_advertisement_data(
+        local_name="wohand_good_signal", service_uuids=[], rssi=-60
+    )
+    inject_advertisement_with_source(
+        hass, switchbot_device_good_signal, switchbot_adv_good_signal, "hci2"
+    )
+
+    assert (
+        bluetooth.async_ble_device_from_address(hass, address)
+        is switchbot_device_good_signal
+    )
+
+    switchbot_device_poor_signal = BLEDevice(address, "wohand_poor_signal")
+    switchbot_adv_poor_signal = generate_advertisement_data(
+        local_name="wohand_poor_signal", service_uuids=[], rssi=-100
+    )
+    inject_advertisement_with_source(
+        hass, switchbot_device_poor_signal, switchbot_adv_poor_signal, "hci0"
+    )
+
+    # We want to prefer the good signal when we have options
+    assert (
+        bluetooth.async_ble_device_from_address(hass, address)
+        is switchbot_device_good_signal
+    )
+
+    cancel_hci2()
+
+    inject_advertisement_with_source(
+        hass, switchbot_device_poor_signal, switchbot_adv_poor_signal, "hci0"
+    )
+
+    # Now that hci2 is gone, we should prefer the poor signal
+    # since no poor signal is better than no signal
+    assert (
+        bluetooth.async_ble_device_from_address(hass, address)
+        is switchbot_device_poor_signal
+    )