diff --git a/homeassistant/components/imap/coordinator.py b/homeassistant/components/imap/coordinator.py
index d77f7fb05bb..34286ce49fa 100644
--- a/homeassistant/components/imap/coordinator.py
+++ b/homeassistant/components/imap/coordinator.py
@@ -167,7 +167,7 @@ class ImapMessage:
             """
             try:
                 return str(part.get_payload(decode=True).decode(self._charset))
-            except Exception:  # pylint: disable=broad-except
+            except ValueError:
                 return str(part.get_payload())
 
         part: Message