diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceLabelFormatter.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceLabelFormatter.php
index ef033cd4665..fe482b2fbbc 100644
--- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceLabelFormatter.php
+++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceLabelFormatter.php
@@ -2,6 +2,7 @@
 
 namespace Drupal\Core\Field\Plugin\Field\FieldFormatter;
 
+use Drupal\Core\Cache\CacheableMetadata;
 use Drupal\Core\Entity\EntityInterface;
 use Drupal\Core\Entity\Exception\UndefinedLinkTemplateException;
 use Drupal\Core\Field\Attribute\FieldFormatter;
@@ -61,7 +62,9 @@ class EntityReferenceLabelFormatter extends EntityReferenceFormatterBase {
     $output_as_link = $this->getSetting('link');
 
     foreach ($this->getEntitiesToView($items, $langcode) as $delta => $entity) {
+      $elements[$delta] = ['#entity' => $entity];
       $label = $entity->label();
+      $cacheability = CacheableMetadata::createFromObject($entity);
       // If the link is to be displayed and the entity has a uri, display a
       // link.
       if ($output_as_link && !$entity->isNew()) {
@@ -74,31 +77,38 @@ class EntityReferenceLabelFormatter extends EntityReferenceFormatterBase {
           // entity type doesn't have a link template nor a valid
           // "uri_callback", so don't bother trying to output a link for the
           // rest of the referenced entities.
-          $output_as_link = FALSE;
+          $elements[$delta]['#plain_text'] = $label;
+          $cacheability->applyTo($elements[$delta]);
+          continue;
         }
-      }
 
-      if ($output_as_link && isset($uri) && !$entity->isNew()) {
-        $elements[$delta] = [
-          '#type' => 'link',
-          '#title' => $label,
-          '#url' => $uri,
-          '#options' => $uri->getOptions(),
-        ];
+        $uri_access = $uri->access(return_as_object: TRUE);
+        $cacheability->addCacheableDependency($uri_access);
+        if ($uri_access->isAllowed()) {
+          $elements[$delta] += [
+            '#type' => 'link',
+            '#title' => $label,
+            '#url' => $uri,
+            '#options' => $uri->getOptions(),
+          ];
 
-        if (!empty($items[$delta]->_attributes)) {
-          $elements[$delta]['#options'] += ['attributes' => []];
-          $elements[$delta]['#options']['attributes'] += $items[$delta]->_attributes;
-          // Unset field item attributes since they have been included in the
-          // formatter output and shouldn't be rendered in the field template.
-          unset($items[$delta]->_attributes);
+          if (!empty($items[$delta]->_attributes)) {
+            $elements[$delta]['#options'] += ['attributes' => []];
+            $elements[$delta]['#options']['attributes'] += $items[$delta]->_attributes;
+            // Unset field item attributes since they have been included in the
+            // formatter output and shouldn't be rendered in the field template.
+            unset($items[$delta]->_attributes);
+          }
+        }
+        else {
+          $elements[$delta]['#plain_text'] = $label;
         }
       }
       else {
-        $elements[$delta] = ['#plain_text' => $label];
+        $elements[$delta]['#plain_text'] = $label;
       }
-      $elements[$delta]['#entity'] = $entity;
-      $elements[$delta]['#cache']['tags'] = $entity->getCacheTags();
+
+      $cacheability->applyTo($elements[$delta]);
     }
 
     return $elements;
diff --git a/core/modules/field/tests/src/Kernel/EntityReference/EntityReferenceFormatterTest.php b/core/modules/field/tests/src/Kernel/EntityReference/EntityReferenceFormatterTest.php
index 9f554f11028..c2e0d85f156 100644
--- a/core/modules/field/tests/src/Kernel/EntityReference/EntityReferenceFormatterTest.php
+++ b/core/modules/field/tests/src/Kernel/EntityReference/EntityReferenceFormatterTest.php
@@ -8,6 +8,7 @@ use Drupal\Core\Cache\Cache;
 use Drupal\Core\Cache\CacheableMetadata;
 use Drupal\Core\Field\FieldStorageDefinitionInterface;
 use Drupal\Core\Field\Plugin\Field\FieldFormatter\EntityReferenceEntityFormatter;
+use Drupal\entity_test\Entity\EntityTest;
 use Drupal\field\Entity\FieldConfig;
 use Drupal\field\Entity\FieldStorageConfig;
 use Drupal\filter\Entity\FilterFormat;
@@ -349,6 +350,13 @@ class EntityReferenceFormatterTest extends EntityKernelTestBase {
     $renderer = $this->container->get('renderer');
     $formatter = 'entity_reference_label';
 
+    // We need to create an anonymous user for access checks in the formatter.
+    $this->createUser(values: [
+      'uid' => 0,
+      'status' => 0,
+      'name' => '',
+    ]);
+
     // The 'link' settings is TRUE by default.
     $build = $this->buildRenderArray([$this->referencedEntity, $this->unsavedReferencedEntity], $formatter);
 
@@ -405,6 +413,15 @@ class EntityReferenceFormatterTest extends EntityKernelTestBase {
 
     $build = $this->buildRenderArray([$referenced_entity_with_no_link_template], $formatter, ['link' => TRUE]);
     $this->assertEquals($referenced_entity_with_no_link_template->label(), $build[0]['#plain_text'], sprintf('The markup returned by the %s formatter is correct for an entity type with no valid link template.', $formatter));
+
+    // Test link visibility if the URL is not accessible.
+    $entity_with_user = EntityTest::create([
+      'name' => $this->randomMachineName(),
+      'user_id' => $this->createUser(),
+    ]);
+    $entity_with_user->save();
+    $build = $entity_with_user->get('user_id')->view(['type' => $formatter, 'settings' => ['link' => TRUE]]);
+    $this->assertEquals($build[0]['#plain_text'], $entity_with_user->get('user_id')->entity->label(), 'For inaccessible links, the label should be displayed in plain text.');
   }
 
   /**
diff --git a/core/modules/media/tests/src/Functional/MediaAccessTest.php b/core/modules/media/tests/src/Functional/MediaAccessTest.php
index 3ca5b7e65e0..62d43942a64 100644
--- a/core/modules/media/tests/src/Functional/MediaAccessTest.php
+++ b/core/modules/media/tests/src/Functional/MediaAccessTest.php
@@ -220,6 +220,7 @@ class MediaAccessTest extends MediaFunctionalTestBase {
     $mediaOverviewRole = $this->createRole(['access content overview', 'access media overview']);
     $this->nonAdminUser->addRole($mediaOverviewRole)->save();
 
+    $this->grantPermissions($role, ['access user profiles']);
     $this->drupalGet('admin/content');
     $assert_session->linkByHrefExists('/admin/content/media');
     $this->clickLink('Media');