diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index d6b550d2a9f..3cdd1a3a39c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -712,6 +712,16 @@ jobs:
         run: |
           echo "Failed to restore Python virtual environment from cache"
           exit 1
+      - name: Register Python problem matcher
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - name: Install Pytest Annotation plugin
+        run: |
+          . venv/bin/activate
+          # Ideally this should be part of our dependencies
+          # However this plugin is fairly new and doesn't run correctly
+          # on a non-GitHub environment.
+          pip install pytest-github-actions-annotate-failures
       - name: Run pytest
         run: |
           . venv/bin/activate
diff --git a/.github/workflows/matchers/python.json b/.github/workflows/matchers/python.json
new file mode 100644
index 00000000000..3e5d8d5b8ba
--- /dev/null
+++ b/.github/workflows/matchers/python.json
@@ -0,0 +1,18 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "python",
+      "pattern": [
+        {
+          "regexp": "^\\s*File\\s\\\"(.*)\\\",\\sline\\s(\\d+),\\sin\\s(.*)$",
+          "file": 1,
+          "line": 2
+        },
+        {
+          "regexp": "^\\s*raise\\s(.*)\\(\\'(.*)\\'\\)$",
+          "message": 2
+        }
+      ]
+    }
+  ]
+}