diff --git a/.github/labeler-protected-branches.yml b/.github/labeler-protected-branches.yml
new file mode 100644
index 0000000000000..b8c0af693338b
--- /dev/null
+++ b/.github/labeler-protected-branches.yml
@@ -0,0 +1,23 @@
+# This file is used by .github/workflows/labels.yml
+# This version is only run for Pull Requests from protected branches like staging-next, haskell-updates or python-updates.
+
+"4.workflow: package set update":
+  - any:
+    - head-branch:
+      - '-updates$'
+
+"4.workflow: staging":
+  - any:
+    - head-branch:
+      - '^staging-next$'
+      - '^staging-next-'
+
+"6.topic: haskell":
+  - any:
+    - head-branch:
+      - '^haskell-updates$'
+
+"6.topic: python":
+  - any:
+    - head-branch:
+      - '^python-updates$'
diff --git a/.github/labeler.yml b/.github/labeler.yml
index 78bfb4483c2be..ef7483d6e20bf 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -3,6 +3,12 @@
 
 # keep-sorted start case=no numeric=yes newline_separated=yes skip_lines=1
 
+"4.workflow: backport":
+  - any:
+    - base-branch:
+      - '^release-'
+      - '^staging-'
+
 # NOTE: bsd, darwin and cross-compilation labels are handled by ofborg
 "6.topic: agda":
   - any:
diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml
index 0ae4ee09d5ca7..4aefb306c0489 100644
--- a/.github/workflows/labels.yml
+++ b/.github/workflows/labels.yml
@@ -20,12 +20,23 @@ jobs:
     if: "github.repository_owner == 'NixOS' && !contains(github.event.pull_request.title, '[skip treewide]')"
     steps:
       - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
+        if: "!(github.pull_request.head.repo == 'NixOS' && github.ref_protected)"
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           configuration-path: .github/labeler.yml # default
           sync-labels: true
       - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
+        if: "!(github.pull_request.head.repo == 'NixOS' && github.ref_protected)"
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           configuration-path: .github/labeler-no-sync.yml
           sync-labels: false
+      - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
+        # Protected branches like staging-next, haskell-updates and python-updates get special labels.
+        # This is to avoid the mass of labels there, which is mostly useless - and really annoying for
+        # the backport labels.
+        if: "github.pull_request.head.repo == 'NixOS' && github.ref_protected"
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          configuration-path: .github/labeler-protected-branches.yml
+          sync-labels: true