diff --git a/hack/cluster-version-util/task_graph.go b/hack/cluster-version-util/task_graph.go
index a0f8e304a..7f04e1eae 100644
--- a/hack/cluster-version-util/task_graph.go
+++ b/hack/cluster-version-util/task_graph.go
@@ -30,7 +30,7 @@ func newTaskGraphCmd() *cobra.Command {
 
 func runTaskGraphCmd(cmd *cobra.Command, args []string) error {
 	manifestDir := args[0]
-	release, err := payload.LoadUpdate(manifestDir, "")
+	release, err := payload.LoadUpdate(manifestDir, "", "")
 	if err != nil {
 		return err
 	}
diff --git a/pkg/cvo/cvo.go b/pkg/cvo/cvo.go
index a4c561488..c0f3a315e 100644
--- a/pkg/cvo/cvo.go
+++ b/pkg/cvo/cvo.go
@@ -236,7 +236,7 @@ func (vcb *verifyClientBuilder) HTTPClient() (*http.Client, error) {
 // controller that loads and applies content to the cluster. It returns an error if the payload appears to
 // be in error rather than continuing.
 func (optr *Operator) InitializeFromPayload(restConfig *rest.Config, burstRestConfig *rest.Config) error {
-	update, err := payload.LoadUpdate(optr.defaultPayloadDir(), optr.releaseImage)
+	update, err := payload.LoadUpdate(optr.defaultPayloadDir(), optr.releaseImage, optr.exclude)
 	if err != nil {
 		return fmt.Errorf("the local release contents are invalid - no current version can be determined from disk: %v", err)
 	}
diff --git a/pkg/cvo/cvo_scenarios_test.go b/pkg/cvo/cvo_scenarios_test.go
index 053c23666..ed2def9d6 100644
--- a/pkg/cvo/cvo_scenarios_test.go
+++ b/pkg/cvo/cvo_scenarios_test.go
@@ -76,6 +76,7 @@ func setupCVOTest(payloadDir string) (*Operator, map[string]runtime.Object, *fak
 		queue:                       workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "cvo-loop-test"),
 		client:                      client,
 		cvLister:                    &clientCVLister{client: client},
+		exclude:                     "exclude-test",
 	}
 
 	dynamicScheme := runtime.NewScheme()
@@ -90,7 +91,7 @@ func setupCVOTest(payloadDir string) (*Operator, map[string]runtime.Object, *fak
 		wait.Backoff{
 			Steps: 1,
 		},
-		"",
+		"exclude-test",
 	)
 	o.configSync = worker
 
diff --git a/pkg/cvo/sync_worker.go b/pkg/cvo/sync_worker.go
index fe4c16283..d03ea2d92 100644
--- a/pkg/cvo/sync_worker.go
+++ b/pkg/cvo/sync_worker.go
@@ -495,7 +495,7 @@ func (w *SyncWorker) syncOnce(ctx context.Context, work *SyncWork, maxWorkers in
 			return err
 		}
 
-		payloadUpdate, err := payload.LoadUpdate(info.Directory, update.Image)
+		payloadUpdate, err := payload.LoadUpdate(info.Directory, update.Image, w.exclude)
 		if err != nil {
 			reporter.Report(SyncWorkerStatus{
 				Generation:  work.Generation,
@@ -653,7 +653,7 @@ func (w *SyncWorker) apply(ctx context.Context, payloadUpdate *payload.Update, w
 			klog.V(4).Infof("Running sync for %s", task)
 			klog.V(5).Infof("Manifest: %s", string(task.Manifest.Raw))
 
-			ov, ok := getOverrideForManifest(work.Overrides, w.exclude, task.Manifest)
+			ov, ok := getOverrideForManifest(work.Overrides, task.Manifest)
 			if ok && ov.Unmanaged {
 				klog.V(4).Infof("Skipping %s as unmanaged", task)
 				continue
@@ -945,7 +945,7 @@ func newMultipleError(errs []error) error {
 }
 
 // getOverrideForManifest returns the override and true when override exists for manifest.
-func getOverrideForManifest(overrides []configv1.ComponentOverride, excludeIdentifier string, manifest *lib.Manifest) (configv1.ComponentOverride, bool) {
+func getOverrideForManifest(overrides []configv1.ComponentOverride, manifest *lib.Manifest) (configv1.ComponentOverride, bool) {
 	for idx, ov := range overrides {
 		kind, namespace, name := manifest.GVK.Kind, manifest.Object().GetNamespace(), manifest.Object().GetName()
 		if ov.Kind == kind &&
@@ -954,10 +954,6 @@ func getOverrideForManifest(overrides []configv1.ComponentOverride, excludeIdent
 			return overrides[idx], true
 		}
 	}
-	excludeAnnotation := fmt.Sprintf("exclude.release.openshift.io/%s", excludeIdentifier)
-	if annotations := manifest.Object().GetAnnotations(); annotations != nil && annotations[excludeAnnotation] == "true" {
-		return configv1.ComponentOverride{Unmanaged: true}, true
-	}
 	return configv1.ComponentOverride{}, false
 }
 
diff --git a/pkg/cvo/testdata/payloadtest/release-manifests/0000_20_a_exclude.yml b/pkg/cvo/testdata/payloadtest/release-manifests/0000_20_a_exclude.yml
new file mode 100644
index 000000000..22ba8b5f9
--- /dev/null
+++ b/pkg/cvo/testdata/payloadtest/release-manifests/0000_20_a_exclude.yml
@@ -0,0 +1,6 @@
+kind: Test
+apiVersion: v1
+metadata:
+  name: file-20-yml
+  annotations:
+    exclude.release.openshift.io/exclude-test: "true"
diff --git a/pkg/payload/payload.go b/pkg/payload/payload.go
index b8e170c11..b5a3eec31 100644
--- a/pkg/payload/payload.go
+++ b/pkg/payload/payload.go
@@ -107,7 +107,7 @@ type Update struct {
 	Manifests    []lib.Manifest
 }
 
-func LoadUpdate(dir, releaseImage string) (*Update, error) {
+func LoadUpdate(dir, releaseImage, excludeIdentifier string) (*Update, error) {
 	payload, tasks, err := loadUpdatePayloadMetadata(dir, releaseImage)
 	if err != nil {
 		return nil, err
@@ -154,6 +154,15 @@ func LoadUpdate(dir, releaseImage string) (*Update, error) {
 				errs = append(errs, errors.Wrapf(err, "error parsing %s", file.Name()))
 				continue
 			}
+			// Filter out manifests that should be excluded based on annotation
+			filteredMs := []lib.Manifest{}
+			for _, manifest := range ms {
+				if shouldExclude(excludeIdentifier, &manifest) {
+					continue
+				}
+				filteredMs = append(filteredMs, manifest)
+			}
+			ms = filteredMs
 			for i := range ms {
 				ms[i].OriginalFilename = filepath.Base(file.Name())
 			}
@@ -179,6 +188,12 @@ func LoadUpdate(dir, releaseImage string) (*Update, error) {
 	return payload, nil
 }
 
+func shouldExclude(excludeIdentifier string, manifest *lib.Manifest) bool {
+	excludeAnnotation := fmt.Sprintf("exclude.release.openshift.io/%s", excludeIdentifier)
+	annotations := manifest.Object().GetAnnotations()
+	return annotations != nil && annotations[excludeAnnotation] == "true"
+}
+
 // ValidateDirectory checks if a directory can be a candidate update by
 // looking for known files. It returns an error if the directory cannot
 // be an update.
diff --git a/pkg/payload/payload_test.go b/pkg/payload/payload_test.go
index 376d3f158..40d64f49a 100644
--- a/pkg/payload/payload_test.go
+++ b/pkg/payload/payload_test.go
@@ -104,7 +104,7 @@ func Test_loadUpdatePayload(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := LoadUpdate(tt.args.dir, tt.args.releaseImage)
+			got, err := LoadUpdate(tt.args.dir, tt.args.releaseImage, "exclude-test")
 			if (err != nil) != tt.wantErr {
 				t.Errorf("loadUpdatePayload() error = %v, wantErr %v", err, tt.wantErr)
 				return
diff --git a/pkg/payload/task_graph_test.go b/pkg/payload/task_graph_test.go
index 3ae17549b..e0230b4ec 100644
--- a/pkg/payload/task_graph_test.go
+++ b/pkg/payload/task_graph_test.go
@@ -487,7 +487,7 @@ func Test_TaskGraph_real(t *testing.T) {
 	if len(path) == 0 {
 		t.Skip("TEST_GRAPH_PATH unset")
 	}
-	p, err := LoadUpdate(path, "arbitrary/image:1")
+	p, err := LoadUpdate(path, "arbitrary/image:1", "")
 	if err != nil {
 		t.Fatal(err)
 	}