diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go
index 6e5a656e629..359b8e57be9 100644
--- a/pkg/store/bucket.go
+++ b/pkg/store/bucket.go
@@ -2978,6 +2978,12 @@ func toPostingGroup(ctx context.Context, lvalsFn func(name string) ([]string, er
 			return nil, nil, err
 		}
 
+		// If the matcher is ="" or =~"", it is the same as removing all values for the label.
+		// We can skip calling `Matches` here.
+		if m.Value == "" && (m.Type == labels.MatchEqual || m.Type == labels.MatchRegexp) {
+			return newPostingGroup(true, m.Name, nil, vals), vals, nil
+		}
+
 		for i, val := range vals {
 			if (i+1)%checkContextEveryNIterations == 0 && ctx.Err() != nil {
 				return nil, nil, ctx.Err()
@@ -3006,6 +3012,12 @@ func toPostingGroup(ctx context.Context, lvalsFn func(name string) ([]string, er
 		return nil, nil, err
 	}
 
+	// If the matcher is !="" or !~"", it is the same as adding all values for the label.
+	// We can skip calling `Matches` here.
+	if m.Value == "" && (m.Type == labels.MatchNotEqual || m.Type == labels.MatchNotRegexp) {
+		return newPostingGroup(false, m.Name, vals, nil), vals, nil
+	}
+
 	var toAdd []string
 	for i, val := range vals {
 		if (i+1)%checkContextEveryNIterations == 0 && ctx.Err() != nil {