diff --git a/lib/asciitable/table.go b/lib/asciitable/table.go index f5ec9b42e3acd..f7c91009e9e70 100644 --- a/lib/asciitable/table.go +++ b/lib/asciitable/table.go @@ -25,6 +25,7 @@ import ( "strings" "text/tabwriter" + "golang.org/x/exp/slices" "golang.org/x/term" ) @@ -208,6 +209,30 @@ func (t *Table) IsHeadless() bool { return true } +// SortRowsBy sorts the table rows with the given column indices as the sorting +// key, optionally performing a stable sort. Column indices out of range are +// ignored - it is the caller's responsibility to ensure the indices are in +// range. +func (t *Table) SortRowsBy(colIdxKey []int, stable bool) { + lessFn := func(a, b []string) bool { + for _, col := range colIdxKey { + limit := min(len(a), len(b)) + if col >= limit { + continue + } + if a[col] != b[col] { + return a[col] < b[col] + } + } + return false + } + if stable { + slices.SortStableFunc(t.rows, lessFn) + } else { + slices.SortFunc(t.rows, lessFn) + } +} + func min(a, b int) int { if a < b { return a diff --git a/tool/tctl/common/collection.go b/tool/tctl/common/collection.go index 10e7ba470cae8..4285e435b82b6 100644 --- a/tool/tctl/common/collection.go +++ b/tool/tctl/common/collection.go @@ -687,7 +687,7 @@ func (c *databaseServerCollection) writeText(w io.Writer, verbose bool) error { labels := stripInternalTeleportLabels(verbose, server.GetDatabase().GetAllLabels()) rows = append(rows, []string{ server.GetHostname(), - server.GetDatabase().GetName(), + nameOrDiscoveredName(server.GetDatabase(), verbose), server.GetDatabase().GetProtocol(), server.GetDatabase().GetURI(), labels, @@ -701,6 +701,8 @@ func (c *databaseServerCollection) writeText(w io.Writer, verbose bool) error { } else { t = asciitable.MakeTableWithTruncatedColumn(headers, rows, "Labels") } + // stable sort by hostname then by name. + t.SortRowsBy([]int{0, 1}, true) _, err := t.AsBuffer().WriteTo(w) return trace.Wrap(err) } @@ -729,7 +731,10 @@ func (c *databaseCollection) writeText(w io.Writer, verbose bool) error { for _, database := range c.databases { labels := stripInternalTeleportLabels(verbose, database.GetAllLabels()) rows = append(rows, []string{ - database.GetName(), database.GetProtocol(), database.GetURI(), labels, + nameOrDiscoveredName(database, verbose), + database.GetProtocol(), + database.GetURI(), + labels, }) } headers := []string{"Name", "Protocol", "URI", "Labels"} @@ -739,6 +744,8 @@ func (c *databaseCollection) writeText(w io.Writer, verbose bool) error { } else { t = asciitable.MakeTableWithTruncatedColumn(headers, rows, "Labels") } + // stable sort by name. + t.SortRowsBy([]int{0}, true) _, err := t.AsBuffer().WriteTo(w) return trace.Wrap(err) } @@ -882,7 +889,7 @@ func (c *kubeServerCollection) writeText(w io.Writer, verbose bool) error { labels := stripInternalTeleportLabels(verbose, types.CombineLabels(kube.GetStaticLabels(), types.LabelsToV2(kube.GetDynamicLabels()))) rows = append(rows, []string{ - kube.GetName(), + nameOrDiscoveredName(kube, verbose), labels, server.GetTeleportVersion(), }) @@ -895,6 +902,8 @@ func (c *kubeServerCollection) writeText(w io.Writer, verbose bool) error { } else { t = asciitable.MakeTableWithTruncatedColumn(headers, rows, "Labels") } + // stable sort by cluster name. + t.SortRowsBy([]int{0}, true) _, err := t.AsBuffer().WriteTo(w) return trace.Wrap(err) @@ -928,12 +937,12 @@ func (c *kubeClusterCollection) resources() (r []types.Resource) { // cluster4 owner=cluster4,region=southcentralus,resource-group=cluster4,subscription-id=subID // If verbose is disabled, labels column can be truncated to fit into the console. func (c *kubeClusterCollection) writeText(w io.Writer, verbose bool) error { - sort.Sort(types.KubeClusters(c.clusters)) var rows [][]string for _, cluster := range c.clusters { labels := stripInternalTeleportLabels(verbose, cluster.GetAllLabels()) rows = append(rows, []string{ - cluster.GetName(), labels, + nameOrDiscoveredName(cluster, verbose), + labels, }) } headers := []string{"Name", "Labels"} @@ -943,6 +952,8 @@ func (c *kubeClusterCollection) writeText(w io.Writer, verbose bool) error { } else { t = asciitable.MakeTableWithTruncatedColumn(headers, rows, "Labels") } + // stable sort by name. + t.SortRowsBy([]int{0}, true) _, err := t.AsBuffer().WriteTo(w) return trace.Wrap(err) } @@ -1193,3 +1204,18 @@ func (c *userGroupCollection) writeText(w io.Writer, verbose bool) error { _, err := t.AsBuffer().WriteTo(w) return trace.Wrap(err) } + +// nameOrDiscoveredName returns the resource's name or its name as originally +// discovered in the cloud by the Teleport Discovery Service. +// In verbose mode, it always returns the resource name. +// In non-verbose mode, if the resource came from discovery and has the +// discovered name label, it returns the discovered name. +func nameOrDiscoveredName(r types.ResourceWithLabels, verbose bool) string { + if !verbose { + originalName, ok := r.GetAllLabels()[types.DiscoveredNameLabel] + if ok && originalName != "" { + return originalName + } + } + return r.GetName() +} diff --git a/tool/tctl/common/collection_test.go b/tool/tctl/common/collection_test.go index 9f52472aebb3f..ee1858974e489 100644 --- a/tool/tctl/common/collection_test.go +++ b/tool/tctl/common/collection_test.go @@ -21,18 +21,23 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" + "github.com/gravitational/teleport/api" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/asciitable" ) var ( - staticLabels = map[string]string{ + staticLabelsFixture = map[string]string{ "label1": "val1", "label2": "val2", "label3": "val3", } + longLabelFixture = map[string]string{ + "ultra_long_label_for_teleport_collection_text_table_formatting": "ultra_long_label_for_teleport_collection_text_table_formatting", + } ) func TestDatabaseResourceMatchersToString(t *testing.T) { @@ -50,80 +55,242 @@ func TestDatabaseResourceMatchersToString(t *testing.T) { require.Equal(t, databaseResourceMatchersToString(resMatch), "(Labels: x=[y])") } -func Test_kubeClusterCollection_writeText(t *testing.T) { - extraLabel := map[string]string{ - "ultra_long_label_for_teleport_kubernetes_list_kube_clusters_method": "ultra_long_label_value_for_teleport_kubernetes_list_kube_clusters_method", +type writeTextTest struct { + collection ResourceCollection + wantVerboseTable func() string + wantNonVerboseTable func() string +} + +func (test *writeTextTest) run(t *testing.T) { + t.Helper() + t.Run("verbose mode", func(t *testing.T) { + t.Helper() + w := &bytes.Buffer{} + err := test.collection.writeText(w, true) + require.NoError(t, err) + diff := cmp.Diff(test.wantVerboseTable(), w.String()) + require.Empty(t, diff) + }) + t.Run("non-verbose mode", func(t *testing.T) { + t.Helper() + w := &bytes.Buffer{} + err := test.collection.writeText(w, false) + require.NoError(t, err) + diff := cmp.Diff(test.wantNonVerboseTable(), w.String()) + require.Empty(t, diff) + }) +} + +func TestResourceCollection_writeText(t *testing.T) { + t.Run("kube clusters", testKubeClusterCollection_writeText) + t.Run("kube servers", testKubeServerCollection_writeText) + t.Run("databases", testDatabaseCollection_writeText) + t.Run("database servers", testDatabaseServerCollection_writeText) +} + +func testKubeClusterCollection_writeText(t *testing.T) { + eksDiscoveredNameLabel := map[string]string{ + types.DiscoveredNameLabel: "cluster3", } kubeClusters := []types.KubeCluster{ mustCreateNewKubeCluster(t, "cluster1", nil), - mustCreateNewKubeCluster(t, "cluster2", extraLabel), - mustCreateNewKubeCluster(t, "afirstCluster", extraLabel), + mustCreateNewKubeCluster(t, "cluster2", longLabelFixture), + mustCreateNewKubeCluster(t, "afirstCluster", longLabelFixture), + mustCreateNewKubeCluster(t, "cluster3-eks-us-west-1-123456789012", eksDiscoveredNameLabel), } - type fields struct { - verbose bool - } - tests := []struct { - name string - fields fields - wantTable func() string - }{ - { - name: "non-verbose mode", - fields: fields{verbose: false}, - wantTable: func() string { - table := asciitable.MakeTableWithTruncatedColumn( - []string{"Name", "Labels"}, - [][]string{ - {"afirstCluster", formatTestLabels(staticLabels, extraLabel, false)}, - {"cluster1", formatTestLabels(staticLabels, nil, false)}, - {"cluster2", formatTestLabels(staticLabels, extraLabel, false)}, - }, - "Labels") - return table.AsBuffer().String() - }, + test := writeTextTest{ + collection: &kubeClusterCollection{clusters: kubeClusters}, + wantNonVerboseTable: func() string { + table := asciitable.MakeTableWithTruncatedColumn( + []string{"Name", "Labels"}, + [][]string{ + {"afirstCluster", formatTestLabels(staticLabelsFixture, longLabelFixture, false)}, + {"cluster1", formatTestLabels(staticLabelsFixture, nil, false)}, + {"cluster2", formatTestLabels(staticLabelsFixture, longLabelFixture, false)}, + {"cluster3", formatTestLabels(staticLabelsFixture, eksDiscoveredNameLabel, false)}, + }, + "Labels") + return table.AsBuffer().String() }, - { - name: "verbose mode", - fields: fields{verbose: true}, - wantTable: func() string { - table := asciitable.MakeTable( - []string{"Name", "Labels"}, - []string{"afirstCluster", formatTestLabels(staticLabels, extraLabel, true)}, - []string{"cluster1", formatTestLabels(staticLabels, nil, true)}, - []string{"cluster2", formatTestLabels(staticLabels, extraLabel, true)}, - ) - return table.AsBuffer().String() - }, + wantVerboseTable: func() string { + table := asciitable.MakeTable( + []string{"Name", "Labels"}, + []string{"afirstCluster", formatTestLabels(staticLabelsFixture, longLabelFixture, true)}, + []string{"cluster1", formatTestLabels(staticLabelsFixture, nil, true)}, + []string{"cluster2", formatTestLabels(staticLabelsFixture, longLabelFixture, true)}, + []string{"cluster3-eks-us-west-1-123456789012", formatTestLabels(staticLabelsFixture, eksDiscoveredNameLabel, true)}, + ) + return table.AsBuffer().String() }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &kubeClusterCollection{ - clusters: kubeClusters, - } - w := &bytes.Buffer{} - err := c.writeText(w, tt.fields.verbose) - require.NoError(t, err) - require.Contains(t, w.String(), tt.wantTable()) - }) - } + test.run(t) } -func mustCreateNewKubeCluster(t *testing.T, name string, extraStaticLabels map[string]string) types.KubeCluster { - labels := make(map[string]string) +func testKubeServerCollection_writeText(t *testing.T) { + eksDiscoveredNameLabel := map[string]string{ + types.DiscoveredNameLabel: "cluster3", + } + kubeServers := []types.KubeServer{ + mustCreateNewKubeServer(t, "cluster1", nil), + mustCreateNewKubeServer(t, "cluster2", longLabelFixture), + mustCreateNewKubeServer(t, "afirstCluster", longLabelFixture), + mustCreateNewKubeServer(t, "cluster3-eks-us-west-1-123456789012", eksDiscoveredNameLabel), + } + test := writeTextTest{ + collection: &kubeServerCollection{servers: kubeServers}, + wantNonVerboseTable: func() string { + table := asciitable.MakeTableWithTruncatedColumn( + []string{"Cluster", "Labels", "Version"}, + [][]string{ + {"afirstCluster", formatTestLabels(staticLabelsFixture, longLabelFixture, false), api.Version}, + {"cluster1", formatTestLabels(staticLabelsFixture, nil, false), api.Version}, + {"cluster2", formatTestLabels(staticLabelsFixture, longLabelFixture, false), api.Version}, + {"cluster3", formatTestLabels(staticLabelsFixture, eksDiscoveredNameLabel, false), api.Version}, + }, + "Labels") + return table.AsBuffer().String() + }, + wantVerboseTable: func() string { + table := asciitable.MakeTable( + []string{"Cluster", "Labels", "Version"}, + []string{"afirstCluster", formatTestLabels(staticLabelsFixture, longLabelFixture, true), api.Version}, + []string{"cluster1", formatTestLabels(staticLabelsFixture, nil, true), api.Version}, + []string{"cluster2", formatTestLabels(staticLabelsFixture, longLabelFixture, true), api.Version}, + []string{"cluster3-eks-us-west-1-123456789012", formatTestLabels(staticLabelsFixture, eksDiscoveredNameLabel, true), api.Version}, + ) + return table.AsBuffer().String() + }, + } + test.run(t) +} - for k, v := range staticLabels { - labels[k] = v +func testDatabaseCollection_writeText(t *testing.T) { + rdsDiscoveredNameLabel := map[string]string{ + types.DiscoveredNameLabel: "database", + } + rdsURI := "database.abcdefghijklmnop.us-west-1.rds.amazonaws.com:5432" + databases := []types.Database{ + mustCreateNewDatabase(t, "database-A", "mysql", "localhost:3306", nil), + mustCreateNewDatabase(t, "database-B", "postgres", "localhost:5432", longLabelFixture), + mustCreateNewDatabase(t, "afirstDatabase", "redis", "localhost:6379", longLabelFixture), + mustCreateNewDatabase(t, "database-rds-us-west-1-123456789012", "postgres", + rdsURI, + rdsDiscoveredNameLabel), + } + test := writeTextTest{ + collection: &databaseCollection{databases: databases}, + wantNonVerboseTable: func() string { + table := asciitable.MakeTableWithTruncatedColumn( + []string{"Name", "Protocol", "URI", "Labels"}, + [][]string{ + {"afirstDatabase", "redis", "localhost:6379", formatTestLabels(staticLabelsFixture, longLabelFixture, false)}, + {"database", "postgres", rdsURI, formatTestLabels(staticLabelsFixture, rdsDiscoveredNameLabel, false)}, + {"database-A", "mysql", "localhost:3306", formatTestLabels(staticLabelsFixture, nil, false)}, + {"database-B", "postgres", "localhost:5432", formatTestLabels(staticLabelsFixture, longLabelFixture, false)}, + }, + "Labels") + return table.AsBuffer().String() + }, + wantVerboseTable: func() string { + table := asciitable.MakeTable( + []string{"Name", "Protocol", "URI", "Labels"}, + []string{"afirstDatabase", "redis", "localhost:6379", formatTestLabels(staticLabelsFixture, longLabelFixture, true)}, + []string{"database-A", "mysql", "localhost:3306", formatTestLabels(staticLabelsFixture, nil, true)}, + []string{"database-B", "postgres", "localhost:5432", formatTestLabels(staticLabelsFixture, longLabelFixture, true)}, + []string{"database-rds-us-west-1-123456789012", "postgres", rdsURI, formatTestLabels(staticLabelsFixture, rdsDiscoveredNameLabel, true)}, + ) + return table.AsBuffer().String() + }, } + test.run(t) +} - for k, v := range extraStaticLabels { - labels[k] = v +func testDatabaseServerCollection_writeText(t *testing.T) { + rdsDiscoveredNameLabel := map[string]string{ + types.DiscoveredNameLabel: "database", } + rdsURI := "database.abcdefghijklmnop.us-west-1.rds.amazonaws.com:5432" + dbServers := []types.DatabaseServer{ + mustCreateNewDatabaseServer(t, "database-A", "mysql", "localhost:3306", nil), + mustCreateNewDatabaseServer(t, "database-B", "postgres", "localhost:5432", longLabelFixture), + mustCreateNewDatabaseServer(t, "afirstDatabase", "redis", "localhost:6379", longLabelFixture), + mustCreateNewDatabaseServer(t, "database-rds-us-west-1-123456789012", "postgres", + rdsURI, + rdsDiscoveredNameLabel), + } + test := writeTextTest{ + collection: &databaseServerCollection{servers: dbServers}, + wantNonVerboseTable: func() string { + table := asciitable.MakeTableWithTruncatedColumn( + []string{"Host", "Name", "Protocol", "URI", "Labels", "Version"}, + [][]string{ + {"some-host", "afirstDatabase", "redis", "localhost:6379", formatTestLabels(staticLabelsFixture, longLabelFixture, false), api.Version}, + {"some-host", "database", "postgres", rdsURI, formatTestLabels(staticLabelsFixture, rdsDiscoveredNameLabel, false), api.Version}, + {"some-host", "database-A", "mysql", "localhost:3306", formatTestLabels(staticLabelsFixture, nil, false), api.Version}, + {"some-host", "database-B", "postgres", "localhost:5432", formatTestLabels(staticLabelsFixture, longLabelFixture, false), api.Version}, + }, + "Labels") + return table.AsBuffer().String() + }, + wantVerboseTable: func() string { + table := asciitable.MakeTable( + []string{"Host", "Name", "Protocol", "URI", "Labels", "Version"}, + []string{"some-host", "afirstDatabase", "redis", "localhost:6379", formatTestLabels(staticLabelsFixture, longLabelFixture, true), api.Version}, + []string{"some-host", "database-A", "mysql", "localhost:3306", formatTestLabels(staticLabelsFixture, nil, true), api.Version}, + []string{"some-host", "database-B", "postgres", "localhost:5432", formatTestLabels(staticLabelsFixture, longLabelFixture, true), api.Version}, + []string{"some-host", "database-rds-us-west-1-123456789012", "postgres", rdsURI, formatTestLabels(staticLabelsFixture, rdsDiscoveredNameLabel, true), api.Version}, + ) + return table.AsBuffer().String() + }, + } + test.run(t) +} + +func mustCreateNewDatabase(t *testing.T, name, protocol, uri string, extraStaticLabels map[string]string) *types.DatabaseV3 { + t.Helper() + db, err := types.NewDatabaseV3( + types.Metadata{ + Name: name, + Labels: makeTestLabels(extraStaticLabels), + }, + types.DatabaseSpecV3{ + Protocol: protocol, + URI: uri, + DynamicLabels: map[string]types.CommandLabelV2{ + "date": { + Period: types.NewDuration(1 * time.Second), + Command: []string{"date"}, + Result: "Tue 11 Oct 2022 10:21:58 WEST", + }, + }, + }, + ) + require.NoError(t, err) + return db +} +func mustCreateNewDatabaseServer(t *testing.T, name, protocol, uri string, extraStaticLabels map[string]string) types.DatabaseServer { + t.Helper() + dbServer, err := types.NewDatabaseServerV3( + types.Metadata{ + Name: name, + Labels: makeTestLabels(extraStaticLabels), + }, types.DatabaseServerSpecV3{ + HostID: "some-hostid", + Hostname: "some-host", + Database: mustCreateNewDatabase(t, name, protocol, uri, extraStaticLabels), + }) + require.NoError(t, err) + + return dbServer +} + +func mustCreateNewKubeCluster(t *testing.T, name string, extraStaticLabels map[string]string) *types.KubernetesClusterV3 { + t.Helper() cluster, err := types.NewKubernetesClusterV3( types.Metadata{ Name: name, - Labels: labels, + Labels: makeTestLabels(extraStaticLabels), }, types.KubernetesClusterSpecV3{ DynamicLabels: map[string]types.CommandLabelV2{ @@ -139,6 +306,14 @@ func mustCreateNewKubeCluster(t *testing.T, name string, extraStaticLabels map[s return cluster } +func mustCreateNewKubeServer(t *testing.T, name string, extraStaticLabels map[string]string) types.KubeServer { + t.Helper() + cluster := mustCreateNewKubeCluster(t, name, extraStaticLabels) + kubeServer, err := types.NewKubernetesServerV3FromCluster(cluster, "some-host", "some-hostid") + require.NoError(t, err) + return kubeServer +} + func formatTestLabels(l1, l2 map[string]string, verbose bool) string { labels := map[string]string{ "date": "Tue 11 Oct 2022 10:21:58 WEST", @@ -152,3 +327,14 @@ func formatTestLabels(l1, l2 map[string]string, verbose bool) string { } return stripInternalTeleportLabels(verbose, labels) } + +func makeTestLabels(extraStaticLabels map[string]string) map[string]string { + labels := make(map[string]string) + for k, v := range staticLabelsFixture { + labels[k] = v + } + for k, v := range extraStaticLabels { + labels[k] = v + } + return labels +} diff --git a/tool/tctl/common/db_command.go b/tool/tctl/common/db_command.go index e3eae83178113..f0f5799de7d98 100644 --- a/tool/tctl/common/db_command.go +++ b/tool/tctl/common/db_command.go @@ -84,24 +84,17 @@ func (c *DBCommand) ListDatabases(ctx context.Context, clt auth.ClientI) error { return trace.Wrap(err) } - var servers []types.DatabaseServer - resources, err := client.GetResourcesWithFilters(ctx, clt, proto.ListResourcesRequest{ + servers, err := client.GetAllResources[types.DatabaseServer](ctx, clt, &proto.ListResourcesRequest{ ResourceType: types.KindDatabaseServer, Labels: labels, PredicateExpression: c.predicateExpr, SearchKeywords: libclient.ParseSearchKeywords(c.searchKeywords, ','), }) - switch { - case err != nil: + if err != nil { if utils.IsPredicateError(err) { return trace.Wrap(utils.PredicateError{Err: err}) } return trace.Wrap(err) - default: - servers, err = types.ResourcesWithLabels(resources).AsDatabaseServers() - if err != nil { - return trace.Wrap(err) - } } coll := &databaseServerCollection{servers: servers} diff --git a/tool/tctl/common/kube_command.go b/tool/tctl/common/kube_command.go index 87b96e27365a2..d89face482bf8 100644 --- a/tool/tctl/common/kube_command.go +++ b/tool/tctl/common/kube_command.go @@ -25,8 +25,13 @@ import ( "github.com/gravitational/trace" "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api/client" + "github.com/gravitational/teleport/api/client/proto" + "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/auth" + libclient "github.com/gravitational/teleport/lib/client" "github.com/gravitational/teleport/lib/service/servicecfg" + "github.com/gravitational/teleport/lib/utils" ) // KubeCommand implements "tctl kube" group of commands. @@ -36,6 +41,10 @@ type KubeCommand struct { // format is the output format (text or yaml) format string + searchKeywords string + predicateExpr string + labels string + // verbose sets whether full table output should be shown for labels verbose bool @@ -49,8 +58,11 @@ func (c *KubeCommand) Initialize(app *kingpin.Application, config *servicecfg.Co kube := app.Command("kube", "Operate on registered Kubernetes clusters.") c.kubeList = kube.Command("ls", "List all Kubernetes clusters registered with the cluster.") + c.kubeList.Arg("labels", labelHelp).StringVar(&c.labels) c.kubeList.Flag("format", "Output format, 'text', 'json', or 'yaml'").Default(teleport.Text).StringVar(&c.format) c.kubeList.Flag("verbose", "Verbose table output, shows full label output").Short('v').BoolVar(&c.verbose) + c.kubeList.Flag("search", searchHelp).StringVar(&c.searchKeywords) + c.kubeList.Flag("query", queryHelp).StringVar(&c.predicateExpr) } // TryRun attempts to run subcommands like "kube ls". @@ -66,10 +78,22 @@ func (c *KubeCommand) TryRun(ctx context.Context, cmd string, client auth.Client // ListKube prints the list of kube clusters that have recently sent heartbeats // to the cluster. -func (c *KubeCommand) ListKube(ctx context.Context, client auth.ClientI) error { +func (c *KubeCommand) ListKube(ctx context.Context, clt auth.ClientI) error { + labels, err := libclient.ParseLabelSpec(c.labels) + if err != nil { + return trace.Wrap(err) + } - kubes, err := client.GetKubernetesServers(ctx) + kubes, err := client.GetAllResources[types.KubeServer](ctx, clt, &proto.ListResourcesRequest{ + ResourceType: types.KindKubeServer, + Labels: labels, + PredicateExpression: c.predicateExpr, + SearchKeywords: libclient.ParseSearchKeywords(c.searchKeywords, ','), + }) if err != nil { + if utils.IsPredicateError(err) { + return trace.Wrap(utils.PredicateError{Err: err}) + } return trace.Wrap(err) }