Backport of batch: reduce server overhead when dispatching jobs from CLI into release/1.11.x

.changelog/27631.txt

@@ -0,0 +1,7 @@
+```release-note:improvement
+cli: Reduced server overhead when dispatching jobs or forcing periodic jobs from the CLI
+```
+
+```release-note:improvement
+cli: Truncate results when job commands return a large set of jobs that match the provided ID prefix
+```

command/alloc_exec.go

@@ -167,7 +167,7 @@ func (l *AllocExecCommand) Run(args []string) int {
 
 	var allocStub *api.AllocationListStub
 	if job {
-		jobID, ns, err := l.JobIDByPrefix(client, args[0], nil)
+		jobID, ns, err := l.JobIDByPrefix(client, args[0], "")
 		if err != nil {
 			l.Ui.Error(err.Error())
 			return 1

command/alloc_fs.go

@@ -171,7 +171,7 @@ func (f *AllocFSCommand) Run(args []string) int {
 	// If -job is specified, use random allocation, otherwise use provided allocation
 	allocID := args[0]
 	if job {
-		jobID, ns, err := f.JobIDByPrefix(client, args[0], nil)
+		jobID, ns, err := f.JobIDByPrefix(client, args[0], "")
 		if err != nil {
 			f.Ui.Error(err.Error())
 			return 1

command/alloc_logs.go

@@ -172,7 +172,7 @@ func (l *AllocLogsCommand) Run(args []string) int {
 	// If -job is specified, use random allocation, otherwise use provided allocation
 	allocID := args[0]
 	if l.job {
-		jobID, ns, err := l.JobIDByPrefix(client, args[0], nil)
+		jobID, ns, err := l.JobIDByPrefix(client, args[0], "")
 		if err != nil {
 			l.Ui.Error(err.Error())
 			return 1

command/job_action.go

@@ -32,10 +32,10 @@ Usage: nomad job action [options] <action>
 
   Perform a predefined command inside the environment of the given allocation
   and job, or given task, group and job.
-  
+
   Either an allocation or a task and group must be provided; for example, either
   of the following will work:
-  
+
     nomad job action -alloc=<alloc-id> -job=<job-name> <action>
     nomad job action -task=<task-name> -group=<group-name> -job=<job-name> <action>
 
@@ -182,7 +182,7 @@ func (c *JobActionCommand) Run(args []string) int {
 			return 1
 		}
 
-		jobID, ns, err := c.JobIDByPrefix(client, job, nil)
+		jobID, ns, err := c.JobIDByPrefix(client, job, "")
 		if err != nil {
 			c.Ui.Error(err.Error())
 			return 1

command/job_allocs.go

@@ -99,7 +99,7 @@ func (c *JobAllocsCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_deployments.go

@@ -104,7 +104,7 @@ func (c *JobDeploymentsCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_dispatch.go

@@ -4,6 +4,7 @@
 package command
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"net/url"
@@ -189,12 +190,14 @@ func (c *JobDispatchCommand) Run(args []string) int {
 		return 1
 	}
 
-	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, func(j *api.JobListStub) bool {
-		return j.ParameterizedJob
-	})
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix,
+		`ParentID == "" and ParameterizedJob is not nil`)
 	if err != nil {
+		var noPrefixErr *NoJobWithPrefixError
+		if errors.As(err, &noPrefixErr) {
+			err = fmt.Errorf("No parameterized job(s) with prefix or ID %q found", jobIDPrefix)
+		}
 		c.Ui.Error(err.Error())
 		return 1
 	}

command/job_eval.go

@@ -107,7 +107,7 @@ func (c *JobEvalCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_history.go

@@ -150,7 +150,7 @@ func (c *JobHistoryCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_inspect.go

@@ -133,7 +133,7 @@ func (c *JobInspectCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_periodic_force.go

@@ -120,9 +120,7 @@ func (c *JobPeriodicForceCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, func(j *api.JobListStub) bool {
-		return j.Periodic
-	})
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "Periodic is not nil")
 	if err != nil {
 		var noPrefixErr *NoJobWithPrefixError
 		if errors.As(err, &noPrefixErr) {

command/job_promote.go

@@ -111,7 +111,7 @@ func (c *JobPromoteCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_restart.go

@@ -256,7 +256,7 @@ func (c *JobRestartCommand) Run(args []string) int {
 	}
 
 	// Use prefix matching to find job.
-	job, err := c.JobByPrefix(c.client, c.jobID, nil)
+	job, err := c.JobByPrefix(c.client, c.jobID, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_revert.go

@@ -117,7 +117,7 @@ func (c *JobRevertCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_scale.go

@@ -128,7 +128,7 @@ func (j *JobScaleCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := j.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := j.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		j.Ui.Error(err.Error())
 		return 1

command/job_scaling_events.go

@@ -90,7 +90,7 @@ func (j *JobScalingEventsCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := j.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := j.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		j.Ui.Error(err.Error())
 		return 1

command/job_start.go

@@ -104,7 +104,7 @@ func (c *JobStartCommand) Run(args []string) int {
 		length = fullId
 	}
 
-	job, err := c.JobByPrefix(client, jobIDPrefix, nil)
+	job, err := c.JobByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_status.go

@@ -206,7 +206,7 @@ func (c *JobStatusCommand) Run(args []string) int {
 
 	// Try querying the job
 	jobIDPrefix := strings.TrimSpace(args[0])
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_stop.go

@@ -148,7 +148,7 @@ func (c *JobStopCommand) Run(args []string) int {
 			}
 
 			// Check if the job exists
-			job, err := c.JobByPrefix(client, jobID, nil)
+			job, err := c.JobByPrefix(client, jobID, "")
 			if err != nil {
 				c.Ui.Error(err.Error())
 				statusCh <- 1

command/job_tag_apply.go

@@ -114,7 +114,7 @@ func (c *JobTagApplyCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(job)
-	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/job_tag_unset.go

@@ -94,7 +94,7 @@ func (c *JobTagUnsetCommand) Run(args []string) int {
 
 	// Check if the job exists
 	jobIDPrefix := strings.TrimSpace(job)
-	jobID, _, err := c.JobIDByPrefix(client, jobIDPrefix, nil)
+	jobID, _, err := c.JobIDByPrefix(client, jobIDPrefix, "")
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1

command/meta.go

@@ -311,10 +311,6 @@ func (m *Meta) FormatWarnings(header string, warnings string) string {
 		))
 }
 
-// JobByPrefixFilterFunc is a function used to filter jobs when performing a
-// prefix match. Only jobs that return true are included in the prefix match.
-type JobByPrefixFilterFunc func(*api.JobListStub) bool
-
 // NoJobWithPrefixError is the error returned when the job prefix doesn't
 // return any matches.
 type NoJobWithPrefixError struct {
@@ -328,7 +324,7 @@ func (e *NoJobWithPrefixError) Error() string {
 // JobByPrefix returns the job that best matches the given prefix. Returns an
 // error if there are no matches or if there are more than one exact match
 // across namespaces.
-func (m *Meta) JobByPrefix(client *api.Client, prefix string, filter JobByPrefixFilterFunc) (*api.Job, error) {
+func (m *Meta) JobByPrefix(client *api.Client, prefix string, filter string) (*api.Job, error) {
 	jobID, namespace, err := m.JobIDByPrefix(client, prefix, filter)
 	if err != nil {
 		return nil, err
@@ -348,37 +344,36 @@ func (m *Meta) JobByPrefix(client *api.Client, prefix string, filter JobByPrefix
 // Returns the prefix itself if job prefix search is not allowed and an error
 // if there are no matches or if there are more than one exact match across
 // namespaces.
-func (m *Meta) JobIDByPrefix(client *api.Client, prefix string, filter JobByPrefixFilterFunc) (string, string, error) {
-	// Search job by prefix. Return an error if there is not an exact match.
-	jobs, _, err := client.Jobs().PrefixList(prefix)
+func (m *Meta) JobIDByPrefix(client *api.Client, prefix string, filter string) (string, string, error) {
+	maxResults := 20 // reduce load for large sets
+	jobs, _, err := client.Jobs().ListOptions(nil, &api.QueryOptions{
+		Prefix:  prefix,
+		Filter:  filter,
+		PerPage: int32(maxResults),
+	})
 	if err != nil {
 		if strings.Contains(err.Error(), api.PermissionDeniedErrorContent) {
 			return prefix, "", nil
 		}
 		return "", "", fmt.Errorf("Error querying job prefix %q: %s", prefix, err)
 	}
 
-	if filter != nil {
-		var filtered []*api.JobListStub
-		for _, j := range jobs {
-			if filter(j) {
-				filtered = append(filtered, j)
-			}
-		}
-		jobs = filtered
-	}
-
 	if len(jobs) == 0 {
 		return "", "", &NoJobWithPrefixError{Prefix: prefix}
 	}
 	if len(jobs) > 1 {
 		exactMatch := prefix == jobs[0].ID
 		matchInMultipleNamespaces := m.allNamespaces() && jobs[0].ID == jobs[1].ID
+		truncatedMsg := ""
+		if len(jobs) >= maxResults {
+			truncatedMsg = "\n(results may be truncated)"
+		}
 		if !exactMatch || matchInMultipleNamespaces {
 			return "", "", fmt.Errorf(
-				"Prefix %q matched multiple jobs\n\n%s",
+				"Prefix %q matched multiple jobs\n\n%s%s",
 				prefix,
 				createStatusListOutput(jobs, m.allNamespaces()),
+				truncatedMsg,
 			)
 		}
 	}

command/meta_test.go

@@ -202,7 +202,7 @@ func TestMeta_JobByPrefix(t *testing.T) {
 	testCases := []struct {
 		name          string
 		prefix        string
-		filterFunc    JobByPrefixFilterFunc
+		filter        string
 		expectedError string
 	}{
 		{
@@ -216,10 +216,8 @@ func TestMeta_JobByPrefix(t *testing.T) {
 		{
 			name:   "match with filter",
 			prefix: "job-",
-			filterFunc: func(j *api.JobListStub) bool {
-				// Filter out jobs with "job-" so that only "job-2" matches.
-				return j.ID == "job-2"
-			},
+			// Filter out jobs so that only "job-2" matches.
+			filter: `ID == "job-2"`,
 		},
 		{
 			name:          "multiple matches",
@@ -240,7 +238,7 @@ func TestMeta_JobByPrefix(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			job, err := meta.JobByPrefix(client, tc.prefix, tc.filterFunc)
+			job, err := meta.JobByPrefix(client, tc.prefix, tc.filter)
 			if tc.expectedError != "" {
 				must.Nil(t, job)
 				must.ErrorContains(t, err, tc.expectedError)

go.mod

@@ -49,7 +49,7 @@ require (
 	github.com/hashicorp/consul/api v1.33.4
 	github.com/hashicorp/consul/sdk v0.17.2
 	github.com/hashicorp/cronexpr v1.1.3
-	github.com/hashicorp/go-bexpr v0.1.15
+	github.com/hashicorp/go-bexpr v0.1.16
 	github.com/hashicorp/go-checkpoint v0.5.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-connlimit v0.3.1

go.sum

@@ -424,8 +424,8 @@ github.com/hashicorp/cronexpr v1.1.3/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-bexpr v0.1.15 h1:flTYJAqZAlK+t8ezezb6WQGlRO1D4+GEF/HmH+xZo5k=
-github.com/hashicorp/go-bexpr v0.1.15/go.mod h1:HGKbAByHn2aJWUV47gL7+IjLK79iU3EZIbOwCXJZLoE=
+github.com/hashicorp/go-bexpr v0.1.16 h1:D+fKoGyUzXVS0FdjOX1ws3vIck8DVtBqQ0tsusmYDR8=
+github.com/hashicorp/go-bexpr v0.1.16/go.mod h1:HGKbAByHn2aJWUV47gL7+IjLK79iU3EZIbOwCXJZLoE=
 github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU=
 github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=