Skip to content

autoupdate canary support: tctl #56473

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hugoShaka merged 2 commits into from
Jul 7, 2025
Merged

autoupdate canary support: tctl #56473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b1595cb
autoupdate canary support: tctl support
hugoShaka Jul 4, 2025
8231029
add `--force` flag to `tctl autoupdate agents start-update`
hugoShaka Jul 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions tool/tctl/common/autoupdate_command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ type AutoUpdateCommand struct {
groups []string

clear bool
force bool

// used for testing purposes
now func() time.Time
Expand Down Expand Up @@ -101,6 +102,7 @@ func (c *AutoUpdateCommand) Initialize(app *kingpin.Application, ccf *tctlcfg.Gl
c.agentsReportCmd = agentsCmd.Command("report", "Aggregates the agent autoupdate reports and displays agent count per version and per update group.")
c.agentsStartUpdateCmd = agentsCmd.Command("start-update", "Starts updating one or many groups.")
c.agentsStartUpdateCmd.Arg("groups", "Groups to start updating.").StringsVar(&c.groups)
c.agentsStartUpdateCmd.Flag("force", "Skips progressive deployment mechanism such as canaries or backpressure.").BoolVar(&c.force)
c.agentsMarkDoneCmd = agentsCmd.Command("mark-done", "Marks one or many groups as done updating.")
c.agentsMarkDoneCmd.Arg("groups", "Groups to mark as done updating.").StringsVar(&c.groups)
c.agentsRollbackCmd = agentsCmd.Command("rollback", "Rolls back one or many groups.")
Expand Down Expand Up @@ -401,9 +403,22 @@ func rolloutGroupTable(rollout *autoupdatev1pb.AutoUpdateAgentRollout, writer io
if i == len(groups)-1 {
groupName = groupName + " (catch-all)"
}
state := userFriendlyState(group.GetState())

// If this is the canary state, we annotate the group state with the canary progress
if group.GetState() == autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_CANARY {
successCount := 0
for _, canary := range group.Canaries {
if canary.Success {
successCount++
}
}
state = fmt.Sprintf("%s (%d/%d)", state, successCount, group.CanaryCount)
}

table.AddRow([]string{
groupName,
userFriendlyState(group.GetState()),
state,
formatTimeIfNotEmpty(group.GetStartTime().AsTime(), time.DateTime),
group.GetLastUpdateReason(),
strconv.FormatUint(groupCount, 10),
Expand Down Expand Up @@ -440,7 +455,11 @@ func (c *AutoUpdateCommand) agentsStartUpdateCommand(ctx context.Context, client
return trace.BadParameter("no groups specified")
}

rollout, err := client.TriggerAutoUpdateAgentGroup(ctx, groups, autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED)
state := autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED
if c.force {
state = autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE
}
rollout, err := client.TriggerAutoUpdateAgentGroup(ctx, groups, state)
if err != nil {
return trace.Wrap(err)
}
Expand Down Expand Up @@ -521,6 +540,8 @@ func userFriendlyState[T autoupdatev1pb.AutoUpdateAgentGroupState | autoupdatev1
return "Done"
case 4:
return "Rolledback"
case 5:
return "Canary"
default:
// If we don't know anything about this state, we display its integer
return fmt.Sprintf("Unknown state (%d)", state)
Expand Down
83 changes: 83 additions & 0 deletions tool/tctl/common/autoupdate_command_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"testing"
"time"

"github.com/google/uuid"
"github.com/gravitational/trace"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
Expand Down Expand Up @@ -346,6 +347,88 @@ Group Name State Start Time State Reason Agent Count Up-to-
dev Done 2025-01-15 12:00:00 outside_window 1023 567
stage Active 2025-01-15 14:00:00 in_window 0 0
prod (catch-all) Unstarted outside_window 789 0
`,
},
{
name: "rollout regular schedule halt-on-error with progress, with canary",
fixture: &autoupdatepb.AutoUpdateAgentRollout{
Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
Schedule: autoupdate.AgentsScheduleRegular,
AutoupdateMode: autoupdate.AgentsUpdateModeEnabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
},
Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{
Groups: []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{
{
Name: "dev",
StartTime: timestamppb.New(time.Date(2025, 1, 15, 12, 00, 0, 0, time.UTC)),
State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE,
LastUpdateTime: nil,
LastUpdateReason: "outside_window",
ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"},
ConfigStartHour: 8,
PresentCount: 1023,
UpToDateCount: 567,
InitialCount: 1012,
},
{
Name: "stage",
StartTime: timestamppb.New(time.Date(2025, 1, 15, 14, 00, 0, 0, time.UTC)),
State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_CANARY,
LastUpdateReason: "in_window",
ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"},
ConfigStartHour: 14,
CanaryCount: 5,
Canaries: []*autoupdatepb.Canary{
{
UpdaterId: uuid.NewString(),
HostId: uuid.NewString(),
Hostname: "host-1",
Success: true,
},
{
UpdaterId: uuid.NewString(),
HostId: uuid.NewString(),
Hostname: "host-2",
Success: false,
},
{
UpdaterId: uuid.NewString(),
HostId: uuid.NewString(),
Hostname: "host-3",
Success: true,
},
},
},
{
Name: "prod",
StartTime: nil,
State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED,
LastUpdateReason: "outside_window",
ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"},
ConfigStartHour: 18,
PresentCount: 789,
},
},
State: autoupdatepb.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE,
StartTime: timestamppb.New(time.Date(2025, 1, 15, 2, 0, 0, 0, time.UTC)),
TimeOverride: nil,
},
},
expectedOutput: `Agent autoupdate mode: enabled
Rollout creation date: 2025-01-15 02:00:00
Start version: 1.2.3
Target version: 1.2.4
Rollout state: Active
Strategy: halt-on-error

Group Name State Start Time State Reason Agent Count Up-to-date
---------------- ------------ ------------------- -------------- ----------- ----------
dev Done 2025-01-15 12:00:00 outside_window 1023 567
stage Canary (2/5) 2025-01-15 14:00:00 in_window 0 0
prod (catch-all) Unstarted outside_window 789 0
`,
},
}
Expand Down
Loading