Skip to content

autoupdate canary support: modulate proxy response #56468

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hugoShaka merged 1 commit into from
Jul 8, 2025
Merged

autoupdate canary support: modulate proxy response #56468

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion lib/web/apiserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1619,6 +1619,7 @@ func (h *Handler) ping(w http.ResponseWriter, r *http.Request, p httprouter.Para
}

group := r.URL.Query().Get(webclient.AgentUpdateGroupParameter)
updaterID := r.URL.Query().Get(webclient.AgentUpdateIDParameter)

return webclient.PingResponse{
Auth: authSettings,
Expand All @@ -1627,7 +1628,7 @@ func (h *Handler) ping(w http.ResponseWriter, r *http.Request, p httprouter.Para
MinClientVersion: teleport.MinClientSemVer().String(),
ClusterName: h.auth.clusterName,
AutomaticUpgrades: pr.ServerFeatures.GetAutomaticUpgrades(),
AutoUpdate: h.automaticUpdateSettings184(r.Context(), group, "" /* updater UUID */),
AutoUpdate: h.automaticUpdateSettings184(r.Context(), group, updaterID),
Edition: modules.GetModules().BuildType(),
FIPS: modules.IsBoringBinary(),
}, nil
Expand Down Expand Up @@ -1666,6 +1667,14 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, p httprouter.Para
if err != nil {
return nil, trace.Wrap(err)
}

// Now we modulate the autoupdate answer on a per-request basis.
// We don't want to cache one answer per updater UUID, so we take the
// cached result and just override what we must.
updaterID := r.URL.Query().Get(webclient.AgentUpdateIDParameter)
if updaterID != "" {
resp.AutoUpdate = h.automaticUpdateSettings184(r.Context(), group, updaterID)
}
return resp, nil
}

Expand Down
227 changes: 202 additions & 25 deletions lib/web/apiserver_ping_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"net/url"
"testing"

"github.com/google/uuid"
"github.com/gravitational/roundtrip"
"github.com/gravitational/trace"
"github.com/stretchr/testify/assert"
Expand All @@ -34,6 +35,7 @@ import (
"github.com/gravitational/teleport/api/client/webclient"
"github.com/gravitational/teleport/api/constants"
autoupdatev1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1"
headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1"
"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/api/types/autoupdate"
"github.com/gravitational/teleport/lib/client"
Expand Down Expand Up @@ -283,22 +285,22 @@ func TestPing_minimalAPI(t *testing.T) {

// TestPing_autoUpdateResources tests that find endpoint return correct data related to auto updates.
func TestPing_autoUpdateResources(t *testing.T) {
env := newWebPack(t, 1, func(cfg *proxyConfig) {
cfg.minimalHandler = true
})
env := newWebPack(t, 1)
proxy := env.proxies[0]

ctx := t.Context()

req, err := http.NewRequest(http.MethodGet, proxy.newClient(t).Endpoint("webapi", "find"), nil)
require.NoError(t, err)
req.Host = proxy.handler.handler.cfg.ProxyPublicAddrs[0].Host()
testGroup := "test-group"
testUpdaterID := uuid.NewString()

// Note: we are not using webclient.Ping() here because we don't have a valid certificate for 127.0.0.1 and
// The webclient doesn't support being passed custom transport.
clt := proxy.newClient(t)
tests := []struct {
name string
config *autoupdatev1pb.AutoUpdateConfigSpec
version *autoupdatev1pb.AutoUpdateVersionSpec
rollout *autoupdatev1pb.AutoUpdateAgentRolloutSpec
rollout *autoupdatev1pb.AutoUpdateAgentRollout
cleanup bool
expected webclient.AutoUpdateSettings
}{
Expand Down Expand Up @@ -330,12 +332,17 @@ func TestPing_autoUpdateResources(t *testing.T) {
},
{
name: "enable agent auto update, immediate schedule",
rollout: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{
AutoupdateMode: autoupdate.AgentsUpdateModeEnabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
Schedule: autoupdate.AgentsScheduleImmediate,
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
rollout: &autoupdatev1pb.AutoUpdateAgentRollout{
Metadata: &headerv1.Metadata{
Name: types.MetaNameAutoUpdateAgentRollout,
},
Spec: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{
AutoupdateMode: autoupdate.AgentsUpdateModeEnabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
Schedule: autoupdate.AgentsScheduleImmediate,
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
},
},
expected: webclient.AutoUpdateSettings{
ToolsVersion: api.Version,
Expand All @@ -348,12 +355,17 @@ func TestPing_autoUpdateResources(t *testing.T) {
},
{
name: "agent rollout present but AU mode is disabled",
rollout: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{
AutoupdateMode: autoupdate.AgentsUpdateModeDisabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
Schedule: autoupdate.AgentsScheduleImmediate,
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
rollout: &autoupdatev1pb.AutoUpdateAgentRollout{
Metadata: &headerv1.Metadata{
Name: types.MetaNameAutoUpdateAgentRollout,
},
Spec: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{
AutoupdateMode: autoupdate.AgentsUpdateModeDisabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
Schedule: autoupdate.AgentsScheduleImmediate,
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
},
},
expected: webclient.AutoUpdateSettings{
ToolsVersion: api.Version,
Expand Down Expand Up @@ -432,6 +444,171 @@ func TestPing_autoUpdateResources(t *testing.T) {
AgentAutoUpdate: false,
AgentVersion: api.Version,
},
cleanup: true,
},
{
name: "group must be updated",
rollout: &autoupdatev1pb.AutoUpdateAgentRollout{
Metadata: &headerv1.Metadata{
Name: types.MetaNameAutoUpdateAgentRollout,
},
Spec: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{
AutoupdateMode: autoupdate.AgentsUpdateModeEnabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
Schedule: autoupdate.AgentsScheduleRegular,
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
},
Status: &autoupdatev1pb.AutoUpdateAgentRolloutStatus{
Groups: []*autoupdatev1pb.AutoUpdateAgentRolloutStatusGroup{
{
Name: testGroup,
State: autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE,
ConfigDays: []string{"*"},
},
{
Name: "unstarted",
State: autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED,
ConfigDays: []string{"*"},
},
},
},
},
expected: webclient.AutoUpdateSettings{
ToolsVersion: api.Version,
ToolsAutoUpdate: false,
AgentVersion: "1.2.4",
AgentAutoUpdate: true,
AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds,
},
cleanup: true,
},
{
name: "group must not be updated",
rollout: &autoupdatev1pb.AutoUpdateAgentRollout{
Metadata: &headerv1.Metadata{
Name: types.MetaNameAutoUpdateAgentRollout,
},
Spec: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{
AutoupdateMode: autoupdate.AgentsUpdateModeEnabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
Schedule: autoupdate.AgentsScheduleRegular,
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
},
Status: &autoupdatev1pb.AutoUpdateAgentRolloutStatus{
Groups: []*autoupdatev1pb.AutoUpdateAgentRolloutStatusGroup{
{
Name: "done",
State: autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE,
ConfigDays: []string{"*"},
},
{
Name: testGroup,
State: autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED,
ConfigDays: []string{"*"},
},
},
},
},
expected: webclient.AutoUpdateSettings{
ToolsVersion: api.Version,
ToolsAutoUpdate: false,
AgentVersion: "1.2.3",
AgentAutoUpdate: false,
AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds,
},
cleanup: true,
},
{
name: "canary must be updated",
rollout: &autoupdatev1pb.AutoUpdateAgentRollout{
Metadata: &headerv1.Metadata{
Name: types.MetaNameAutoUpdateAgentRollout,
},
Spec: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{
AutoupdateMode: autoupdate.AgentsUpdateModeEnabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
Schedule: autoupdate.AgentsScheduleRegular,
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
},
Status: &autoupdatev1pb.AutoUpdateAgentRolloutStatus{
Groups: []*autoupdatev1pb.AutoUpdateAgentRolloutStatusGroup{
{
Name: testGroup,
State: autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_CANARY,
ConfigDays: []string{"*"},
Canaries: []*autoupdatev1pb.Canary{
{
UpdaterId: testUpdaterID,
HostId: uuid.NewString(),
Hostname: "test-host",
Success: false,
},
},
},
{
Name: "unstarted",
State: autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED,
ConfigDays: []string{"*"},
},
},
},
},
expected: webclient.AutoUpdateSettings{
ToolsVersion: api.Version,
ToolsAutoUpdate: false,
AgentVersion: "1.2.4",
AgentAutoUpdate: true,
AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds,
},
cleanup: true,
},
{
name: "canary must not be updated",
rollout: &autoupdatev1pb.AutoUpdateAgentRollout{
Metadata: &headerv1.Metadata{
Name: types.MetaNameAutoUpdateAgentRollout,
},
Spec: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{
AutoupdateMode: autoupdate.AgentsUpdateModeEnabled,
Strategy: autoupdate.AgentsStrategyHaltOnError,
Schedule: autoupdate.AgentsScheduleRegular,
StartVersion: "1.2.3",
TargetVersion: "1.2.4",
},
Status: &autoupdatev1pb.AutoUpdateAgentRolloutStatus{
Groups: []*autoupdatev1pb.AutoUpdateAgentRolloutStatusGroup{
{
Name: testGroup,
State: autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_CANARY,
Canaries: []*autoupdatev1pb.Canary{
{
UpdaterId: uuid.NewString(),
HostId: uuid.NewString(),
Hostname: "test-host",
Success: false,
},
},
ConfigDays: []string{"*"},
},
{
Name: "unstarted",
State: autoupdatev1pb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED,
ConfigDays: []string{"*"},
},
},
},
},
expected: webclient.AutoUpdateSettings{
ToolsVersion: api.Version,
ToolsAutoUpdate: false,
AgentVersion: "1.2.3",
AgentAutoUpdate: false,
AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds,
},
cleanup: true,
},
}
for _, tc := range tests {
Expand All @@ -449,9 +626,7 @@ func TestPing_autoUpdateResources(t *testing.T) {
require.NoError(t, err)
}
if tc.rollout != nil {
rollout, err := autoupdate.NewAutoUpdateAgentRollout(tc.rollout)
require.NoError(t, err)
_, err = env.server.Auth().UpsertAutoUpdateAgentRollout(ctx, rollout)
_, err := env.server.Auth().UpsertAutoUpdateAgentRollout(ctx, tc.rollout)
require.NoError(t, err)
}

Expand All @@ -460,12 +635,14 @@ func TestPing_autoUpdateResources(t *testing.T) {
proxy.clock.Advance(2 * findEndpointCacheTTL)
}

resp, err := client.NewInsecureWebClient().Do(req)
resp, err := clt.Get(ctx, clt.Endpoint("webapi", "ping"), url.Values{
webclient.AgentUpdateIDParameter: []string{testUpdaterID},
webclient.AgentUpdateGroupParameter: []string{testGroup},
})
require.NoError(t, err)

pr := &webclient.PingResponse{}
require.NoError(t, json.NewDecoder(resp.Body).Decode(pr))
require.NoError(t, resp.Body.Close())
require.NoError(t, json.NewDecoder(resp.Reader()).Decode(pr))

assert.Equal(t, tc.expected, pr.AutoUpdate)

Expand Down
19 changes: 19 additions & 0 deletions lib/web/autoupdate_common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,28 @@ func getVersionFromRollout(
case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE,
autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE:
return version.EnsureSemver(rollout.GetSpec().GetTargetVersion())
case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_CANARY:
if updaterIsCanary(group, updaterUUID) {
return version.EnsureSemver(rollout.GetSpec().GetTargetVersion())
}
return version.EnsureSemver(rollout.GetSpec().GetStartVersion())
default:
return nil, trace.NotImplemented("unsupported group state %q", group.GetState())
}
}

func updaterIsCanary(group *autoupdatepb.AutoUpdateAgentRolloutStatusGroup, updaterUUID string) bool {
if updaterUUID == "" {
return false
}
for _, canary := range group.GetCanaries() {
Copy link
Copy Markdown
Member

@sclevine sclevine Jul 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Could use slices.ContainsFunc to save a few lines

Copy link
Copy Markdown
Contributor Author

@hugoShaka hugoShaka Jul 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried the ContainsFunc variant and find it adds more complexity for a return condition this trivial, I'm not gaining much space from rewriting this so I'd prefer leaving the for loop.

if canary.UpdaterId == updaterUUID {
return true
}
}
return false
}

// getTriggerFromRollout returns the version we should serve to the agent based
// on the RFD184 agent rollout, the agent group name, and its UUID.
// This logic is pretty complex and described in RFD 184.
Expand Down Expand Up @@ -166,6 +183,8 @@ func getTriggerFromRollout(rollout *autoupdatepb.AutoUpdateAgentRollout, groupNa
return true, nil
case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE:
return rollout.GetSpec().GetStrategy() == autoupdate.AgentsStrategyHaltOnError, nil
case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_CANARY:
return updaterIsCanary(group, updaterUUID), nil
default:
return false, trace.NotImplemented("Unsupported group state %q", group.GetState())
}
Expand Down
Loading
Loading